1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // archtecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all int registers (excluding RSP)
  245 reg_class int_reg %{
  246   return _INT_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RAX, RDX, and RSP)
  250 reg_class int_no_rax_rdx_reg %{
  251   return _INT_NO_RAX_RDX_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RCX and RSP)
  255 reg_class int_no_rcx_reg %{
  256   return _INT_NO_RCX_REG_mask;
  257 %}
  258 
  259 // Singleton class for RAX pointer register
  260 reg_class ptr_rax_reg(RAX, RAX_H);
  261 
  262 // Singleton class for RBX pointer register
  263 reg_class ptr_rbx_reg(RBX, RBX_H);
  264 
  265 // Singleton class for RSI pointer register
  266 reg_class ptr_rsi_reg(RSI, RSI_H);
  267 
  268 // Singleton class for RBP pointer register
  269 reg_class ptr_rbp_reg(RBP, RBP_H);
  270 
  271 // Singleton class for RDI pointer register
  272 reg_class ptr_rdi_reg(RDI, RDI_H);
  273 
  274 // Singleton class for stack pointer
  275 reg_class ptr_rsp_reg(RSP, RSP_H);
  276 
  277 // Singleton class for TLS pointer
  278 reg_class ptr_r15_reg(R15, R15_H);
  279 
  280 // Singleton class for RAX long register
  281 reg_class long_rax_reg(RAX, RAX_H);
  282 
  283 // Singleton class for RCX long register
  284 reg_class long_rcx_reg(RCX, RCX_H);
  285 
  286 // Singleton class for RDX long register
  287 reg_class long_rdx_reg(RDX, RDX_H);
  288 
  289 // Singleton class for RAX int register
  290 reg_class int_rax_reg(RAX);
  291 
  292 // Singleton class for RBX int register
  293 reg_class int_rbx_reg(RBX);
  294 
  295 // Singleton class for RCX int register
  296 reg_class int_rcx_reg(RCX);
  297 
  298 // Singleton class for RCX int register
  299 reg_class int_rdx_reg(RDX);
  300 
  301 // Singleton class for RCX int register
  302 reg_class int_rdi_reg(RDI);
  303 
  304 // Singleton class for instruction pointer
  305 // reg_class ip_reg(RIP);
  306 
  307 %}
  308 
  309 //----------SOURCE BLOCK-------------------------------------------------------
  310 // This is a block of C++ code which provides values, functions, and
  311 // definitions necessary in the rest of the architecture description
  312 source_hpp %{
  313 
  314 extern RegMask _ANY_REG_mask;
  315 extern RegMask _PTR_REG_mask;
  316 extern RegMask _PTR_REG_NO_RBP_mask;
  317 extern RegMask _PTR_NO_RAX_REG_mask;
  318 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  319 extern RegMask _LONG_REG_mask;
  320 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  321 extern RegMask _LONG_NO_RCX_REG_mask;
  322 extern RegMask _INT_REG_mask;
  323 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  324 extern RegMask _INT_NO_RCX_REG_mask;
  325 extern RegMask _FLOAT_REG_mask;
  326 
  327 extern RegMask _STACK_OR_PTR_REG_mask;
  328 extern RegMask _STACK_OR_LONG_REG_mask;
  329 extern RegMask _STACK_OR_INT_REG_mask;
  330 
  331 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  332 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  333 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  334 
  335 %}
  336 
  337 source %{
  338 #define   RELOC_IMM64    Assembler::imm_operand
  339 #define   RELOC_DISP32   Assembler::disp32_operand
  340 
  341 #define __ _masm.
  342 
  343 RegMask _ANY_REG_mask;
  344 RegMask _PTR_REG_mask;
  345 RegMask _PTR_REG_NO_RBP_mask;
  346 RegMask _PTR_NO_RAX_REG_mask;
  347 RegMask _PTR_NO_RAX_RBX_REG_mask;
  348 RegMask _LONG_REG_mask;
  349 RegMask _LONG_NO_RAX_RDX_REG_mask;
  350 RegMask _LONG_NO_RCX_REG_mask;
  351 RegMask _INT_REG_mask;
  352 RegMask _INT_NO_RAX_RDX_REG_mask;
  353 RegMask _INT_NO_RCX_REG_mask;
  354 RegMask _FLOAT_REG_mask;
  355 RegMask _STACK_OR_PTR_REG_mask;
  356 RegMask _STACK_OR_LONG_REG_mask;
  357 RegMask _STACK_OR_INT_REG_mask;
  358 
  359 static bool need_r12_heapbase() {
  360   return UseCompressedOops;
  361 }
  362 
  363 void reg_mask_init() {
  364   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  365   // We derive a number of subsets from it.
  366   _ANY_REG_mask = _ALL_REG_mask;
  367 
  368   if (PreserveFramePointer) {
  369     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  371   }
  372   if (need_r12_heapbase()) {
  373     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  374     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  375   }
  376 
  377   _PTR_REG_mask = _ANY_REG_mask;
  378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  379   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  380   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  381   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  382 
  383   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  384   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  385 
  386   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  387   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  388   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  389 
  390   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  391   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  392   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  393 
  394   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  395   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  396   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  397 
  398   _LONG_REG_mask = _PTR_REG_mask;
  399   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  400   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  401 
  402   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  404   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  405   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  406   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  407 
  408   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  409   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  410   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  411 
  412   _INT_REG_mask = _ALL_INT_REG_mask;
  413   if (PreserveFramePointer) {
  414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  415   }
  416   if (need_r12_heapbase()) {
  417     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  418   }
  419 
  420   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  421   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  422 
  423   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  424   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  425   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  426 
  427   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  428   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  429 
  430   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  431   // from the float_reg_legacy/float_reg_evex register class.
  432   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  433 
  434   if (Matcher::has_predicated_vectors()) {
  435     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
  436     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
  437     // curruption of value held in K1 register.
  438     if (PostLoopMultiversioning) {
  439       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()));
  440       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()->next()));
  441     }
  442   }
  443 }
  444 
  445 static bool generate_vzeroupper(Compile* C) {
  446   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  447 }
  448 
  449 static int clear_avx_size() {
  450   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  451 }
  452 
  453 // !!!!! Special hack to get all types of calls to specify the byte offset
  454 //       from the start of the call to the point where the return address
  455 //       will point.
  456 int MachCallStaticJavaNode::ret_addr_offset()
  457 {
  458   int offset = 5; // 5 bytes from start of call to where return address points
  459   offset += clear_avx_size();
  460   return offset;
  461 }
  462 
  463 int MachCallDynamicJavaNode::ret_addr_offset()
  464 {
  465   int offset = 15; // 15 bytes from start of call to where return address points
  466   offset += clear_avx_size();
  467   return offset;
  468 }
  469 
  470 int MachCallRuntimeNode::ret_addr_offset() {
  471   int offset = 13; // movq r10,#addr; callq (r10)
  472   if (this->ideal_Opcode() != Op_CallLeafVector) {
  473     offset += clear_avx_size();
  474   }
  475   return offset;
  476 }
  477 
  478 int MachCallNativeNode::ret_addr_offset() {
  479   int offset = 13; // movq r10,#addr; callq (r10)
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 //
  484 // Compute padding required for nodes which need alignment
  485 //
  486 
  487 // The address of the call instruction needs to be 4-byte aligned to
  488 // ensure that it does not span a cache line so that it can be patched.
  489 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  490 {
  491   current_offset += clear_avx_size(); // skip vzeroupper
  492   current_offset += 1; // skip call opcode byte
  493   return align_up(current_offset, alignment_required()) - current_offset;
  494 }
  495 
  496 // The address of the call instruction needs to be 4-byte aligned to
  497 // ensure that it does not span a cache line so that it can be patched.
  498 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  499 {
  500   current_offset += clear_avx_size(); // skip vzeroupper
  501   current_offset += 11; // skip movq instruction + call opcode byte
  502   return align_up(current_offset, alignment_required()) - current_offset;
  503 }
  504 
  505 // EMIT_RM()
  506 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  507   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  508   cbuf.insts()->emit_int8(c);
  509 }
  510 
  511 // EMIT_CC()
  512 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  513   unsigned char c = (unsigned char) (f1 | f2);
  514   cbuf.insts()->emit_int8(c);
  515 }
  516 
  517 // EMIT_OPCODE()
  518 void emit_opcode(CodeBuffer &cbuf, int code) {
  519   cbuf.insts()->emit_int8((unsigned char) code);
  520 }
  521 
  522 // EMIT_OPCODE() w/ relocation information
  523 void emit_opcode(CodeBuffer &cbuf,
  524                  int code, relocInfo::relocType reloc, int offset, int format)
  525 {
  526   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  527   emit_opcode(cbuf, code);
  528 }
  529 
  530 // EMIT_D8()
  531 void emit_d8(CodeBuffer &cbuf, int d8) {
  532   cbuf.insts()->emit_int8((unsigned char) d8);
  533 }
  534 
  535 // EMIT_D16()
  536 void emit_d16(CodeBuffer &cbuf, int d16) {
  537   cbuf.insts()->emit_int16(d16);
  538 }
  539 
  540 // EMIT_D32()
  541 void emit_d32(CodeBuffer &cbuf, int d32) {
  542   cbuf.insts()->emit_int32(d32);
  543 }
  544 
  545 // EMIT_D64()
  546 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  547   cbuf.insts()->emit_int64(d64);
  548 }
  549 
  550 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  551 void emit_d32_reloc(CodeBuffer& cbuf,
  552                     int d32,
  553                     relocInfo::relocType reloc,
  554                     int format)
  555 {
  556   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  557   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  558   cbuf.insts()->emit_int32(d32);
  559 }
  560 
  561 // emit 32 bit value and construct relocation entry from RelocationHolder
  562 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  563 #ifdef ASSERT
  564   if (rspec.reloc()->type() == relocInfo::oop_type &&
  565       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  566     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  567     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  568   }
  569 #endif
  570   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  571   cbuf.insts()->emit_int32(d32);
  572 }
  573 
  574 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  575   address next_ip = cbuf.insts_end() + 4;
  576   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  577                  external_word_Relocation::spec(addr),
  578                  RELOC_DISP32);
  579 }
  580 
  581 
  582 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  583 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  584   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  585   cbuf.insts()->emit_int64(d64);
  586 }
  587 
  588 // emit 64 bit value and construct relocation entry from RelocationHolder
  589 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  590 #ifdef ASSERT
  591   if (rspec.reloc()->type() == relocInfo::oop_type &&
  592       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  593     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  594     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  595   }
  596 #endif
  597   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  598   cbuf.insts()->emit_int64(d64);
  599 }
  600 
  601 // Access stack slot for load or store
  602 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  603 {
  604   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  605   if (-0x80 <= disp && disp < 0x80) {
  606     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  607     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  608     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  609   } else {
  610     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  611     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  612     emit_d32(cbuf, disp);     // Displacement // R/M byte
  613   }
  614 }
  615 
  616    // rRegI ereg, memory mem) %{    // emit_reg_mem
  617 void encode_RegMem(CodeBuffer &cbuf,
  618                    int reg,
  619                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  620 {
  621   assert(disp_reloc == relocInfo::none, "cannot have disp");
  622   int regenc = reg & 7;
  623   int baseenc = base & 7;
  624   int indexenc = index & 7;
  625 
  626   // There is no index & no scale, use form without SIB byte
  627   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  628     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  629     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  630       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  631     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  632       // If 8-bit displacement, mode 0x1
  633       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  634       emit_d8(cbuf, disp);
  635     } else {
  636       // If 32-bit displacement
  637       if (base == -1) { // Special flag for absolute address
  638         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  639         if (disp_reloc != relocInfo::none) {
  640           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  641         } else {
  642           emit_d32(cbuf, disp);
  643         }
  644       } else {
  645         // Normal base + offset
  646         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       }
  653     }
  654   } else {
  655     // Else, encode with the SIB byte
  656     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  657     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  658       // If no displacement
  659       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  660       emit_rm(cbuf, scale, indexenc, baseenc);
  661     } else {
  662       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  663         // If 8-bit displacement, mode 0x1
  664         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  665         emit_rm(cbuf, scale, indexenc, baseenc);
  666         emit_d8(cbuf, disp);
  667       } else {
  668         // If 32-bit displacement
  669         if (base == 0x04 ) {
  670           emit_rm(cbuf, 0x2, regenc, 0x4);
  671           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  672         } else {
  673           emit_rm(cbuf, 0x2, regenc, 0x4);
  674           emit_rm(cbuf, scale, indexenc, baseenc); // *
  675         }
  676         if (disp_reloc != relocInfo::none) {
  677           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  678         } else {
  679           emit_d32(cbuf, disp);
  680         }
  681       }
  682     }
  683   }
  684 }
  685 
  686 // This could be in MacroAssembler but it's fairly C2 specific
  687 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  688   Label exit;
  689   __ jccb(Assembler::noParity, exit);
  690   __ pushf();
  691   //
  692   // comiss/ucomiss instructions set ZF,PF,CF flags and
  693   // zero OF,AF,SF for NaN values.
  694   // Fixup flags by zeroing ZF,PF so that compare of NaN
  695   // values returns 'less than' result (CF is set).
  696   // Leave the rest of flags unchanged.
  697   //
  698   //    7 6 5 4 3 2 1 0
  699   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  700   //    0 0 1 0 1 0 1 1   (0x2B)
  701   //
  702   __ andq(Address(rsp, 0), 0xffffff2b);
  703   __ popf();
  704   __ bind(exit);
  705 }
  706 
  707 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  708   Label done;
  709   __ movl(dst, -1);
  710   __ jcc(Assembler::parity, done);
  711   __ jcc(Assembler::below, done);
  712   __ setb(Assembler::notEqual, dst);
  713   __ movzbl(dst, dst);
  714   __ bind(done);
  715 }
  716 
  717 // Math.min()    # Math.max()
  718 // --------------------------
  719 // ucomis[s/d]   #
  720 // ja   -> b     # a
  721 // jp   -> NaN   # NaN
  722 // jb   -> a     # b
  723 // je            #
  724 // |-jz -> a | b # a & b
  725 // |    -> a     #
  726 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  727                      XMMRegister a, XMMRegister b,
  728                      XMMRegister xmmt, Register rt,
  729                      bool min, bool single) {
  730 
  731   Label nan, zero, below, above, done;
  732 
  733   if (single)
  734     __ ucomiss(a, b);
  735   else
  736     __ ucomisd(a, b);
  737 
  738   if (dst->encoding() != (min ? b : a)->encoding())
  739     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  740   else
  741     __ jccb(Assembler::above, done);
  742 
  743   __ jccb(Assembler::parity, nan);  // PF=1
  744   __ jccb(Assembler::below, below); // CF=1
  745 
  746   // equal
  747   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  748   if (single) {
  749     __ ucomiss(a, xmmt);
  750     __ jccb(Assembler::equal, zero);
  751 
  752     __ movflt(dst, a);
  753     __ jmp(done);
  754   }
  755   else {
  756     __ ucomisd(a, xmmt);
  757     __ jccb(Assembler::equal, zero);
  758 
  759     __ movdbl(dst, a);
  760     __ jmp(done);
  761   }
  762 
  763   __ bind(zero);
  764   if (min)
  765     __ vpor(dst, a, b, Assembler::AVX_128bit);
  766   else
  767     __ vpand(dst, a, b, Assembler::AVX_128bit);
  768 
  769   __ jmp(done);
  770 
  771   __ bind(above);
  772   if (single)
  773     __ movflt(dst, min ? b : a);
  774   else
  775     __ movdbl(dst, min ? b : a);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(nan);
  780   if (single) {
  781     __ movl(rt, 0x7fc00000); // Float.NaN
  782     __ movdl(dst, rt);
  783   }
  784   else {
  785     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  786     __ movdq(dst, rt);
  787   }
  788   __ jmp(done);
  789 
  790   __ bind(below);
  791   if (single)
  792     __ movflt(dst, min ? a : b);
  793   else
  794     __ movdbl(dst, min ? a : b);
  795 
  796   __ bind(done);
  797 }
  798 
  799 //=============================================================================
  800 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  801 
  802 int ConstantTable::calculate_table_base_offset() const {
  803   return 0;  // absolute addressing, no offset
  804 }
  805 
  806 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  807 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  808   ShouldNotReachHere();
  809 }
  810 
  811 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  812   // Empty encoding
  813 }
  814 
  815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  816   return 0;
  817 }
  818 
  819 #ifndef PRODUCT
  820 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  821   st->print("# MachConstantBaseNode (empty encoding)");
  822 }
  823 #endif
  824 
  825 
  826 //=============================================================================
  827 #ifndef PRODUCT
  828 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   Compile* C = ra_->C;
  830 
  831   int framesize = C->output()->frame_size_in_bytes();
  832   int bangsize = C->output()->bang_size_in_bytes();
  833   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  834   // Remove wordSize for return addr which is already pushed.
  835   framesize -= wordSize;
  836 
  837   if (C->output()->need_stack_bang(bangsize)) {
  838     framesize -= wordSize;
  839     st->print("# stack bang (%d bytes)", bangsize);
  840     st->print("\n\t");
  841     st->print("pushq   rbp\t# Save rbp");
  842     if (PreserveFramePointer) {
  843         st->print("\n\t");
  844         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  845     }
  846     if (framesize) {
  847       st->print("\n\t");
  848       st->print("subq    rsp, #%d\t# Create frame",framesize);
  849     }
  850   } else {
  851     st->print("subq    rsp, #%d\t# Create frame",framesize);
  852     st->print("\n\t");
  853     framesize -= wordSize;
  854     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  855     if (PreserveFramePointer) {
  856       st->print("\n\t");
  857       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  858       if (framesize > 0) {
  859         st->print("\n\t");
  860         st->print("addq    rbp, #%d", framesize);
  861       }
  862     }
  863   }
  864 
  865   if (VerifyStackAtCalls) {
  866     st->print("\n\t");
  867     framesize -= wordSize;
  868     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  869 #ifdef ASSERT
  870     st->print("\n\t");
  871     st->print("# stack alignment check");
  872 #endif
  873   }
  874   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  875     st->print("\n\t");
  876     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  877     st->print("\n\t");
  878     st->print("je      fast_entry\t");
  879     st->print("\n\t");
  880     st->print("call    #nmethod_entry_barrier_stub\t");
  881     st->print("\n\tfast_entry:");
  882   }
  883   st->cr();
  884 }
  885 #endif
  886 
  887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  888   Compile* C = ra_->C;
  889   MacroAssembler _masm(&cbuf);
  890 
  891   int framesize = C->output()->frame_size_in_bytes();
  892   int bangsize = C->output()->bang_size_in_bytes();
  893 
  894   if (C->clinit_barrier_on_entry()) {
  895     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  896     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  897 
  898     Label L_skip_barrier;
  899     Register klass = rscratch1;
  900 
  901     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  902     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  903 
  904     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  905 
  906     __ bind(L_skip_barrier);
  907   }
  908 
  909   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  910 
  911   C->output()->set_frame_complete(cbuf.insts_size());
  912 
  913   if (C->has_mach_constant_base_node()) {
  914     // NOTE: We set the table base offset here because users might be
  915     // emitted before MachConstantBaseNode.
  916     ConstantTable& constant_table = C->output()->constant_table();
  917     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  918   }
  919 }
  920 
  921 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  922 {
  923   return MachNode::size(ra_); // too many variables; just compute it
  924                               // the hard way
  925 }
  926 
  927 int MachPrologNode::reloc() const
  928 {
  929   return 0; // a large enough number
  930 }
  931 
  932 //=============================================================================
  933 #ifndef PRODUCT
  934 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  935 {
  936   Compile* C = ra_->C;
  937   if (generate_vzeroupper(C)) {
  938     st->print("vzeroupper");
  939     st->cr(); st->print("\t");
  940   }
  941 
  942   int framesize = C->output()->frame_size_in_bytes();
  943   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  944   // Remove word for return adr already pushed
  945   // and RBP
  946   framesize -= 2*wordSize;
  947 
  948   if (framesize) {
  949     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  950     st->print("\t");
  951   }
  952 
  953   st->print_cr("popq    rbp");
  954   if (do_polling() && C->is_method_compilation()) {
  955     st->print("\t");
  956     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  957                  "ja       #safepoint_stub\t"
  958                  "# Safepoint: poll for GC");
  959   }
  960 }
  961 #endif
  962 
  963 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  964 {
  965   Compile* C = ra_->C;
  966   MacroAssembler _masm(&cbuf);
  967 
  968   if (generate_vzeroupper(C)) {
  969     // Clear upper bits of YMM registers when current compiled code uses
  970     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  971     __ vzeroupper();
  972   }
  973 
  974   int framesize = C->output()->frame_size_in_bytes();
  975   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  976   // Remove word for return adr already pushed
  977   // and RBP
  978   framesize -= 2*wordSize;
  979 
  980   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  981 
  982   if (framesize) {
  983     emit_opcode(cbuf, Assembler::REX_W);
  984     if (framesize < 0x80) {
  985       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  986       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  987       emit_d8(cbuf, framesize);
  988     } else {
  989       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  990       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  991       emit_d32(cbuf, framesize);
  992     }
  993   }
  994 
  995   // popq rbp
  996   emit_opcode(cbuf, 0x58 | RBP_enc);
  997 
  998   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  999     __ reserved_stack_check();
 1000   }
 1001 
 1002   if (do_polling() && C->is_method_compilation()) {
 1003     MacroAssembler _masm(&cbuf);
 1004     Label dummy_label;
 1005     Label* code_stub = &dummy_label;
 1006     if (!C->output()->in_scratch_emit_size()) {
 1007       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1008     }
 1009     __ relocate(relocInfo::poll_return_type);
 1010     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1011   }
 1012 }
 1013 
 1014 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1015 {
 1016   return MachNode::size(ra_); // too many variables; just compute it
 1017                               // the hard way
 1018 }
 1019 
 1020 int MachEpilogNode::reloc() const
 1021 {
 1022   return 2; // a large enough number
 1023 }
 1024 
 1025 const Pipeline* MachEpilogNode::pipeline() const
 1026 {
 1027   return MachNode::pipeline_class();
 1028 }
 1029 
 1030 //=============================================================================
 1031 
 1032 enum RC {
 1033   rc_bad,
 1034   rc_int,
 1035   rc_kreg,
 1036   rc_float,
 1037   rc_stack
 1038 };
 1039 
 1040 static enum RC rc_class(OptoReg::Name reg)
 1041 {
 1042   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1043 
 1044   if (OptoReg::is_stack(reg)) return rc_stack;
 1045 
 1046   VMReg r = OptoReg::as_VMReg(reg);
 1047 
 1048   if (r->is_Register()) return rc_int;
 1049 
 1050   if (r->is_KRegister()) return rc_kreg;
 1051 
 1052   assert(r->is_XMMRegister(), "must be");
 1053   return rc_float;
 1054 }
 1055 
 1056 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1057 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1058                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1059 
 1060 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1061                      int stack_offset, int reg, uint ireg, outputStream* st);
 1062 
 1063 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1064                                       int dst_offset, uint ireg, outputStream* st) {
 1065   if (cbuf) {
 1066     MacroAssembler _masm(cbuf);
 1067     switch (ireg) {
 1068     case Op_VecS:
 1069       __ movq(Address(rsp, -8), rax);
 1070       __ movl(rax, Address(rsp, src_offset));
 1071       __ movl(Address(rsp, dst_offset), rax);
 1072       __ movq(rax, Address(rsp, -8));
 1073       break;
 1074     case Op_VecD:
 1075       __ pushq(Address(rsp, src_offset));
 1076       __ popq (Address(rsp, dst_offset));
 1077       break;
 1078     case Op_VecX:
 1079       __ pushq(Address(rsp, src_offset));
 1080       __ popq (Address(rsp, dst_offset));
 1081       __ pushq(Address(rsp, src_offset+8));
 1082       __ popq (Address(rsp, dst_offset+8));
 1083       break;
 1084     case Op_VecY:
 1085       __ vmovdqu(Address(rsp, -32), xmm0);
 1086       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1087       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1088       __ vmovdqu(xmm0, Address(rsp, -32));
 1089       break;
 1090     case Op_VecZ:
 1091       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1092       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1093       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1094       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1095       break;
 1096     default:
 1097       ShouldNotReachHere();
 1098     }
 1099 #ifndef PRODUCT
 1100   } else {
 1101     switch (ireg) {
 1102     case Op_VecS:
 1103       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1104                 "movl    rax, [rsp + #%d]\n\t"
 1105                 "movl    [rsp + #%d], rax\n\t"
 1106                 "movq    rax, [rsp - #8]",
 1107                 src_offset, dst_offset);
 1108       break;
 1109     case Op_VecD:
 1110       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1111                 "popq    [rsp + #%d]",
 1112                 src_offset, dst_offset);
 1113       break;
 1114      case Op_VecX:
 1115       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1116                 "popq    [rsp + #%d]\n\t"
 1117                 "pushq   [rsp + #%d]\n\t"
 1118                 "popq    [rsp + #%d]",
 1119                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1120       break;
 1121     case Op_VecY:
 1122       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1123                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1124                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1125                 "vmovdqu xmm0, [rsp - #32]",
 1126                 src_offset, dst_offset);
 1127       break;
 1128     case Op_VecZ:
 1129       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1130                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1131                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1132                 "vmovdqu xmm0, [rsp - #64]",
 1133                 src_offset, dst_offset);
 1134       break;
 1135     default:
 1136       ShouldNotReachHere();
 1137     }
 1138 #endif
 1139   }
 1140 }
 1141 
 1142 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1143                                        PhaseRegAlloc* ra_,
 1144                                        bool do_size,
 1145                                        outputStream* st) const {
 1146   assert(cbuf != NULL || st  != NULL, "sanity");
 1147   // Get registers to move
 1148   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1149   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1150   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1151   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1152 
 1153   enum RC src_second_rc = rc_class(src_second);
 1154   enum RC src_first_rc = rc_class(src_first);
 1155   enum RC dst_second_rc = rc_class(dst_second);
 1156   enum RC dst_first_rc = rc_class(dst_first);
 1157 
 1158   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1159          "must move at least 1 register" );
 1160 
 1161   if (src_first == dst_first && src_second == dst_second) {
 1162     // Self copy, no move
 1163     return 0;
 1164   }
 1165   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1166     uint ireg = ideal_reg();
 1167     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1168     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1169     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1170       // mem -> mem
 1171       int src_offset = ra_->reg2offset(src_first);
 1172       int dst_offset = ra_->reg2offset(dst_first);
 1173       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1174     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1175       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1176     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1177       int stack_offset = ra_->reg2offset(dst_first);
 1178       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1179     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1180       int stack_offset = ra_->reg2offset(src_first);
 1181       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1182     } else {
 1183       ShouldNotReachHere();
 1184     }
 1185     return 0;
 1186   }
 1187   if (src_first_rc == rc_stack) {
 1188     // mem ->
 1189     if (dst_first_rc == rc_stack) {
 1190       // mem -> mem
 1191       assert(src_second != dst_first, "overlap");
 1192       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1193           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1194         // 64-bit
 1195         int src_offset = ra_->reg2offset(src_first);
 1196         int dst_offset = ra_->reg2offset(dst_first);
 1197         if (cbuf) {
 1198           MacroAssembler _masm(cbuf);
 1199           __ pushq(Address(rsp, src_offset));
 1200           __ popq (Address(rsp, dst_offset));
 1201 #ifndef PRODUCT
 1202         } else {
 1203           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1204                     "popq    [rsp + #%d]",
 1205                      src_offset, dst_offset);
 1206 #endif
 1207         }
 1208       } else {
 1209         // 32-bit
 1210         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1211         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1212         // No pushl/popl, so:
 1213         int src_offset = ra_->reg2offset(src_first);
 1214         int dst_offset = ra_->reg2offset(dst_first);
 1215         if (cbuf) {
 1216           MacroAssembler _masm(cbuf);
 1217           __ movq(Address(rsp, -8), rax);
 1218           __ movl(rax, Address(rsp, src_offset));
 1219           __ movl(Address(rsp, dst_offset), rax);
 1220           __ movq(rax, Address(rsp, -8));
 1221 #ifndef PRODUCT
 1222         } else {
 1223           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1224                     "movl    rax, [rsp + #%d]\n\t"
 1225                     "movl    [rsp + #%d], rax\n\t"
 1226                     "movq    rax, [rsp - #8]",
 1227                      src_offset, dst_offset);
 1228 #endif
 1229         }
 1230       }
 1231       return 0;
 1232     } else if (dst_first_rc == rc_int) {
 1233       // mem -> gpr
 1234       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1235           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1236         // 64-bit
 1237         int offset = ra_->reg2offset(src_first);
 1238         if (cbuf) {
 1239           MacroAssembler _masm(cbuf);
 1240           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1241 #ifndef PRODUCT
 1242         } else {
 1243           st->print("movq    %s, [rsp + #%d]\t# spill",
 1244                      Matcher::regName[dst_first],
 1245                      offset);
 1246 #endif
 1247         }
 1248       } else {
 1249         // 32-bit
 1250         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1251         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1252         int offset = ra_->reg2offset(src_first);
 1253         if (cbuf) {
 1254           MacroAssembler _masm(cbuf);
 1255           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1256 #ifndef PRODUCT
 1257         } else {
 1258           st->print("movl    %s, [rsp + #%d]\t# spill",
 1259                      Matcher::regName[dst_first],
 1260                      offset);
 1261 #endif
 1262         }
 1263       }
 1264       return 0;
 1265     } else if (dst_first_rc == rc_float) {
 1266       // mem-> xmm
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(src_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("%s  %s, [rsp + #%d]\t# spill",
 1277                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1278                      Matcher::regName[dst_first],
 1279                      offset);
 1280 #endif
 1281         }
 1282       } else {
 1283         // 32-bit
 1284         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1285         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1286         int offset = ra_->reg2offset(src_first);
 1287         if (cbuf) {
 1288           MacroAssembler _masm(cbuf);
 1289           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1290 #ifndef PRODUCT
 1291         } else {
 1292           st->print("movss   %s, [rsp + #%d]\t# spill",
 1293                      Matcher::regName[dst_first],
 1294                      offset);
 1295 #endif
 1296         }
 1297       }
 1298       return 0;
 1299     } else if (dst_first_rc == rc_kreg) {
 1300       // mem -> kreg
 1301       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1302           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1303         // 64-bit
 1304         int offset = ra_->reg2offset(src_first);
 1305         if (cbuf) {
 1306           MacroAssembler _masm(cbuf);
 1307           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1308 #ifndef PRODUCT
 1309         } else {
 1310           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1311                      Matcher::regName[dst_first],
 1312                      offset);
 1313 #endif
 1314         }
 1315       }
 1316       return 0;
 1317     }
 1318   } else if (src_first_rc == rc_int) {
 1319     // gpr ->
 1320     if (dst_first_rc == rc_stack) {
 1321       // gpr -> mem
 1322       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1323           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1324         // 64-bit
 1325         int offset = ra_->reg2offset(dst_first);
 1326         if (cbuf) {
 1327           MacroAssembler _masm(cbuf);
 1328           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1329 #ifndef PRODUCT
 1330         } else {
 1331           st->print("movq    [rsp + #%d], %s\t# spill",
 1332                      offset,
 1333                      Matcher::regName[src_first]);
 1334 #endif
 1335         }
 1336       } else {
 1337         // 32-bit
 1338         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1339         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1340         int offset = ra_->reg2offset(dst_first);
 1341         if (cbuf) {
 1342           MacroAssembler _masm(cbuf);
 1343           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1344 #ifndef PRODUCT
 1345         } else {
 1346           st->print("movl    [rsp + #%d], %s\t# spill",
 1347                      offset,
 1348                      Matcher::regName[src_first]);
 1349 #endif
 1350         }
 1351       }
 1352       return 0;
 1353     } else if (dst_first_rc == rc_int) {
 1354       // gpr -> gpr
 1355       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1356           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1357         // 64-bit
 1358         if (cbuf) {
 1359           MacroAssembler _masm(cbuf);
 1360           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1361                   as_Register(Matcher::_regEncode[src_first]));
 1362 #ifndef PRODUCT
 1363         } else {
 1364           st->print("movq    %s, %s\t# spill",
 1365                      Matcher::regName[dst_first],
 1366                      Matcher::regName[src_first]);
 1367 #endif
 1368         }
 1369         return 0;
 1370       } else {
 1371         // 32-bit
 1372         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1373         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1374         if (cbuf) {
 1375           MacroAssembler _masm(cbuf);
 1376           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1377                   as_Register(Matcher::_regEncode[src_first]));
 1378 #ifndef PRODUCT
 1379         } else {
 1380           st->print("movl    %s, %s\t# spill",
 1381                      Matcher::regName[dst_first],
 1382                      Matcher::regName[src_first]);
 1383 #endif
 1384         }
 1385         return 0;
 1386       }
 1387     } else if (dst_first_rc == rc_float) {
 1388       // gpr -> xmm
 1389       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1390           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1391         // 64-bit
 1392         if (cbuf) {
 1393           MacroAssembler _masm(cbuf);
 1394           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1395 #ifndef PRODUCT
 1396         } else {
 1397           st->print("movdq   %s, %s\t# spill",
 1398                      Matcher::regName[dst_first],
 1399                      Matcher::regName[src_first]);
 1400 #endif
 1401         }
 1402       } else {
 1403         // 32-bit
 1404         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1405         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdl   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_kreg) {
 1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1421         // 64-bit
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425   #ifndef PRODUCT
 1426         } else {
 1427            st->print("kmovq   %s, %s\t# spill",
 1428                        Matcher::regName[dst_first],
 1429                        Matcher::regName[src_first]);
 1430   #endif
 1431         }
 1432       }
 1433       Unimplemented();
 1434       return 0;
 1435     }
 1436   } else if (src_first_rc == rc_float) {
 1437     // xmm ->
 1438     if (dst_first_rc == rc_stack) {
 1439       // xmm -> mem
 1440       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1441           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1442         // 64-bit
 1443         int offset = ra_->reg2offset(dst_first);
 1444         if (cbuf) {
 1445           MacroAssembler _masm(cbuf);
 1446           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1447 #ifndef PRODUCT
 1448         } else {
 1449           st->print("movsd   [rsp + #%d], %s\t# spill",
 1450                      offset,
 1451                      Matcher::regName[src_first]);
 1452 #endif
 1453         }
 1454       } else {
 1455         // 32-bit
 1456         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1457         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1458         int offset = ra_->reg2offset(dst_first);
 1459         if (cbuf) {
 1460           MacroAssembler _masm(cbuf);
 1461           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1462 #ifndef PRODUCT
 1463         } else {
 1464           st->print("movss   [rsp + #%d], %s\t# spill",
 1465                      offset,
 1466                      Matcher::regName[src_first]);
 1467 #endif
 1468         }
 1469       }
 1470       return 0;
 1471     } else if (dst_first_rc == rc_int) {
 1472       // xmm -> gpr
 1473       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1474           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1475         // 64-bit
 1476         if (cbuf) {
 1477           MacroAssembler _masm(cbuf);
 1478           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1479 #ifndef PRODUCT
 1480         } else {
 1481           st->print("movdq   %s, %s\t# spill",
 1482                      Matcher::regName[dst_first],
 1483                      Matcher::regName[src_first]);
 1484 #endif
 1485         }
 1486       } else {
 1487         // 32-bit
 1488         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1489         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1490         if (cbuf) {
 1491           MacroAssembler _masm(cbuf);
 1492           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1493 #ifndef PRODUCT
 1494         } else {
 1495           st->print("movdl   %s, %s\t# spill",
 1496                      Matcher::regName[dst_first],
 1497                      Matcher::regName[src_first]);
 1498 #endif
 1499         }
 1500       }
 1501       return 0;
 1502     } else if (dst_first_rc == rc_float) {
 1503       // xmm -> xmm
 1504       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1505           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1506         // 64-bit
 1507         if (cbuf) {
 1508           MacroAssembler _masm(cbuf);
 1509           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1510 #ifndef PRODUCT
 1511         } else {
 1512           st->print("%s  %s, %s\t# spill",
 1513                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1514                      Matcher::regName[dst_first],
 1515                      Matcher::regName[src_first]);
 1516 #endif
 1517         }
 1518       } else {
 1519         // 32-bit
 1520         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1521         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1522         if (cbuf) {
 1523           MacroAssembler _masm(cbuf);
 1524           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1525 #ifndef PRODUCT
 1526         } else {
 1527           st->print("%s  %s, %s\t# spill",
 1528                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1529                      Matcher::regName[dst_first],
 1530                      Matcher::regName[src_first]);
 1531 #endif
 1532         }
 1533       }
 1534       return 0;
 1535     } else if (dst_first_rc == rc_kreg) {
 1536       assert(false, "Illegal spilling");
 1537       return 0;
 1538     }
 1539   } else if (src_first_rc == rc_kreg) {
 1540     if (dst_first_rc == rc_stack) {
 1541       // mem -> kreg
 1542       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1543           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1544         // 64-bit
 1545         int offset = ra_->reg2offset(dst_first);
 1546         if (cbuf) {
 1547           MacroAssembler _masm(cbuf);
 1548           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1549 #ifndef PRODUCT
 1550         } else {
 1551           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1552                      offset,
 1553                      Matcher::regName[src_first]);
 1554 #endif
 1555         }
 1556       }
 1557       return 0;
 1558     } else if (dst_first_rc == rc_int) {
 1559       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1560           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1561         // 64-bit
 1562         if (cbuf) {
 1563           MacroAssembler _masm(cbuf);
 1564           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1565 #ifndef PRODUCT
 1566         } else {
 1567          st->print("kmovq   %s, %s\t# spill",
 1568                      Matcher::regName[dst_first],
 1569                      Matcher::regName[src_first]);
 1570 #endif
 1571         }
 1572       }
 1573       Unimplemented();
 1574       return 0;
 1575     } else if (dst_first_rc == rc_kreg) {
 1576       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1577           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1578         // 64-bit
 1579         if (cbuf) {
 1580           MacroAssembler _masm(cbuf);
 1581           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1582 #ifndef PRODUCT
 1583         } else {
 1584          st->print("kmovq   %s, %s\t# spill",
 1585                      Matcher::regName[dst_first],
 1586                      Matcher::regName[src_first]);
 1587 #endif
 1588         }
 1589       }
 1590       return 0;
 1591     } else if (dst_first_rc == rc_float) {
 1592       assert(false, "Illegal spill");
 1593       return 0;
 1594     }
 1595   }
 1596 
 1597   assert(0," foo ");
 1598   Unimplemented();
 1599   return 0;
 1600 }
 1601 
 1602 #ifndef PRODUCT
 1603 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1604   implementation(NULL, ra_, false, st);
 1605 }
 1606 #endif
 1607 
 1608 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1609   implementation(&cbuf, ra_, false, NULL);
 1610 }
 1611 
 1612 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1613   return MachNode::size(ra_);
 1614 }
 1615 
 1616 //=============================================================================
 1617 #ifndef PRODUCT
 1618 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1619 {
 1620   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1621   int reg = ra_->get_reg_first(this);
 1622   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1623             Matcher::regName[reg], offset);
 1624 }
 1625 #endif
 1626 
 1627 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1628 {
 1629   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1630   int reg = ra_->get_encode(this);
 1631   if (offset >= 0x80) {
 1632     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1633     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1634     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1635     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1636     emit_d32(cbuf, offset);
 1637   } else {
 1638     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1639     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1640     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1641     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1642     emit_d8(cbuf, offset);
 1643   }
 1644 }
 1645 
 1646 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1647 {
 1648   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1649   return (offset < 0x80) ? 5 : 8; // REX
 1650 }
 1651 
 1652 //=============================================================================
 1653 #ifndef PRODUCT
 1654 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1655 {
 1656   if (UseCompressedClassPointers) {
 1657     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1658     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1659     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1660   } else {
 1661     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1662                  "# Inline cache check");
 1663   }
 1664   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1665   st->print_cr("\tnop\t# nops to align entry point");
 1666 }
 1667 #endif
 1668 
 1669 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1670 {
 1671   MacroAssembler masm(&cbuf);
 1672   uint insts_size = cbuf.insts_size();
 1673   if (UseCompressedClassPointers) {
 1674     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1675     masm.cmpptr(rax, rscratch1);
 1676   } else {
 1677     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1678   }
 1679 
 1680   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1681 
 1682   /* WARNING these NOPs are critical so that verified entry point is properly
 1683      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1684   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1685   if (OptoBreakpoint) {
 1686     // Leave space for int3
 1687     nops_cnt -= 1;
 1688   }
 1689   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1690   if (nops_cnt > 0)
 1691     masm.nop(nops_cnt);
 1692 }
 1693 
 1694 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1695 {
 1696   return MachNode::size(ra_); // too many variables; just compute it
 1697                               // the hard way
 1698 }
 1699 
 1700 
 1701 //=============================================================================
 1702 
 1703 const bool Matcher::supports_vector_calling_convention(void) {
 1704   if (EnableVectorSupport && UseVectorStubs) {
 1705     return true;
 1706   }
 1707   return false;
 1708 }
 1709 
 1710 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1711   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1712   int lo = XMM0_num;
 1713   int hi = XMM0b_num;
 1714   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1715   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1716   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1717   return OptoRegPair(hi, lo);
 1718 }
 1719 
 1720 // Is this branch offset short enough that a short branch can be used?
 1721 //
 1722 // NOTE: If the platform does not provide any short branch variants, then
 1723 //       this method should return false for offset 0.
 1724 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1725   // The passed offset is relative to address of the branch.
 1726   // On 86 a branch displacement is calculated relative to address
 1727   // of a next instruction.
 1728   offset -= br_size;
 1729 
 1730   // the short version of jmpConUCF2 contains multiple branches,
 1731   // making the reach slightly less
 1732   if (rule == jmpConUCF2_rule)
 1733     return (-126 <= offset && offset <= 125);
 1734   return (-128 <= offset && offset <= 127);
 1735 }
 1736 
 1737 // Return whether or not this register is ever used as an argument.
 1738 // This function is used on startup to build the trampoline stubs in
 1739 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1740 // call in the trampoline, and arguments in those registers not be
 1741 // available to the callee.
 1742 bool Matcher::can_be_java_arg(int reg)
 1743 {
 1744   return
 1745     reg ==  RDI_num || reg == RDI_H_num ||
 1746     reg ==  RSI_num || reg == RSI_H_num ||
 1747     reg ==  RDX_num || reg == RDX_H_num ||
 1748     reg ==  RCX_num || reg == RCX_H_num ||
 1749     reg ==   R8_num || reg ==  R8_H_num ||
 1750     reg ==   R9_num || reg ==  R9_H_num ||
 1751     reg ==  R12_num || reg == R12_H_num ||
 1752     reg == XMM0_num || reg == XMM0b_num ||
 1753     reg == XMM1_num || reg == XMM1b_num ||
 1754     reg == XMM2_num || reg == XMM2b_num ||
 1755     reg == XMM3_num || reg == XMM3b_num ||
 1756     reg == XMM4_num || reg == XMM4b_num ||
 1757     reg == XMM5_num || reg == XMM5b_num ||
 1758     reg == XMM6_num || reg == XMM6b_num ||
 1759     reg == XMM7_num || reg == XMM7b_num;
 1760 }
 1761 
 1762 bool Matcher::is_spillable_arg(int reg)
 1763 {
 1764   return can_be_java_arg(reg);
 1765 }
 1766 
 1767 uint Matcher::int_pressure_limit()
 1768 {
 1769   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1770 }
 1771 
 1772 uint Matcher::float_pressure_limit()
 1773 {
 1774   // After experiment around with different values, the following default threshold
 1775   // works best for LCM's register pressure scheduling on x64.
 1776   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1777   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1778   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1779 }
 1780 
 1781 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1782   // In 64 bit mode a code which use multiply when
 1783   // devisor is constant is faster than hardware
 1784   // DIV instruction (it uses MulHiL).
 1785   return false;
 1786 }
 1787 
 1788 // Register for DIVI projection of divmodI
 1789 RegMask Matcher::divI_proj_mask() {
 1790   return INT_RAX_REG_mask();
 1791 }
 1792 
 1793 // Register for MODI projection of divmodI
 1794 RegMask Matcher::modI_proj_mask() {
 1795   return INT_RDX_REG_mask();
 1796 }
 1797 
 1798 // Register for DIVL projection of divmodL
 1799 RegMask Matcher::divL_proj_mask() {
 1800   return LONG_RAX_REG_mask();
 1801 }
 1802 
 1803 // Register for MODL projection of divmodL
 1804 RegMask Matcher::modL_proj_mask() {
 1805   return LONG_RDX_REG_mask();
 1806 }
 1807 
 1808 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1809 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1810     return NO_REG_mask();
 1811 }
 1812 
 1813 %}
 1814 
 1815 //----------ENCODING BLOCK-----------------------------------------------------
 1816 // This block specifies the encoding classes used by the compiler to
 1817 // output byte streams.  Encoding classes are parameterized macros
 1818 // used by Machine Instruction Nodes in order to generate the bit
 1819 // encoding of the instruction.  Operands specify their base encoding
 1820 // interface with the interface keyword.  There are currently
 1821 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1822 // COND_INTER.  REG_INTER causes an operand to generate a function
 1823 // which returns its register number when queried.  CONST_INTER causes
 1824 // an operand to generate a function which returns the value of the
 1825 // constant when queried.  MEMORY_INTER causes an operand to generate
 1826 // four functions which return the Base Register, the Index Register,
 1827 // the Scale Value, and the Offset Value of the operand when queried.
 1828 // COND_INTER causes an operand to generate six functions which return
 1829 // the encoding code (ie - encoding bits for the instruction)
 1830 // associated with each basic boolean condition for a conditional
 1831 // instruction.
 1832 //
 1833 // Instructions specify two basic values for encoding.  Again, a
 1834 // function is available to check if the constant displacement is an
 1835 // oop. They use the ins_encode keyword to specify their encoding
 1836 // classes (which must be a sequence of enc_class names, and their
 1837 // parameters, specified in the encoding block), and they use the
 1838 // opcode keyword to specify, in order, their primary, secondary, and
 1839 // tertiary opcode.  Only the opcode sections which a particular
 1840 // instruction needs for encoding need to be specified.
 1841 encode %{
 1842   // Build emit functions for each basic byte or larger field in the
 1843   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1844   // from C++ code in the enc_class source block.  Emit functions will
 1845   // live in the main source block for now.  In future, we can
 1846   // generalize this by adding a syntax that specifies the sizes of
 1847   // fields in an order, so that the adlc can build the emit functions
 1848   // automagically
 1849 
 1850   // Emit primary opcode
 1851   enc_class OpcP
 1852   %{
 1853     emit_opcode(cbuf, $primary);
 1854   %}
 1855 
 1856   // Emit secondary opcode
 1857   enc_class OpcS
 1858   %{
 1859     emit_opcode(cbuf, $secondary);
 1860   %}
 1861 
 1862   // Emit tertiary opcode
 1863   enc_class OpcT
 1864   %{
 1865     emit_opcode(cbuf, $tertiary);
 1866   %}
 1867 
 1868   // Emit opcode directly
 1869   enc_class Opcode(immI d8)
 1870   %{
 1871     emit_opcode(cbuf, $d8$$constant);
 1872   %}
 1873 
 1874   // Emit size prefix
 1875   enc_class SizePrefix
 1876   %{
 1877     emit_opcode(cbuf, 0x66);
 1878   %}
 1879 
 1880   enc_class reg(rRegI reg)
 1881   %{
 1882     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1883   %}
 1884 
 1885   enc_class reg_reg(rRegI dst, rRegI src)
 1886   %{
 1887     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1888   %}
 1889 
 1890   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1891   %{
 1892     emit_opcode(cbuf, $opcode$$constant);
 1893     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1894   %}
 1895 
 1896   enc_class cdql_enc(no_rax_rdx_RegI div)
 1897   %{
 1898     // Full implementation of Java idiv and irem; checks for
 1899     // special case as described in JVM spec., p.243 & p.271.
 1900     //
 1901     //         normal case                           special case
 1902     //
 1903     // input : rax: dividend                         min_int
 1904     //         reg: divisor                          -1
 1905     //
 1906     // output: rax: quotient  (= rax idiv reg)       min_int
 1907     //         rdx: remainder (= rax irem reg)       0
 1908     //
 1909     //  Code sequnce:
 1910     //
 1911     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1912     //    5:   75 07/08                jne    e <normal>
 1913     //    7:   33 d2                   xor    %edx,%edx
 1914     //  [div >= 8 -> offset + 1]
 1915     //  [REX_B]
 1916     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1917     //    c:   74 03/04                je     11 <done>
 1918     // 000000000000000e <normal>:
 1919     //    e:   99                      cltd
 1920     //  [div >= 8 -> offset + 1]
 1921     //  [REX_B]
 1922     //    f:   f7 f9                   idiv   $div
 1923     // 0000000000000011 <done>:
 1924     MacroAssembler _masm(&cbuf);
 1925     Label normal;
 1926     Label done;
 1927 
 1928     // cmp    $0x80000000,%eax
 1929     __ cmp(as_Register(RAX_enc), 0x80000000);
 1930 
 1931     // jne    e <normal>
 1932     __ jccb(Assembler::notEqual, normal);
 1933 
 1934     // xor    %edx,%edx
 1935     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1936 
 1937     // cmp    $0xffffffffffffffff,%ecx
 1938     __ cmpl($div$$Register, -1);
 1939 
 1940     // je     11 <done>
 1941     __ jccb(Assembler::equal, done);
 1942 
 1943     // <normal>
 1944     // cltd
 1945     __ bind(normal);
 1946     __ cdql();
 1947 
 1948     // idivl
 1949     // <done>
 1950     __ idivl($div$$Register);
 1951     __ bind(done);
 1952   %}
 1953 
 1954   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1955   %{
 1956     // Full implementation of Java ldiv and lrem; checks for
 1957     // special case as described in JVM spec., p.243 & p.271.
 1958     //
 1959     //         normal case                           special case
 1960     //
 1961     // input : rax: dividend                         min_long
 1962     //         reg: divisor                          -1
 1963     //
 1964     // output: rax: quotient  (= rax idiv reg)       min_long
 1965     //         rdx: remainder (= rax irem reg)       0
 1966     //
 1967     //  Code sequnce:
 1968     //
 1969     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1970     //    7:   00 00 80
 1971     //    a:   48 39 d0                cmp    %rdx,%rax
 1972     //    d:   75 08                   jne    17 <normal>
 1973     //    f:   33 d2                   xor    %edx,%edx
 1974     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1975     //   15:   74 05                   je     1c <done>
 1976     // 0000000000000017 <normal>:
 1977     //   17:   48 99                   cqto
 1978     //   19:   48 f7 f9                idiv   $div
 1979     // 000000000000001c <done>:
 1980     MacroAssembler _masm(&cbuf);
 1981     Label normal;
 1982     Label done;
 1983 
 1984     // mov    $0x8000000000000000,%rdx
 1985     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1986 
 1987     // cmp    %rdx,%rax
 1988     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1989 
 1990     // jne    17 <normal>
 1991     __ jccb(Assembler::notEqual, normal);
 1992 
 1993     // xor    %edx,%edx
 1994     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1995 
 1996     // cmp    $0xffffffffffffffff,$div
 1997     __ cmpq($div$$Register, -1);
 1998 
 1999     // je     1e <done>
 2000     __ jccb(Assembler::equal, done);
 2001 
 2002     // <normal>
 2003     // cqto
 2004     __ bind(normal);
 2005     __ cdqq();
 2006 
 2007     // idivq (note: must be emitted by the user of this rule)
 2008     // <done>
 2009     __ idivq($div$$Register);
 2010     __ bind(done);
 2011   %}
 2012 
 2013   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2014   enc_class OpcSE(immI imm)
 2015   %{
 2016     // Emit primary opcode and set sign-extend bit
 2017     // Check for 8-bit immediate, and set sign extend bit in opcode
 2018     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2019       emit_opcode(cbuf, $primary | 0x02);
 2020     } else {
 2021       // 32-bit immediate
 2022       emit_opcode(cbuf, $primary);
 2023     }
 2024   %}
 2025 
 2026   enc_class OpcSErm(rRegI dst, immI imm)
 2027   %{
 2028     // OpcSEr/m
 2029     int dstenc = $dst$$reg;
 2030     if (dstenc >= 8) {
 2031       emit_opcode(cbuf, Assembler::REX_B);
 2032       dstenc -= 8;
 2033     }
 2034     // Emit primary opcode and set sign-extend bit
 2035     // Check for 8-bit immediate, and set sign extend bit in opcode
 2036     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2037       emit_opcode(cbuf, $primary | 0x02);
 2038     } else {
 2039       // 32-bit immediate
 2040       emit_opcode(cbuf, $primary);
 2041     }
 2042     // Emit r/m byte with secondary opcode, after primary opcode.
 2043     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2044   %}
 2045 
 2046   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2047   %{
 2048     // OpcSEr/m
 2049     int dstenc = $dst$$reg;
 2050     if (dstenc < 8) {
 2051       emit_opcode(cbuf, Assembler::REX_W);
 2052     } else {
 2053       emit_opcode(cbuf, Assembler::REX_WB);
 2054       dstenc -= 8;
 2055     }
 2056     // Emit primary opcode and set sign-extend bit
 2057     // Check for 8-bit immediate, and set sign extend bit in opcode
 2058     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2059       emit_opcode(cbuf, $primary | 0x02);
 2060     } else {
 2061       // 32-bit immediate
 2062       emit_opcode(cbuf, $primary);
 2063     }
 2064     // Emit r/m byte with secondary opcode, after primary opcode.
 2065     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2066   %}
 2067 
 2068   enc_class Con8or32(immI imm)
 2069   %{
 2070     // Check for 8-bit immediate, and set sign extend bit in opcode
 2071     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2072       $$$emit8$imm$$constant;
 2073     } else {
 2074       // 32-bit immediate
 2075       $$$emit32$imm$$constant;
 2076     }
 2077   %}
 2078 
 2079   enc_class opc2_reg(rRegI dst)
 2080   %{
 2081     // BSWAP
 2082     emit_cc(cbuf, $secondary, $dst$$reg);
 2083   %}
 2084 
 2085   enc_class opc3_reg(rRegI dst)
 2086   %{
 2087     // BSWAP
 2088     emit_cc(cbuf, $tertiary, $dst$$reg);
 2089   %}
 2090 
 2091   enc_class reg_opc(rRegI div)
 2092   %{
 2093     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2094     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2095   %}
 2096 
 2097   enc_class enc_cmov(cmpOp cop)
 2098   %{
 2099     // CMOV
 2100     $$$emit8$primary;
 2101     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2102   %}
 2103 
 2104   enc_class enc_PartialSubtypeCheck()
 2105   %{
 2106     Register Rrdi = as_Register(RDI_enc); // result register
 2107     Register Rrax = as_Register(RAX_enc); // super class
 2108     Register Rrcx = as_Register(RCX_enc); // killed
 2109     Register Rrsi = as_Register(RSI_enc); // sub class
 2110     Label miss;
 2111     const bool set_cond_codes = true;
 2112 
 2113     MacroAssembler _masm(&cbuf);
 2114     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2115                                      NULL, &miss,
 2116                                      /*set_cond_codes:*/ true);
 2117     if ($primary) {
 2118       __ xorptr(Rrdi, Rrdi);
 2119     }
 2120     __ bind(miss);
 2121   %}
 2122 
 2123   enc_class clear_avx %{
 2124     debug_only(int off0 = cbuf.insts_size());
 2125     if (generate_vzeroupper(Compile::current())) {
 2126       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2127       // Clear upper bits of YMM registers when current compiled code uses
 2128       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2129       MacroAssembler _masm(&cbuf);
 2130       __ vzeroupper();
 2131     }
 2132     debug_only(int off1 = cbuf.insts_size());
 2133     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2134   %}
 2135 
 2136   enc_class Java_To_Runtime(method meth) %{
 2137     // No relocation needed
 2138     MacroAssembler _masm(&cbuf);
 2139     __ mov64(r10, (int64_t) $meth$$method);
 2140     __ call(r10);
 2141     __ post_call_nop();
 2142   %}
 2143 
 2144   enc_class Java_To_Interpreter(method meth)
 2145   %{
 2146     // CALL Java_To_Interpreter
 2147     // This is the instruction starting address for relocation info.
 2148     cbuf.set_insts_mark();
 2149     $$$emit8$primary;
 2150     // CALL directly to the runtime
 2151     emit_d32_reloc(cbuf,
 2152                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2153                    runtime_call_Relocation::spec(),
 2154                    RELOC_DISP32);
 2155     __ post_call_nop();
 2156   %}
 2157 
 2158   enc_class Java_Static_Call(method meth)
 2159   %{
 2160     // JAVA STATIC CALL
 2161     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2162     // determine who we intended to call.
 2163     MacroAssembler _masm(&cbuf);
 2164     cbuf.set_insts_mark();
 2165     $$$emit8$primary;
 2166 
 2167     if (!_method) {
 2168       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2169                      runtime_call_Relocation::spec(),
 2170                      RELOC_DISP32);
 2171     } else {
 2172       int method_index = resolved_method_index(cbuf);
 2173       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2174                                                   : static_call_Relocation::spec(method_index);
 2175       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2176                      rspec, RELOC_DISP32);
 2177       // Emit stubs for static call.
 2178       address mark = cbuf.insts_mark();
 2179       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2180       if (stub == NULL) {
 2181         ciEnv::current()->record_failure("CodeCache is full");
 2182         return;
 2183       }
 2184     }
 2185     __ post_call_nop();
 2186   %}
 2187 
 2188   enc_class Java_Dynamic_Call(method meth) %{
 2189     MacroAssembler _masm(&cbuf);
 2190     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2191     __ post_call_nop();
 2192   %}
 2193 
 2194   enc_class Java_Compiled_Call(method meth)
 2195   %{
 2196     // JAVA COMPILED CALL
 2197     int disp = in_bytes(Method:: from_compiled_offset());
 2198 
 2199     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2200     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2201 
 2202     // callq *disp(%rax)
 2203     MacroAssembler _masm(&cbuf);
 2204     cbuf.set_insts_mark();
 2205     $$$emit8$primary;
 2206     if (disp < 0x80) {
 2207       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2208       emit_d8(cbuf, disp); // Displacement
 2209     } else {
 2210       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2211       emit_d32(cbuf, disp); // Displacement
 2212     }
 2213     __ post_call_nop();
 2214   %}
 2215 
 2216   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2217   %{
 2218     // SAL, SAR, SHR
 2219     int dstenc = $dst$$reg;
 2220     if (dstenc >= 8) {
 2221       emit_opcode(cbuf, Assembler::REX_B);
 2222       dstenc -= 8;
 2223     }
 2224     $$$emit8$primary;
 2225     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2226     $$$emit8$shift$$constant;
 2227   %}
 2228 
 2229   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2230   %{
 2231     // SAL, SAR, SHR
 2232     int dstenc = $dst$$reg;
 2233     if (dstenc < 8) {
 2234       emit_opcode(cbuf, Assembler::REX_W);
 2235     } else {
 2236       emit_opcode(cbuf, Assembler::REX_WB);
 2237       dstenc -= 8;
 2238     }
 2239     $$$emit8$primary;
 2240     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2241     $$$emit8$shift$$constant;
 2242   %}
 2243 
 2244   enc_class load_immI(rRegI dst, immI src)
 2245   %{
 2246     int dstenc = $dst$$reg;
 2247     if (dstenc >= 8) {
 2248       emit_opcode(cbuf, Assembler::REX_B);
 2249       dstenc -= 8;
 2250     }
 2251     emit_opcode(cbuf, 0xB8 | dstenc);
 2252     $$$emit32$src$$constant;
 2253   %}
 2254 
 2255   enc_class load_immL(rRegL dst, immL src)
 2256   %{
 2257     int dstenc = $dst$$reg;
 2258     if (dstenc < 8) {
 2259       emit_opcode(cbuf, Assembler::REX_W);
 2260     } else {
 2261       emit_opcode(cbuf, Assembler::REX_WB);
 2262       dstenc -= 8;
 2263     }
 2264     emit_opcode(cbuf, 0xB8 | dstenc);
 2265     emit_d64(cbuf, $src$$constant);
 2266   %}
 2267 
 2268   enc_class load_immUL32(rRegL dst, immUL32 src)
 2269   %{
 2270     // same as load_immI, but this time we care about zeroes in the high word
 2271     int dstenc = $dst$$reg;
 2272     if (dstenc >= 8) {
 2273       emit_opcode(cbuf, Assembler::REX_B);
 2274       dstenc -= 8;
 2275     }
 2276     emit_opcode(cbuf, 0xB8 | dstenc);
 2277     $$$emit32$src$$constant;
 2278   %}
 2279 
 2280   enc_class load_immL32(rRegL dst, immL32 src)
 2281   %{
 2282     int dstenc = $dst$$reg;
 2283     if (dstenc < 8) {
 2284       emit_opcode(cbuf, Assembler::REX_W);
 2285     } else {
 2286       emit_opcode(cbuf, Assembler::REX_WB);
 2287       dstenc -= 8;
 2288     }
 2289     emit_opcode(cbuf, 0xC7);
 2290     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2291     $$$emit32$src$$constant;
 2292   %}
 2293 
 2294   enc_class load_immP31(rRegP dst, immP32 src)
 2295   %{
 2296     // same as load_immI, but this time we care about zeroes in the high word
 2297     int dstenc = $dst$$reg;
 2298     if (dstenc >= 8) {
 2299       emit_opcode(cbuf, Assembler::REX_B);
 2300       dstenc -= 8;
 2301     }
 2302     emit_opcode(cbuf, 0xB8 | dstenc);
 2303     $$$emit32$src$$constant;
 2304   %}
 2305 
 2306   enc_class load_immP(rRegP dst, immP src)
 2307   %{
 2308     int dstenc = $dst$$reg;
 2309     if (dstenc < 8) {
 2310       emit_opcode(cbuf, Assembler::REX_W);
 2311     } else {
 2312       emit_opcode(cbuf, Assembler::REX_WB);
 2313       dstenc -= 8;
 2314     }
 2315     emit_opcode(cbuf, 0xB8 | dstenc);
 2316     // This next line should be generated from ADLC
 2317     if ($src->constant_reloc() != relocInfo::none) {
 2318       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2319     } else {
 2320       emit_d64(cbuf, $src$$constant);
 2321     }
 2322   %}
 2323 
 2324   enc_class Con32(immI src)
 2325   %{
 2326     // Output immediate
 2327     $$$emit32$src$$constant;
 2328   %}
 2329 
 2330   enc_class Con32F_as_bits(immF src)
 2331   %{
 2332     // Output Float immediate bits
 2333     jfloat jf = $src$$constant;
 2334     jint jf_as_bits = jint_cast(jf);
 2335     emit_d32(cbuf, jf_as_bits);
 2336   %}
 2337 
 2338   enc_class Con16(immI src)
 2339   %{
 2340     // Output immediate
 2341     $$$emit16$src$$constant;
 2342   %}
 2343 
 2344   // How is this different from Con32??? XXX
 2345   enc_class Con_d32(immI src)
 2346   %{
 2347     emit_d32(cbuf,$src$$constant);
 2348   %}
 2349 
 2350   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2351     // Output immediate memory reference
 2352     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2353     emit_d32(cbuf, 0x00);
 2354   %}
 2355 
 2356   enc_class lock_prefix()
 2357   %{
 2358     emit_opcode(cbuf, 0xF0); // lock
 2359   %}
 2360 
 2361   enc_class REX_mem(memory mem)
 2362   %{
 2363     if ($mem$$base >= 8) {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_B);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_XB);
 2368       }
 2369     } else {
 2370       if ($mem$$index >= 8) {
 2371         emit_opcode(cbuf, Assembler::REX_X);
 2372       }
 2373     }
 2374   %}
 2375 
 2376   enc_class REX_mem_wide(memory mem)
 2377   %{
 2378     if ($mem$$base >= 8) {
 2379       if ($mem$$index < 8) {
 2380         emit_opcode(cbuf, Assembler::REX_WB);
 2381       } else {
 2382         emit_opcode(cbuf, Assembler::REX_WXB);
 2383       }
 2384     } else {
 2385       if ($mem$$index < 8) {
 2386         emit_opcode(cbuf, Assembler::REX_W);
 2387       } else {
 2388         emit_opcode(cbuf, Assembler::REX_WX);
 2389       }
 2390     }
 2391   %}
 2392 
 2393   // for byte regs
 2394   enc_class REX_breg(rRegI reg)
 2395   %{
 2396     if ($reg$$reg >= 4) {
 2397       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2398     }
 2399   %}
 2400 
 2401   // for byte regs
 2402   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2403   %{
 2404     if ($dst$$reg < 8) {
 2405       if ($src$$reg >= 4) {
 2406         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2407       }
 2408     } else {
 2409       if ($src$$reg < 8) {
 2410         emit_opcode(cbuf, Assembler::REX_R);
 2411       } else {
 2412         emit_opcode(cbuf, Assembler::REX_RB);
 2413       }
 2414     }
 2415   %}
 2416 
 2417   // for byte regs
 2418   enc_class REX_breg_mem(rRegI reg, memory mem)
 2419   %{
 2420     if ($reg$$reg < 8) {
 2421       if ($mem$$base < 8) {
 2422         if ($mem$$index >= 8) {
 2423           emit_opcode(cbuf, Assembler::REX_X);
 2424         } else if ($reg$$reg >= 4) {
 2425           emit_opcode(cbuf, Assembler::REX);
 2426         }
 2427       } else {
 2428         if ($mem$$index < 8) {
 2429           emit_opcode(cbuf, Assembler::REX_B);
 2430         } else {
 2431           emit_opcode(cbuf, Assembler::REX_XB);
 2432         }
 2433       }
 2434     } else {
 2435       if ($mem$$base < 8) {
 2436         if ($mem$$index < 8) {
 2437           emit_opcode(cbuf, Assembler::REX_R);
 2438         } else {
 2439           emit_opcode(cbuf, Assembler::REX_RX);
 2440         }
 2441       } else {
 2442         if ($mem$$index < 8) {
 2443           emit_opcode(cbuf, Assembler::REX_RB);
 2444         } else {
 2445           emit_opcode(cbuf, Assembler::REX_RXB);
 2446         }
 2447       }
 2448     }
 2449   %}
 2450 
 2451   enc_class REX_reg(rRegI reg)
 2452   %{
 2453     if ($reg$$reg >= 8) {
 2454       emit_opcode(cbuf, Assembler::REX_B);
 2455     }
 2456   %}
 2457 
 2458   enc_class REX_reg_wide(rRegI reg)
 2459   %{
 2460     if ($reg$$reg < 8) {
 2461       emit_opcode(cbuf, Assembler::REX_W);
 2462     } else {
 2463       emit_opcode(cbuf, Assembler::REX_WB);
 2464     }
 2465   %}
 2466 
 2467   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2468   %{
 2469     if ($dst$$reg < 8) {
 2470       if ($src$$reg >= 8) {
 2471         emit_opcode(cbuf, Assembler::REX_B);
 2472       }
 2473     } else {
 2474       if ($src$$reg < 8) {
 2475         emit_opcode(cbuf, Assembler::REX_R);
 2476       } else {
 2477         emit_opcode(cbuf, Assembler::REX_RB);
 2478       }
 2479     }
 2480   %}
 2481 
 2482   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2483   %{
 2484     if ($dst$$reg < 8) {
 2485       if ($src$$reg < 8) {
 2486         emit_opcode(cbuf, Assembler::REX_W);
 2487       } else {
 2488         emit_opcode(cbuf, Assembler::REX_WB);
 2489       }
 2490     } else {
 2491       if ($src$$reg < 8) {
 2492         emit_opcode(cbuf, Assembler::REX_WR);
 2493       } else {
 2494         emit_opcode(cbuf, Assembler::REX_WRB);
 2495       }
 2496     }
 2497   %}
 2498 
 2499   enc_class REX_reg_mem(rRegI reg, memory mem)
 2500   %{
 2501     if ($reg$$reg < 8) {
 2502       if ($mem$$base < 8) {
 2503         if ($mem$$index >= 8) {
 2504           emit_opcode(cbuf, Assembler::REX_X);
 2505         }
 2506       } else {
 2507         if ($mem$$index < 8) {
 2508           emit_opcode(cbuf, Assembler::REX_B);
 2509         } else {
 2510           emit_opcode(cbuf, Assembler::REX_XB);
 2511         }
 2512       }
 2513     } else {
 2514       if ($mem$$base < 8) {
 2515         if ($mem$$index < 8) {
 2516           emit_opcode(cbuf, Assembler::REX_R);
 2517         } else {
 2518           emit_opcode(cbuf, Assembler::REX_RX);
 2519         }
 2520       } else {
 2521         if ($mem$$index < 8) {
 2522           emit_opcode(cbuf, Assembler::REX_RB);
 2523         } else {
 2524           emit_opcode(cbuf, Assembler::REX_RXB);
 2525         }
 2526       }
 2527     }
 2528   %}
 2529 
 2530   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2531   %{
 2532     if ($reg$$reg < 8) {
 2533       if ($mem$$base < 8) {
 2534         if ($mem$$index < 8) {
 2535           emit_opcode(cbuf, Assembler::REX_W);
 2536         } else {
 2537           emit_opcode(cbuf, Assembler::REX_WX);
 2538         }
 2539       } else {
 2540         if ($mem$$index < 8) {
 2541           emit_opcode(cbuf, Assembler::REX_WB);
 2542         } else {
 2543           emit_opcode(cbuf, Assembler::REX_WXB);
 2544         }
 2545       }
 2546     } else {
 2547       if ($mem$$base < 8) {
 2548         if ($mem$$index < 8) {
 2549           emit_opcode(cbuf, Assembler::REX_WR);
 2550         } else {
 2551           emit_opcode(cbuf, Assembler::REX_WRX);
 2552         }
 2553       } else {
 2554         if ($mem$$index < 8) {
 2555           emit_opcode(cbuf, Assembler::REX_WRB);
 2556         } else {
 2557           emit_opcode(cbuf, Assembler::REX_WRXB);
 2558         }
 2559       }
 2560     }
 2561   %}
 2562 
 2563   enc_class reg_mem(rRegI ereg, memory mem)
 2564   %{
 2565     // High registers handle in encode_RegMem
 2566     int reg = $ereg$$reg;
 2567     int base = $mem$$base;
 2568     int index = $mem$$index;
 2569     int scale = $mem$$scale;
 2570     int disp = $mem$$disp;
 2571     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2572 
 2573     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2574   %}
 2575 
 2576   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2577   %{
 2578     int rm_byte_opcode = $rm_opcode$$constant;
 2579 
 2580     // High registers handle in encode_RegMem
 2581     int base = $mem$$base;
 2582     int index = $mem$$index;
 2583     int scale = $mem$$scale;
 2584     int displace = $mem$$disp;
 2585 
 2586     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2587                                             // working with static
 2588                                             // globals
 2589     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2590                   disp_reloc);
 2591   %}
 2592 
 2593   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2594   %{
 2595     int reg_encoding = $dst$$reg;
 2596     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2597     int index        = 0x04;            // 0x04 indicates no index
 2598     int scale        = 0x00;            // 0x00 indicates no scale
 2599     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2600     relocInfo::relocType disp_reloc = relocInfo::none;
 2601     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2602                   disp_reloc);
 2603   %}
 2604 
 2605   enc_class neg_reg(rRegI dst)
 2606   %{
 2607     int dstenc = $dst$$reg;
 2608     if (dstenc >= 8) {
 2609       emit_opcode(cbuf, Assembler::REX_B);
 2610       dstenc -= 8;
 2611     }
 2612     // NEG $dst
 2613     emit_opcode(cbuf, 0xF7);
 2614     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2615   %}
 2616 
 2617   enc_class neg_reg_wide(rRegI dst)
 2618   %{
 2619     int dstenc = $dst$$reg;
 2620     if (dstenc < 8) {
 2621       emit_opcode(cbuf, Assembler::REX_W);
 2622     } else {
 2623       emit_opcode(cbuf, Assembler::REX_WB);
 2624       dstenc -= 8;
 2625     }
 2626     // NEG $dst
 2627     emit_opcode(cbuf, 0xF7);
 2628     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2629   %}
 2630 
 2631   enc_class setLT_reg(rRegI dst)
 2632   %{
 2633     int dstenc = $dst$$reg;
 2634     if (dstenc >= 8) {
 2635       emit_opcode(cbuf, Assembler::REX_B);
 2636       dstenc -= 8;
 2637     } else if (dstenc >= 4) {
 2638       emit_opcode(cbuf, Assembler::REX);
 2639     }
 2640     // SETLT $dst
 2641     emit_opcode(cbuf, 0x0F);
 2642     emit_opcode(cbuf, 0x9C);
 2643     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2644   %}
 2645 
 2646   enc_class setNZ_reg(rRegI dst)
 2647   %{
 2648     int dstenc = $dst$$reg;
 2649     if (dstenc >= 8) {
 2650       emit_opcode(cbuf, Assembler::REX_B);
 2651       dstenc -= 8;
 2652     } else if (dstenc >= 4) {
 2653       emit_opcode(cbuf, Assembler::REX);
 2654     }
 2655     // SETNZ $dst
 2656     emit_opcode(cbuf, 0x0F);
 2657     emit_opcode(cbuf, 0x95);
 2658     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2659   %}
 2660 
 2661 
 2662   // Compare the lonogs and set -1, 0, or 1 into dst
 2663   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2664   %{
 2665     int src1enc = $src1$$reg;
 2666     int src2enc = $src2$$reg;
 2667     int dstenc = $dst$$reg;
 2668 
 2669     // cmpq $src1, $src2
 2670     if (src1enc < 8) {
 2671       if (src2enc < 8) {
 2672         emit_opcode(cbuf, Assembler::REX_W);
 2673       } else {
 2674         emit_opcode(cbuf, Assembler::REX_WB);
 2675       }
 2676     } else {
 2677       if (src2enc < 8) {
 2678         emit_opcode(cbuf, Assembler::REX_WR);
 2679       } else {
 2680         emit_opcode(cbuf, Assembler::REX_WRB);
 2681       }
 2682     }
 2683     emit_opcode(cbuf, 0x3B);
 2684     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2685 
 2686     // movl $dst, -1
 2687     if (dstenc >= 8) {
 2688       emit_opcode(cbuf, Assembler::REX_B);
 2689     }
 2690     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2691     emit_d32(cbuf, -1);
 2692 
 2693     // jl,s done
 2694     emit_opcode(cbuf, 0x7C);
 2695     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2696 
 2697     // setne $dst
 2698     if (dstenc >= 4) {
 2699       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2700     }
 2701     emit_opcode(cbuf, 0x0F);
 2702     emit_opcode(cbuf, 0x95);
 2703     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2704 
 2705     // movzbl $dst, $dst
 2706     if (dstenc >= 4) {
 2707       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2708     }
 2709     emit_opcode(cbuf, 0x0F);
 2710     emit_opcode(cbuf, 0xB6);
 2711     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2712   %}
 2713 
 2714   enc_class Push_ResultXD(regD dst) %{
 2715     MacroAssembler _masm(&cbuf);
 2716     __ fstp_d(Address(rsp, 0));
 2717     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2718     __ addptr(rsp, 8);
 2719   %}
 2720 
 2721   enc_class Push_SrcXD(regD src) %{
 2722     MacroAssembler _masm(&cbuf);
 2723     __ subptr(rsp, 8);
 2724     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2725     __ fld_d(Address(rsp, 0));
 2726   %}
 2727 
 2728 
 2729   enc_class enc_rethrow()
 2730   %{
 2731     cbuf.set_insts_mark();
 2732     emit_opcode(cbuf, 0xE9); // jmp entry
 2733     emit_d32_reloc(cbuf,
 2734                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2735                    runtime_call_Relocation::spec(),
 2736                    RELOC_DISP32);
 2737   %}
 2738 
 2739 %}
 2740 
 2741 
 2742 
 2743 //----------FRAME--------------------------------------------------------------
 2744 // Definition of frame structure and management information.
 2745 //
 2746 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2747 //                             |   (to get allocators register number
 2748 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2749 //  r   CALLER     |        |
 2750 //  o     |        +--------+      pad to even-align allocators stack-slot
 2751 //  w     V        |  pad0  |        numbers; owned by CALLER
 2752 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2753 //  h     ^        |   in   |  5
 2754 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2755 //  |     |        |        |  3
 2756 //  |     |        +--------+
 2757 //  V     |        | old out|      Empty on Intel, window on Sparc
 2758 //        |    old |preserve|      Must be even aligned.
 2759 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2760 //        |        |   in   |  3   area for Intel ret address
 2761 //     Owned by    |preserve|      Empty on Sparc.
 2762 //       SELF      +--------+
 2763 //        |        |  pad2  |  2   pad to align old SP
 2764 //        |        +--------+  1
 2765 //        |        | locks  |  0
 2766 //        |        +--------+----> OptoReg::stack0(), even aligned
 2767 //        |        |  pad1  | 11   pad to align new SP
 2768 //        |        +--------+
 2769 //        |        |        | 10
 2770 //        |        | spills |  9   spills
 2771 //        V        |        |  8   (pad0 slot for callee)
 2772 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2773 //        ^        |  out   |  7
 2774 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2775 //     Owned by    +--------+
 2776 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2777 //        |    new |preserve|      Must be even-aligned.
 2778 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2779 //        |        |        |
 2780 //
 2781 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2782 //         known from SELF's arguments and the Java calling convention.
 2783 //         Region 6-7 is determined per call site.
 2784 // Note 2: If the calling convention leaves holes in the incoming argument
 2785 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2786 //         are owned by the CALLEE.  Holes should not be nessecary in the
 2787 //         incoming area, as the Java calling convention is completely under
 2788 //         the control of the AD file.  Doubles can be sorted and packed to
 2789 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 2790 //         varargs C calling conventions.
 2791 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2792 //         even aligned with pad0 as needed.
 2793 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2794 //         region 6-11 is even aligned; it may be padded out more so that
 2795 //         the region from SP to FP meets the minimum stack alignment.
 2796 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2797 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2798 //         SP meets the minimum alignment.
 2799 
 2800 frame
 2801 %{
 2802   // These three registers define part of the calling convention
 2803   // between compiled code and the interpreter.
 2804   inline_cache_reg(RAX);                // Inline Cache Register
 2805 
 2806   // Optional: name the operand used by cisc-spilling to access
 2807   // [stack_pointer + offset]
 2808   cisc_spilling_operand_name(indOffset32);
 2809 
 2810   // Number of stack slots consumed by locking an object
 2811   sync_stack_slots(2);
 2812 
 2813   // Compiled code's Frame Pointer
 2814   frame_pointer(RSP);
 2815 
 2816   // Interpreter stores its frame pointer in a register which is
 2817   // stored to the stack by I2CAdaptors.
 2818   // I2CAdaptors convert from interpreted java to compiled java.
 2819   interpreter_frame_pointer(RBP);
 2820 
 2821   // Stack alignment requirement
 2822   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2823 
 2824   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2825   // for calls to C.  Supports the var-args backing area for register parms.
 2826   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2827 
 2828   // The after-PROLOG location of the return address.  Location of
 2829   // return address specifies a type (REG or STACK) and a number
 2830   // representing the register number (i.e. - use a register name) or
 2831   // stack slot.
 2832   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2833   // Otherwise, it is above the locks and verification slot and alignment word
 2834   return_addr(STACK - 2 +
 2835               align_up((Compile::current()->in_preserve_stack_slots() +
 2836                         Compile::current()->fixed_slots()),
 2837                        stack_alignment_in_slots()));
 2838 
 2839   // Location of compiled Java return values.  Same as C for now.
 2840   return_value
 2841   %{
 2842     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2843            "only return normal values");
 2844 
 2845     static const int lo[Op_RegL + 1] = {
 2846       0,
 2847       0,
 2848       RAX_num,  // Op_RegN
 2849       RAX_num,  // Op_RegI
 2850       RAX_num,  // Op_RegP
 2851       XMM0_num, // Op_RegF
 2852       XMM0_num, // Op_RegD
 2853       RAX_num   // Op_RegL
 2854     };
 2855     static const int hi[Op_RegL + 1] = {
 2856       0,
 2857       0,
 2858       OptoReg::Bad, // Op_RegN
 2859       OptoReg::Bad, // Op_RegI
 2860       RAX_H_num,    // Op_RegP
 2861       OptoReg::Bad, // Op_RegF
 2862       XMM0b_num,    // Op_RegD
 2863       RAX_H_num     // Op_RegL
 2864     };
 2865     // Excluded flags and vector registers.
 2866     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2867     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2868   %}
 2869 %}
 2870 
 2871 //----------ATTRIBUTES---------------------------------------------------------
 2872 //----------Operand Attributes-------------------------------------------------
 2873 op_attrib op_cost(0);        // Required cost attribute
 2874 
 2875 //----------Instruction Attributes---------------------------------------------
 2876 ins_attrib ins_cost(100);       // Required cost attribute
 2877 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2878 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2879                                 // a non-matching short branch variant
 2880                                 // of some long branch?
 2881 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2882                                 // be a power of 2) specifies the
 2883                                 // alignment that some part of the
 2884                                 // instruction (not necessarily the
 2885                                 // start) requires.  If > 1, a
 2886                                 // compute_padding() function must be
 2887                                 // provided for the instruction
 2888 
 2889 //----------OPERANDS-----------------------------------------------------------
 2890 // Operand definitions must precede instruction definitions for correct parsing
 2891 // in the ADLC because operands constitute user defined types which are used in
 2892 // instruction definitions.
 2893 
 2894 //----------Simple Operands----------------------------------------------------
 2895 // Immediate Operands
 2896 // Integer Immediate
 2897 operand immI()
 2898 %{
 2899   match(ConI);
 2900 
 2901   op_cost(10);
 2902   format %{ %}
 2903   interface(CONST_INTER);
 2904 %}
 2905 
 2906 // Constant for test vs zero
 2907 operand immI_0()
 2908 %{
 2909   predicate(n->get_int() == 0);
 2910   match(ConI);
 2911 
 2912   op_cost(0);
 2913   format %{ %}
 2914   interface(CONST_INTER);
 2915 %}
 2916 
 2917 // Constant for increment
 2918 operand immI_1()
 2919 %{
 2920   predicate(n->get_int() == 1);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 // Constant for decrement
 2929 operand immI_M1()
 2930 %{
 2931   predicate(n->get_int() == -1);
 2932   match(ConI);
 2933 
 2934   op_cost(0);
 2935   format %{ %}
 2936   interface(CONST_INTER);
 2937 %}
 2938 
 2939 operand immI_2()
 2940 %{
 2941   predicate(n->get_int() == 2);
 2942   match(ConI);
 2943 
 2944   op_cost(0);
 2945   format %{ %}
 2946   interface(CONST_INTER);
 2947 %}
 2948 
 2949 operand immI_4()
 2950 %{
 2951   predicate(n->get_int() == 4);
 2952   match(ConI);
 2953 
 2954   op_cost(0);
 2955   format %{ %}
 2956   interface(CONST_INTER);
 2957 %}
 2958 
 2959 operand immI_8()
 2960 %{
 2961   predicate(n->get_int() == 8);
 2962   match(ConI);
 2963 
 2964   op_cost(0);
 2965   format %{ %}
 2966   interface(CONST_INTER);
 2967 %}
 2968 
 2969 // Valid scale values for addressing modes
 2970 operand immI2()
 2971 %{
 2972   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2973   match(ConI);
 2974 
 2975   format %{ %}
 2976   interface(CONST_INTER);
 2977 %}
 2978 
 2979 operand immU7()
 2980 %{
 2981   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2982   match(ConI);
 2983 
 2984   op_cost(5);
 2985   format %{ %}
 2986   interface(CONST_INTER);
 2987 %}
 2988 
 2989 operand immI8()
 2990 %{
 2991   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2992   match(ConI);
 2993 
 2994   op_cost(5);
 2995   format %{ %}
 2996   interface(CONST_INTER);
 2997 %}
 2998 
 2999 operand immU8()
 3000 %{
 3001   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 3002   match(ConI);
 3003 
 3004   op_cost(5);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 operand immI16()
 3010 %{
 3011   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 3012   match(ConI);
 3013 
 3014   op_cost(10);
 3015   format %{ %}
 3016   interface(CONST_INTER);
 3017 %}
 3018 
 3019 // Int Immediate non-negative
 3020 operand immU31()
 3021 %{
 3022   predicate(n->get_int() >= 0);
 3023   match(ConI);
 3024 
 3025   op_cost(0);
 3026   format %{ %}
 3027   interface(CONST_INTER);
 3028 %}
 3029 
 3030 // Constant for long shifts
 3031 operand immI_32()
 3032 %{
 3033   predicate( n->get_int() == 32 );
 3034   match(ConI);
 3035 
 3036   op_cost(0);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // Constant for long shifts
 3042 operand immI_64()
 3043 %{
 3044   predicate( n->get_int() == 64 );
 3045   match(ConI);
 3046 
 3047   op_cost(0);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immP()
 3054 %{
 3055   match(ConP);
 3056 
 3057   op_cost(10);
 3058   format %{ %}
 3059   interface(CONST_INTER);
 3060 %}
 3061 
 3062 // NULL Pointer Immediate
 3063 operand immP0()
 3064 %{
 3065   predicate(n->get_ptr() == 0);
 3066   match(ConP);
 3067 
 3068   op_cost(5);
 3069   format %{ %}
 3070   interface(CONST_INTER);
 3071 %}
 3072 
 3073 // Pointer Immediate
 3074 operand immN() %{
 3075   match(ConN);
 3076 
 3077   op_cost(10);
 3078   format %{ %}
 3079   interface(CONST_INTER);
 3080 %}
 3081 
 3082 operand immNKlass() %{
 3083   match(ConNKlass);
 3084 
 3085   op_cost(10);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 // NULL Pointer Immediate
 3091 operand immN0() %{
 3092   predicate(n->get_narrowcon() == 0);
 3093   match(ConN);
 3094 
 3095   op_cost(5);
 3096   format %{ %}
 3097   interface(CONST_INTER);
 3098 %}
 3099 
 3100 operand immP31()
 3101 %{
 3102   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3103             && (n->get_ptr() >> 31) == 0);
 3104   match(ConP);
 3105 
 3106   op_cost(5);
 3107   format %{ %}
 3108   interface(CONST_INTER);
 3109 %}
 3110 
 3111 
 3112 // Long Immediate
 3113 operand immL()
 3114 %{
 3115   match(ConL);
 3116 
 3117   op_cost(20);
 3118   format %{ %}
 3119   interface(CONST_INTER);
 3120 %}
 3121 
 3122 // Long Immediate 8-bit
 3123 operand immL8()
 3124 %{
 3125   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3126   match(ConL);
 3127 
 3128   op_cost(5);
 3129   format %{ %}
 3130   interface(CONST_INTER);
 3131 %}
 3132 
 3133 // Long Immediate 32-bit unsigned
 3134 operand immUL32()
 3135 %{
 3136   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(10);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 // Long Immediate 32-bit signed
 3145 operand immL32()
 3146 %{
 3147   predicate(n->get_long() == (int) (n->get_long()));
 3148   match(ConL);
 3149 
 3150   op_cost(15);
 3151   format %{ %}
 3152   interface(CONST_INTER);
 3153 %}
 3154 
 3155 operand immL_Pow2()
 3156 %{
 3157   predicate(is_power_of_2((julong)n->get_long()));
 3158   match(ConL);
 3159 
 3160   op_cost(15);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 operand immL_NotPow2()
 3166 %{
 3167   predicate(is_power_of_2((julong)~n->get_long()));
 3168   match(ConL);
 3169 
 3170   op_cost(15);
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Long Immediate zero
 3176 operand immL0()
 3177 %{
 3178   predicate(n->get_long() == 0L);
 3179   match(ConL);
 3180 
 3181   op_cost(10);
 3182   format %{ %}
 3183   interface(CONST_INTER);
 3184 %}
 3185 
 3186 // Constant for increment
 3187 operand immL1()
 3188 %{
 3189   predicate(n->get_long() == 1);
 3190   match(ConL);
 3191 
 3192   format %{ %}
 3193   interface(CONST_INTER);
 3194 %}
 3195 
 3196 // Constant for decrement
 3197 operand immL_M1()
 3198 %{
 3199   predicate(n->get_long() == -1);
 3200   match(ConL);
 3201 
 3202   format %{ %}
 3203   interface(CONST_INTER);
 3204 %}
 3205 
 3206 // Long Immediate: the value 10
 3207 operand immL10()
 3208 %{
 3209   predicate(n->get_long() == 10);
 3210   match(ConL);
 3211 
 3212   format %{ %}
 3213   interface(CONST_INTER);
 3214 %}
 3215 
 3216 // Long immediate from 0 to 127.
 3217 // Used for a shorter form of long mul by 10.
 3218 operand immL_127()
 3219 %{
 3220   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3221   match(ConL);
 3222 
 3223   op_cost(10);
 3224   format %{ %}
 3225   interface(CONST_INTER);
 3226 %}
 3227 
 3228 // Long Immediate: low 32-bit mask
 3229 operand immL_32bits()
 3230 %{
 3231   predicate(n->get_long() == 0xFFFFFFFFL);
 3232   match(ConL);
 3233   op_cost(20);
 3234 
 3235   format %{ %}
 3236   interface(CONST_INTER);
 3237 %}
 3238 
 3239 // Int Immediate: 2^n-1, postive
 3240 operand immI_Pow2M1()
 3241 %{
 3242   predicate((n->get_int() > 0)
 3243             && is_power_of_2(n->get_int() + 1));
 3244   match(ConI);
 3245 
 3246   op_cost(20);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Float Immediate zero
 3252 operand immF0()
 3253 %{
 3254   predicate(jint_cast(n->getf()) == 0);
 3255   match(ConF);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Float Immediate
 3263 operand immF()
 3264 %{
 3265   match(ConF);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Double Immediate zero
 3273 operand immD0()
 3274 %{
 3275   predicate(jlong_cast(n->getd()) == 0);
 3276   match(ConD);
 3277 
 3278   op_cost(5);
 3279   format %{ %}
 3280   interface(CONST_INTER);
 3281 %}
 3282 
 3283 // Double Immediate
 3284 operand immD()
 3285 %{
 3286   match(ConD);
 3287 
 3288   op_cost(15);
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Immediates for special shifts (sign extend)
 3294 
 3295 // Constants for increment
 3296 operand immI_16()
 3297 %{
 3298   predicate(n->get_int() == 16);
 3299   match(ConI);
 3300 
 3301   format %{ %}
 3302   interface(CONST_INTER);
 3303 %}
 3304 
 3305 operand immI_24()
 3306 %{
 3307   predicate(n->get_int() == 24);
 3308   match(ConI);
 3309 
 3310   format %{ %}
 3311   interface(CONST_INTER);
 3312 %}
 3313 
 3314 // Constant for byte-wide masking
 3315 operand immI_255()
 3316 %{
 3317   predicate(n->get_int() == 255);
 3318   match(ConI);
 3319 
 3320   format %{ %}
 3321   interface(CONST_INTER);
 3322 %}
 3323 
 3324 // Constant for short-wide masking
 3325 operand immI_65535()
 3326 %{
 3327   predicate(n->get_int() == 65535);
 3328   match(ConI);
 3329 
 3330   format %{ %}
 3331   interface(CONST_INTER);
 3332 %}
 3333 
 3334 // Constant for byte-wide masking
 3335 operand immL_255()
 3336 %{
 3337   predicate(n->get_long() == 255);
 3338   match(ConL);
 3339 
 3340   format %{ %}
 3341   interface(CONST_INTER);
 3342 %}
 3343 
 3344 // Constant for short-wide masking
 3345 operand immL_65535()
 3346 %{
 3347   predicate(n->get_long() == 65535);
 3348   match(ConL);
 3349 
 3350   format %{ %}
 3351   interface(CONST_INTER);
 3352 %}
 3353 
 3354 operand kReg()
 3355 %{
 3356   constraint(ALLOC_IN_RC(vectmask_reg));
 3357   match(RegVectMask);
 3358   format %{%}
 3359   interface(REG_INTER);
 3360 %}
 3361 
 3362 operand kReg_K1()
 3363 %{
 3364   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3365   match(RegVectMask);
 3366   format %{%}
 3367   interface(REG_INTER);
 3368 %}
 3369 
 3370 operand kReg_K2()
 3371 %{
 3372   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3373   match(RegVectMask);
 3374   format %{%}
 3375   interface(REG_INTER);
 3376 %}
 3377 
 3378 // Special Registers
 3379 operand kReg_K3()
 3380 %{
 3381   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3382   match(RegVectMask);
 3383   format %{%}
 3384   interface(REG_INTER);
 3385 %}
 3386 
 3387 operand kReg_K4()
 3388 %{
 3389   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3390   match(RegVectMask);
 3391   format %{%}
 3392   interface(REG_INTER);
 3393 %}
 3394 
 3395 operand kReg_K5()
 3396 %{
 3397   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3398   match(RegVectMask);
 3399   format %{%}
 3400   interface(REG_INTER);
 3401 %}
 3402 
 3403 operand kReg_K6()
 3404 %{
 3405   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3406   match(RegVectMask);
 3407   format %{%}
 3408   interface(REG_INTER);
 3409 %}
 3410 
 3411 // Special Registers
 3412 operand kReg_K7()
 3413 %{
 3414   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3415   match(RegVectMask);
 3416   format %{%}
 3417   interface(REG_INTER);
 3418 %}
 3419 
 3420 // Register Operands
 3421 // Integer Register
 3422 operand rRegI()
 3423 %{
 3424   constraint(ALLOC_IN_RC(int_reg));
 3425   match(RegI);
 3426 
 3427   match(rax_RegI);
 3428   match(rbx_RegI);
 3429   match(rcx_RegI);
 3430   match(rdx_RegI);
 3431   match(rdi_RegI);
 3432 
 3433   format %{ %}
 3434   interface(REG_INTER);
 3435 %}
 3436 
 3437 // Special Registers
 3438 operand rax_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rax_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RAX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 // Special Registers
 3449 operand rbx_RegI()
 3450 %{
 3451   constraint(ALLOC_IN_RC(int_rbx_reg));
 3452   match(RegI);
 3453   match(rRegI);
 3454 
 3455   format %{ "RBX" %}
 3456   interface(REG_INTER);
 3457 %}
 3458 
 3459 operand rcx_RegI()
 3460 %{
 3461   constraint(ALLOC_IN_RC(int_rcx_reg));
 3462   match(RegI);
 3463   match(rRegI);
 3464 
 3465   format %{ "RCX" %}
 3466   interface(REG_INTER);
 3467 %}
 3468 
 3469 operand rdx_RegI()
 3470 %{
 3471   constraint(ALLOC_IN_RC(int_rdx_reg));
 3472   match(RegI);
 3473   match(rRegI);
 3474 
 3475   format %{ "RDX" %}
 3476   interface(REG_INTER);
 3477 %}
 3478 
 3479 operand rdi_RegI()
 3480 %{
 3481   constraint(ALLOC_IN_RC(int_rdi_reg));
 3482   match(RegI);
 3483   match(rRegI);
 3484 
 3485   format %{ "RDI" %}
 3486   interface(REG_INTER);
 3487 %}
 3488 
 3489 operand no_rax_rdx_RegI()
 3490 %{
 3491   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3492   match(RegI);
 3493   match(rbx_RegI);
 3494   match(rcx_RegI);
 3495   match(rdi_RegI);
 3496 
 3497   format %{ %}
 3498   interface(REG_INTER);
 3499 %}
 3500 
 3501 // Pointer Register
 3502 operand any_RegP()
 3503 %{
 3504   constraint(ALLOC_IN_RC(any_reg));
 3505   match(RegP);
 3506   match(rax_RegP);
 3507   match(rbx_RegP);
 3508   match(rdi_RegP);
 3509   match(rsi_RegP);
 3510   match(rbp_RegP);
 3511   match(r15_RegP);
 3512   match(rRegP);
 3513 
 3514   format %{ %}
 3515   interface(REG_INTER);
 3516 %}
 3517 
 3518 operand rRegP()
 3519 %{
 3520   constraint(ALLOC_IN_RC(ptr_reg));
 3521   match(RegP);
 3522   match(rax_RegP);
 3523   match(rbx_RegP);
 3524   match(rdi_RegP);
 3525   match(rsi_RegP);
 3526   match(rbp_RegP);  // See Q&A below about
 3527   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3528 
 3529   format %{ %}
 3530   interface(REG_INTER);
 3531 %}
 3532 
 3533 operand rRegN() %{
 3534   constraint(ALLOC_IN_RC(int_reg));
 3535   match(RegN);
 3536 
 3537   format %{ %}
 3538   interface(REG_INTER);
 3539 %}
 3540 
 3541 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3542 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3543 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3544 // The output of an instruction is controlled by the allocator, which respects
 3545 // register class masks, not match rules.  Unless an instruction mentions
 3546 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3547 // by the allocator as an input.
 3548 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3549 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3550 // result, RBP is not included in the output of the instruction either.
 3551 
 3552 operand no_rax_RegP()
 3553 %{
 3554   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3555   match(RegP);
 3556   match(rbx_RegP);
 3557   match(rsi_RegP);
 3558   match(rdi_RegP);
 3559 
 3560   format %{ %}
 3561   interface(REG_INTER);
 3562 %}
 3563 
 3564 // This operand is not allowed to use RBP even if
 3565 // RBP is not used to hold the frame pointer.
 3566 operand no_rbp_RegP()
 3567 %{
 3568   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3569   match(RegP);
 3570   match(rbx_RegP);
 3571   match(rsi_RegP);
 3572   match(rdi_RegP);
 3573 
 3574   format %{ %}
 3575   interface(REG_INTER);
 3576 %}
 3577 
 3578 operand no_rax_rbx_RegP()
 3579 %{
 3580   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3581   match(RegP);
 3582   match(rsi_RegP);
 3583   match(rdi_RegP);
 3584 
 3585   format %{ %}
 3586   interface(REG_INTER);
 3587 %}
 3588 
 3589 // Special Registers
 3590 // Return a pointer value
 3591 operand rax_RegP()
 3592 %{
 3593   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3594   match(RegP);
 3595   match(rRegP);
 3596 
 3597   format %{ %}
 3598   interface(REG_INTER);
 3599 %}
 3600 
 3601 // Special Registers
 3602 // Return a compressed pointer value
 3603 operand rax_RegN()
 3604 %{
 3605   constraint(ALLOC_IN_RC(int_rax_reg));
 3606   match(RegN);
 3607   match(rRegN);
 3608 
 3609   format %{ %}
 3610   interface(REG_INTER);
 3611 %}
 3612 
 3613 // Used in AtomicAdd
 3614 operand rbx_RegP()
 3615 %{
 3616   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3617   match(RegP);
 3618   match(rRegP);
 3619 
 3620   format %{ %}
 3621   interface(REG_INTER);
 3622 %}
 3623 
 3624 operand rsi_RegP()
 3625 %{
 3626   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3627   match(RegP);
 3628   match(rRegP);
 3629 
 3630   format %{ %}
 3631   interface(REG_INTER);
 3632 %}
 3633 
 3634 operand rbp_RegP()
 3635 %{
 3636   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3637   match(RegP);
 3638   match(rRegP);
 3639 
 3640   format %{ %}
 3641   interface(REG_INTER);
 3642 %}
 3643 
 3644 // Used in rep stosq
 3645 operand rdi_RegP()
 3646 %{
 3647   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3648   match(RegP);
 3649   match(rRegP);
 3650 
 3651   format %{ %}
 3652   interface(REG_INTER);
 3653 %}
 3654 
 3655 operand r15_RegP()
 3656 %{
 3657   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3658   match(RegP);
 3659   match(rRegP);
 3660 
 3661   format %{ %}
 3662   interface(REG_INTER);
 3663 %}
 3664 
 3665 operand rRegL()
 3666 %{
 3667   constraint(ALLOC_IN_RC(long_reg));
 3668   match(RegL);
 3669   match(rax_RegL);
 3670   match(rdx_RegL);
 3671 
 3672   format %{ %}
 3673   interface(REG_INTER);
 3674 %}
 3675 
 3676 // Special Registers
 3677 operand no_rax_rdx_RegL()
 3678 %{
 3679   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3680   match(RegL);
 3681   match(rRegL);
 3682 
 3683   format %{ %}
 3684   interface(REG_INTER);
 3685 %}
 3686 
 3687 operand no_rax_RegL()
 3688 %{
 3689   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3690   match(RegL);
 3691   match(rRegL);
 3692   match(rdx_RegL);
 3693 
 3694   format %{ %}
 3695   interface(REG_INTER);
 3696 %}
 3697 
 3698 operand rax_RegL()
 3699 %{
 3700   constraint(ALLOC_IN_RC(long_rax_reg));
 3701   match(RegL);
 3702   match(rRegL);
 3703 
 3704   format %{ "RAX" %}
 3705   interface(REG_INTER);
 3706 %}
 3707 
 3708 operand rcx_RegL()
 3709 %{
 3710   constraint(ALLOC_IN_RC(long_rcx_reg));
 3711   match(RegL);
 3712   match(rRegL);
 3713 
 3714   format %{ %}
 3715   interface(REG_INTER);
 3716 %}
 3717 
 3718 operand rdx_RegL()
 3719 %{
 3720   constraint(ALLOC_IN_RC(long_rdx_reg));
 3721   match(RegL);
 3722   match(rRegL);
 3723 
 3724   format %{ %}
 3725   interface(REG_INTER);
 3726 %}
 3727 
 3728 // Flags register, used as output of compare instructions
 3729 operand rFlagsReg()
 3730 %{
 3731   constraint(ALLOC_IN_RC(int_flags));
 3732   match(RegFlags);
 3733 
 3734   format %{ "RFLAGS" %}
 3735   interface(REG_INTER);
 3736 %}
 3737 
 3738 // Flags register, used as output of FLOATING POINT compare instructions
 3739 operand rFlagsRegU()
 3740 %{
 3741   constraint(ALLOC_IN_RC(int_flags));
 3742   match(RegFlags);
 3743 
 3744   format %{ "RFLAGS_U" %}
 3745   interface(REG_INTER);
 3746 %}
 3747 
 3748 operand rFlagsRegUCF() %{
 3749   constraint(ALLOC_IN_RC(int_flags));
 3750   match(RegFlags);
 3751   predicate(false);
 3752 
 3753   format %{ "RFLAGS_U_CF" %}
 3754   interface(REG_INTER);
 3755 %}
 3756 
 3757 // Float register operands
 3758 operand regF() %{
 3759    constraint(ALLOC_IN_RC(float_reg));
 3760    match(RegF);
 3761 
 3762    format %{ %}
 3763    interface(REG_INTER);
 3764 %}
 3765 
 3766 // Float register operands
 3767 operand legRegF() %{
 3768    constraint(ALLOC_IN_RC(float_reg_legacy));
 3769    match(RegF);
 3770 
 3771    format %{ %}
 3772    interface(REG_INTER);
 3773 %}
 3774 
 3775 // Float register operands
 3776 operand vlRegF() %{
 3777    constraint(ALLOC_IN_RC(float_reg_vl));
 3778    match(RegF);
 3779 
 3780    format %{ %}
 3781    interface(REG_INTER);
 3782 %}
 3783 
 3784 // Double register operands
 3785 operand regD() %{
 3786    constraint(ALLOC_IN_RC(double_reg));
 3787    match(RegD);
 3788 
 3789    format %{ %}
 3790    interface(REG_INTER);
 3791 %}
 3792 
 3793 // Double register operands
 3794 operand legRegD() %{
 3795    constraint(ALLOC_IN_RC(double_reg_legacy));
 3796    match(RegD);
 3797 
 3798    format %{ %}
 3799    interface(REG_INTER);
 3800 %}
 3801 
 3802 // Double register operands
 3803 operand vlRegD() %{
 3804    constraint(ALLOC_IN_RC(double_reg_vl));
 3805    match(RegD);
 3806 
 3807    format %{ %}
 3808    interface(REG_INTER);
 3809 %}
 3810 
 3811 //----------Memory Operands----------------------------------------------------
 3812 // Direct Memory Operand
 3813 // operand direct(immP addr)
 3814 // %{
 3815 //   match(addr);
 3816 
 3817 //   format %{ "[$addr]" %}
 3818 //   interface(MEMORY_INTER) %{
 3819 //     base(0xFFFFFFFF);
 3820 //     index(0x4);
 3821 //     scale(0x0);
 3822 //     disp($addr);
 3823 //   %}
 3824 // %}
 3825 
 3826 // Indirect Memory Operand
 3827 operand indirect(any_RegP reg)
 3828 %{
 3829   constraint(ALLOC_IN_RC(ptr_reg));
 3830   match(reg);
 3831 
 3832   format %{ "[$reg]" %}
 3833   interface(MEMORY_INTER) %{
 3834     base($reg);
 3835     index(0x4);
 3836     scale(0x0);
 3837     disp(0x0);
 3838   %}
 3839 %}
 3840 
 3841 // Indirect Memory Plus Short Offset Operand
 3842 operand indOffset8(any_RegP reg, immL8 off)
 3843 %{
 3844   constraint(ALLOC_IN_RC(ptr_reg));
 3845   match(AddP reg off);
 3846 
 3847   format %{ "[$reg + $off (8-bit)]" %}
 3848   interface(MEMORY_INTER) %{
 3849     base($reg);
 3850     index(0x4);
 3851     scale(0x0);
 3852     disp($off);
 3853   %}
 3854 %}
 3855 
 3856 // Indirect Memory Plus Long Offset Operand
 3857 operand indOffset32(any_RegP reg, immL32 off)
 3858 %{
 3859   constraint(ALLOC_IN_RC(ptr_reg));
 3860   match(AddP reg off);
 3861 
 3862   format %{ "[$reg + $off (32-bit)]" %}
 3863   interface(MEMORY_INTER) %{
 3864     base($reg);
 3865     index(0x4);
 3866     scale(0x0);
 3867     disp($off);
 3868   %}
 3869 %}
 3870 
 3871 // Indirect Memory Plus Index Register Plus Offset Operand
 3872 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3873 %{
 3874   constraint(ALLOC_IN_RC(ptr_reg));
 3875   match(AddP (AddP reg lreg) off);
 3876 
 3877   op_cost(10);
 3878   format %{"[$reg + $off + $lreg]" %}
 3879   interface(MEMORY_INTER) %{
 3880     base($reg);
 3881     index($lreg);
 3882     scale(0x0);
 3883     disp($off);
 3884   %}
 3885 %}
 3886 
 3887 // Indirect Memory Plus Index Register Plus Offset Operand
 3888 operand indIndex(any_RegP reg, rRegL lreg)
 3889 %{
 3890   constraint(ALLOC_IN_RC(ptr_reg));
 3891   match(AddP reg lreg);
 3892 
 3893   op_cost(10);
 3894   format %{"[$reg + $lreg]" %}
 3895   interface(MEMORY_INTER) %{
 3896     base($reg);
 3897     index($lreg);
 3898     scale(0x0);
 3899     disp(0x0);
 3900   %}
 3901 %}
 3902 
 3903 // Indirect Memory Times Scale Plus Index Register
 3904 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3905 %{
 3906   constraint(ALLOC_IN_RC(ptr_reg));
 3907   match(AddP reg (LShiftL lreg scale));
 3908 
 3909   op_cost(10);
 3910   format %{"[$reg + $lreg << $scale]" %}
 3911   interface(MEMORY_INTER) %{
 3912     base($reg);
 3913     index($lreg);
 3914     scale($scale);
 3915     disp(0x0);
 3916   %}
 3917 %}
 3918 
 3919 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3920 %{
 3921   constraint(ALLOC_IN_RC(ptr_reg));
 3922   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3923   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3924 
 3925   op_cost(10);
 3926   format %{"[$reg + pos $idx << $scale]" %}
 3927   interface(MEMORY_INTER) %{
 3928     base($reg);
 3929     index($idx);
 3930     scale($scale);
 3931     disp(0x0);
 3932   %}
 3933 %}
 3934 
 3935 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3936 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3937 %{
 3938   constraint(ALLOC_IN_RC(ptr_reg));
 3939   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3940 
 3941   op_cost(10);
 3942   format %{"[$reg + $off + $lreg << $scale]" %}
 3943   interface(MEMORY_INTER) %{
 3944     base($reg);
 3945     index($lreg);
 3946     scale($scale);
 3947     disp($off);
 3948   %}
 3949 %}
 3950 
 3951 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3952 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3953 %{
 3954   constraint(ALLOC_IN_RC(ptr_reg));
 3955   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3956   match(AddP (AddP reg (ConvI2L idx)) off);
 3957 
 3958   op_cost(10);
 3959   format %{"[$reg + $off + $idx]" %}
 3960   interface(MEMORY_INTER) %{
 3961     base($reg);
 3962     index($idx);
 3963     scale(0x0);
 3964     disp($off);
 3965   %}
 3966 %}
 3967 
 3968 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3969 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3970 %{
 3971   constraint(ALLOC_IN_RC(ptr_reg));
 3972   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3973   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3974 
 3975   op_cost(10);
 3976   format %{"[$reg + $off + $idx << $scale]" %}
 3977   interface(MEMORY_INTER) %{
 3978     base($reg);
 3979     index($idx);
 3980     scale($scale);
 3981     disp($off);
 3982   %}
 3983 %}
 3984 
 3985 // Indirect Narrow Oop Plus Offset Operand
 3986 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3987 // we can't free r12 even with CompressedOops::base() == NULL.
 3988 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3989   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3990   constraint(ALLOC_IN_RC(ptr_reg));
 3991   match(AddP (DecodeN reg) off);
 3992 
 3993   op_cost(10);
 3994   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3995   interface(MEMORY_INTER) %{
 3996     base(0xc); // R12
 3997     index($reg);
 3998     scale(0x3);
 3999     disp($off);
 4000   %}
 4001 %}
 4002 
 4003 // Indirect Memory Operand
 4004 operand indirectNarrow(rRegN reg)
 4005 %{
 4006   predicate(CompressedOops::shift() == 0);
 4007   constraint(ALLOC_IN_RC(ptr_reg));
 4008   match(DecodeN reg);
 4009 
 4010   format %{ "[$reg]" %}
 4011   interface(MEMORY_INTER) %{
 4012     base($reg);
 4013     index(0x4);
 4014     scale(0x0);
 4015     disp(0x0);
 4016   %}
 4017 %}
 4018 
 4019 // Indirect Memory Plus Short Offset Operand
 4020 operand indOffset8Narrow(rRegN reg, immL8 off)
 4021 %{
 4022   predicate(CompressedOops::shift() == 0);
 4023   constraint(ALLOC_IN_RC(ptr_reg));
 4024   match(AddP (DecodeN reg) off);
 4025 
 4026   format %{ "[$reg + $off (8-bit)]" %}
 4027   interface(MEMORY_INTER) %{
 4028     base($reg);
 4029     index(0x4);
 4030     scale(0x0);
 4031     disp($off);
 4032   %}
 4033 %}
 4034 
 4035 // Indirect Memory Plus Long Offset Operand
 4036 operand indOffset32Narrow(rRegN reg, immL32 off)
 4037 %{
 4038   predicate(CompressedOops::shift() == 0);
 4039   constraint(ALLOC_IN_RC(ptr_reg));
 4040   match(AddP (DecodeN reg) off);
 4041 
 4042   format %{ "[$reg + $off (32-bit)]" %}
 4043   interface(MEMORY_INTER) %{
 4044     base($reg);
 4045     index(0x4);
 4046     scale(0x0);
 4047     disp($off);
 4048   %}
 4049 %}
 4050 
 4051 // Indirect Memory Plus Index Register Plus Offset Operand
 4052 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4053 %{
 4054   predicate(CompressedOops::shift() == 0);
 4055   constraint(ALLOC_IN_RC(ptr_reg));
 4056   match(AddP (AddP (DecodeN reg) lreg) off);
 4057 
 4058   op_cost(10);
 4059   format %{"[$reg + $off + $lreg]" %}
 4060   interface(MEMORY_INTER) %{
 4061     base($reg);
 4062     index($lreg);
 4063     scale(0x0);
 4064     disp($off);
 4065   %}
 4066 %}
 4067 
 4068 // Indirect Memory Plus Index Register Plus Offset Operand
 4069 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4070 %{
 4071   predicate(CompressedOops::shift() == 0);
 4072   constraint(ALLOC_IN_RC(ptr_reg));
 4073   match(AddP (DecodeN reg) lreg);
 4074 
 4075   op_cost(10);
 4076   format %{"[$reg + $lreg]" %}
 4077   interface(MEMORY_INTER) %{
 4078     base($reg);
 4079     index($lreg);
 4080     scale(0x0);
 4081     disp(0x0);
 4082   %}
 4083 %}
 4084 
 4085 // Indirect Memory Times Scale Plus Index Register
 4086 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4087 %{
 4088   predicate(CompressedOops::shift() == 0);
 4089   constraint(ALLOC_IN_RC(ptr_reg));
 4090   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4091 
 4092   op_cost(10);
 4093   format %{"[$reg + $lreg << $scale]" %}
 4094   interface(MEMORY_INTER) %{
 4095     base($reg);
 4096     index($lreg);
 4097     scale($scale);
 4098     disp(0x0);
 4099   %}
 4100 %}
 4101 
 4102 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4103 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4104 %{
 4105   predicate(CompressedOops::shift() == 0);
 4106   constraint(ALLOC_IN_RC(ptr_reg));
 4107   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4108 
 4109   op_cost(10);
 4110   format %{"[$reg + $off + $lreg << $scale]" %}
 4111   interface(MEMORY_INTER) %{
 4112     base($reg);
 4113     index($lreg);
 4114     scale($scale);
 4115     disp($off);
 4116   %}
 4117 %}
 4118 
 4119 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4120 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4121 %{
 4122   constraint(ALLOC_IN_RC(ptr_reg));
 4123   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4124   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4125 
 4126   op_cost(10);
 4127   format %{"[$reg + $off + $idx]" %}
 4128   interface(MEMORY_INTER) %{
 4129     base($reg);
 4130     index($idx);
 4131     scale(0x0);
 4132     disp($off);
 4133   %}
 4134 %}
 4135 
 4136 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4137 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4138 %{
 4139   constraint(ALLOC_IN_RC(ptr_reg));
 4140   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4141   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4142 
 4143   op_cost(10);
 4144   format %{"[$reg + $off + $idx << $scale]" %}
 4145   interface(MEMORY_INTER) %{
 4146     base($reg);
 4147     index($idx);
 4148     scale($scale);
 4149     disp($off);
 4150   %}
 4151 %}
 4152 
 4153 //----------Special Memory Operands--------------------------------------------
 4154 // Stack Slot Operand - This operand is used for loading and storing temporary
 4155 //                      values on the stack where a match requires a value to
 4156 //                      flow through memory.
 4157 operand stackSlotP(sRegP reg)
 4158 %{
 4159   constraint(ALLOC_IN_RC(stack_slots));
 4160   // No match rule because this operand is only generated in matching
 4161 
 4162   format %{ "[$reg]" %}
 4163   interface(MEMORY_INTER) %{
 4164     base(0x4);   // RSP
 4165     index(0x4);  // No Index
 4166     scale(0x0);  // No Scale
 4167     disp($reg);  // Stack Offset
 4168   %}
 4169 %}
 4170 
 4171 operand stackSlotI(sRegI reg)
 4172 %{
 4173   constraint(ALLOC_IN_RC(stack_slots));
 4174   // No match rule because this operand is only generated in matching
 4175 
 4176   format %{ "[$reg]" %}
 4177   interface(MEMORY_INTER) %{
 4178     base(0x4);   // RSP
 4179     index(0x4);  // No Index
 4180     scale(0x0);  // No Scale
 4181     disp($reg);  // Stack Offset
 4182   %}
 4183 %}
 4184 
 4185 operand stackSlotF(sRegF reg)
 4186 %{
 4187   constraint(ALLOC_IN_RC(stack_slots));
 4188   // No match rule because this operand is only generated in matching
 4189 
 4190   format %{ "[$reg]" %}
 4191   interface(MEMORY_INTER) %{
 4192     base(0x4);   // RSP
 4193     index(0x4);  // No Index
 4194     scale(0x0);  // No Scale
 4195     disp($reg);  // Stack Offset
 4196   %}
 4197 %}
 4198 
 4199 operand stackSlotD(sRegD reg)
 4200 %{
 4201   constraint(ALLOC_IN_RC(stack_slots));
 4202   // No match rule because this operand is only generated in matching
 4203 
 4204   format %{ "[$reg]" %}
 4205   interface(MEMORY_INTER) %{
 4206     base(0x4);   // RSP
 4207     index(0x4);  // No Index
 4208     scale(0x0);  // No Scale
 4209     disp($reg);  // Stack Offset
 4210   %}
 4211 %}
 4212 operand stackSlotL(sRegL reg)
 4213 %{
 4214   constraint(ALLOC_IN_RC(stack_slots));
 4215   // No match rule because this operand is only generated in matching
 4216 
 4217   format %{ "[$reg]" %}
 4218   interface(MEMORY_INTER) %{
 4219     base(0x4);   // RSP
 4220     index(0x4);  // No Index
 4221     scale(0x0);  // No Scale
 4222     disp($reg);  // Stack Offset
 4223   %}
 4224 %}
 4225 
 4226 //----------Conditional Branch Operands----------------------------------------
 4227 // Comparison Op  - This is the operation of the comparison, and is limited to
 4228 //                  the following set of codes:
 4229 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4230 //
 4231 // Other attributes of the comparison, such as unsignedness, are specified
 4232 // by the comparison instruction that sets a condition code flags register.
 4233 // That result is represented by a flags operand whose subtype is appropriate
 4234 // to the unsignedness (etc.) of the comparison.
 4235 //
 4236 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4237 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4238 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4239 
 4240 // Comparision Code
 4241 operand cmpOp()
 4242 %{
 4243   match(Bool);
 4244 
 4245   format %{ "" %}
 4246   interface(COND_INTER) %{
 4247     equal(0x4, "e");
 4248     not_equal(0x5, "ne");
 4249     less(0xC, "l");
 4250     greater_equal(0xD, "ge");
 4251     less_equal(0xE, "le");
 4252     greater(0xF, "g");
 4253     overflow(0x0, "o");
 4254     no_overflow(0x1, "no");
 4255   %}
 4256 %}
 4257 
 4258 // Comparison Code, unsigned compare.  Used by FP also, with
 4259 // C2 (unordered) turned into GT or LT already.  The other bits
 4260 // C0 and C3 are turned into Carry & Zero flags.
 4261 operand cmpOpU()
 4262 %{
 4263   match(Bool);
 4264 
 4265   format %{ "" %}
 4266   interface(COND_INTER) %{
 4267     equal(0x4, "e");
 4268     not_equal(0x5, "ne");
 4269     less(0x2, "b");
 4270     greater_equal(0x3, "nb");
 4271     less_equal(0x6, "be");
 4272     greater(0x7, "nbe");
 4273     overflow(0x0, "o");
 4274     no_overflow(0x1, "no");
 4275   %}
 4276 %}
 4277 
 4278 
 4279 // Floating comparisons that don't require any fixup for the unordered case
 4280 operand cmpOpUCF() %{
 4281   match(Bool);
 4282   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4283             n->as_Bool()->_test._test == BoolTest::ge ||
 4284             n->as_Bool()->_test._test == BoolTest::le ||
 4285             n->as_Bool()->_test._test == BoolTest::gt);
 4286   format %{ "" %}
 4287   interface(COND_INTER) %{
 4288     equal(0x4, "e");
 4289     not_equal(0x5, "ne");
 4290     less(0x2, "b");
 4291     greater_equal(0x3, "nb");
 4292     less_equal(0x6, "be");
 4293     greater(0x7, "nbe");
 4294     overflow(0x0, "o");
 4295     no_overflow(0x1, "no");
 4296   %}
 4297 %}
 4298 
 4299 
 4300 // Floating comparisons that can be fixed up with extra conditional jumps
 4301 operand cmpOpUCF2() %{
 4302   match(Bool);
 4303   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4304             n->as_Bool()->_test._test == BoolTest::eq);
 4305   format %{ "" %}
 4306   interface(COND_INTER) %{
 4307     equal(0x4, "e");
 4308     not_equal(0x5, "ne");
 4309     less(0x2, "b");
 4310     greater_equal(0x3, "nb");
 4311     less_equal(0x6, "be");
 4312     greater(0x7, "nbe");
 4313     overflow(0x0, "o");
 4314     no_overflow(0x1, "no");
 4315   %}
 4316 %}
 4317 
 4318 //----------OPERAND CLASSES----------------------------------------------------
 4319 // Operand Classes are groups of operands that are used as to simplify
 4320 // instruction definitions by not requiring the AD writer to specify separate
 4321 // instructions for every form of operand when the instruction accepts
 4322 // multiple operand types with the same basic encoding and format.  The classic
 4323 // case of this is memory operands.
 4324 
 4325 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4326                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4327                indCompressedOopOffset,
 4328                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4329                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4330                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4331 
 4332 //----------PIPELINE-----------------------------------------------------------
 4333 // Rules which define the behavior of the target architectures pipeline.
 4334 pipeline %{
 4335 
 4336 //----------ATTRIBUTES---------------------------------------------------------
 4337 attributes %{
 4338   variable_size_instructions;        // Fixed size instructions
 4339   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4340   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4341   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4342   instruction_fetch_units = 1;       // of 16 bytes
 4343 
 4344   // List of nop instructions
 4345   nops( MachNop );
 4346 %}
 4347 
 4348 //----------RESOURCES----------------------------------------------------------
 4349 // Resources are the functional units available to the machine
 4350 
 4351 // Generic P2/P3 pipeline
 4352 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4353 // 3 instructions decoded per cycle.
 4354 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4355 // 3 ALU op, only ALU0 handles mul instructions.
 4356 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4357            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4358            BR, FPU,
 4359            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4360 
 4361 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4362 // Pipeline Description specifies the stages in the machine's pipeline
 4363 
 4364 // Generic P2/P3 pipeline
 4365 pipe_desc(S0, S1, S2, S3, S4, S5);
 4366 
 4367 //----------PIPELINE CLASSES---------------------------------------------------
 4368 // Pipeline Classes describe the stages in which input and output are
 4369 // referenced by the hardware pipeline.
 4370 
 4371 // Naming convention: ialu or fpu
 4372 // Then: _reg
 4373 // Then: _reg if there is a 2nd register
 4374 // Then: _long if it's a pair of instructions implementing a long
 4375 // Then: _fat if it requires the big decoder
 4376 //   Or: _mem if it requires the big decoder and a memory unit.
 4377 
 4378 // Integer ALU reg operation
 4379 pipe_class ialu_reg(rRegI dst)
 4380 %{
 4381     single_instruction;
 4382     dst    : S4(write);
 4383     dst    : S3(read);
 4384     DECODE : S0;        // any decoder
 4385     ALU    : S3;        // any alu
 4386 %}
 4387 
 4388 // Long ALU reg operation
 4389 pipe_class ialu_reg_long(rRegL dst)
 4390 %{
 4391     instruction_count(2);
 4392     dst    : S4(write);
 4393     dst    : S3(read);
 4394     DECODE : S0(2);     // any 2 decoders
 4395     ALU    : S3(2);     // both alus
 4396 %}
 4397 
 4398 // Integer ALU reg operation using big decoder
 4399 pipe_class ialu_reg_fat(rRegI dst)
 4400 %{
 4401     single_instruction;
 4402     dst    : S4(write);
 4403     dst    : S3(read);
 4404     D0     : S0;        // big decoder only
 4405     ALU    : S3;        // any alu
 4406 %}
 4407 
 4408 // Integer ALU reg-reg operation
 4409 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4410 %{
 4411     single_instruction;
 4412     dst    : S4(write);
 4413     src    : S3(read);
 4414     DECODE : S0;        // any decoder
 4415     ALU    : S3;        // any alu
 4416 %}
 4417 
 4418 // Integer ALU reg-reg operation
 4419 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4420 %{
 4421     single_instruction;
 4422     dst    : S4(write);
 4423     src    : S3(read);
 4424     D0     : S0;        // big decoder only
 4425     ALU    : S3;        // any alu
 4426 %}
 4427 
 4428 // Integer ALU reg-mem operation
 4429 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4430 %{
 4431     single_instruction;
 4432     dst    : S5(write);
 4433     mem    : S3(read);
 4434     D0     : S0;        // big decoder only
 4435     ALU    : S4;        // any alu
 4436     MEM    : S3;        // any mem
 4437 %}
 4438 
 4439 // Integer mem operation (prefetch)
 4440 pipe_class ialu_mem(memory mem)
 4441 %{
 4442     single_instruction;
 4443     mem    : S3(read);
 4444     D0     : S0;        // big decoder only
 4445     MEM    : S3;        // any mem
 4446 %}
 4447 
 4448 // Integer Store to Memory
 4449 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4450 %{
 4451     single_instruction;
 4452     mem    : S3(read);
 4453     src    : S5(read);
 4454     D0     : S0;        // big decoder only
 4455     ALU    : S4;        // any alu
 4456     MEM    : S3;
 4457 %}
 4458 
 4459 // // Long Store to Memory
 4460 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4461 // %{
 4462 //     instruction_count(2);
 4463 //     mem    : S3(read);
 4464 //     src    : S5(read);
 4465 //     D0     : S0(2);          // big decoder only; twice
 4466 //     ALU    : S4(2);     // any 2 alus
 4467 //     MEM    : S3(2);  // Both mems
 4468 // %}
 4469 
 4470 // Integer Store to Memory
 4471 pipe_class ialu_mem_imm(memory mem)
 4472 %{
 4473     single_instruction;
 4474     mem    : S3(read);
 4475     D0     : S0;        // big decoder only
 4476     ALU    : S4;        // any alu
 4477     MEM    : S3;
 4478 %}
 4479 
 4480 // Integer ALU0 reg-reg operation
 4481 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4482 %{
 4483     single_instruction;
 4484     dst    : S4(write);
 4485     src    : S3(read);
 4486     D0     : S0;        // Big decoder only
 4487     ALU0   : S3;        // only alu0
 4488 %}
 4489 
 4490 // Integer ALU0 reg-mem operation
 4491 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4492 %{
 4493     single_instruction;
 4494     dst    : S5(write);
 4495     mem    : S3(read);
 4496     D0     : S0;        // big decoder only
 4497     ALU0   : S4;        // ALU0 only
 4498     MEM    : S3;        // any mem
 4499 %}
 4500 
 4501 // Integer ALU reg-reg operation
 4502 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4503 %{
 4504     single_instruction;
 4505     cr     : S4(write);
 4506     src1   : S3(read);
 4507     src2   : S3(read);
 4508     DECODE : S0;        // any decoder
 4509     ALU    : S3;        // any alu
 4510 %}
 4511 
 4512 // Integer ALU reg-imm operation
 4513 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     DECODE : S0;        // any decoder
 4519     ALU    : S3;        // any alu
 4520 %}
 4521 
 4522 // Integer ALU reg-mem operation
 4523 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4524 %{
 4525     single_instruction;
 4526     cr     : S4(write);
 4527     src1   : S3(read);
 4528     src2   : S3(read);
 4529     D0     : S0;        // big decoder only
 4530     ALU    : S4;        // any alu
 4531     MEM    : S3;
 4532 %}
 4533 
 4534 // Conditional move reg-reg
 4535 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4536 %{
 4537     instruction_count(4);
 4538     y      : S4(read);
 4539     q      : S3(read);
 4540     p      : S3(read);
 4541     DECODE : S0(4);     // any decoder
 4542 %}
 4543 
 4544 // Conditional move reg-reg
 4545 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4546 %{
 4547     single_instruction;
 4548     dst    : S4(write);
 4549     src    : S3(read);
 4550     cr     : S3(read);
 4551     DECODE : S0;        // any decoder
 4552 %}
 4553 
 4554 // Conditional move reg-mem
 4555 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4556 %{
 4557     single_instruction;
 4558     dst    : S4(write);
 4559     src    : S3(read);
 4560     cr     : S3(read);
 4561     DECODE : S0;        // any decoder
 4562     MEM    : S3;
 4563 %}
 4564 
 4565 // Conditional move reg-reg long
 4566 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0(2);     // any 2 decoders
 4573 %}
 4574 
 4575 // XXX
 4576 // // Conditional move double reg-reg
 4577 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4578 // %{
 4579 //     single_instruction;
 4580 //     dst    : S4(write);
 4581 //     src    : S3(read);
 4582 //     cr     : S3(read);
 4583 //     DECODE : S0;     // any decoder
 4584 // %}
 4585 
 4586 // Float reg-reg operation
 4587 pipe_class fpu_reg(regD dst)
 4588 %{
 4589     instruction_count(2);
 4590     dst    : S3(read);
 4591     DECODE : S0(2);     // any 2 decoders
 4592     FPU    : S3;
 4593 %}
 4594 
 4595 // Float reg-reg operation
 4596 pipe_class fpu_reg_reg(regD dst, regD src)
 4597 %{
 4598     instruction_count(2);
 4599     dst    : S4(write);
 4600     src    : S3(read);
 4601     DECODE : S0(2);     // any 2 decoders
 4602     FPU    : S3;
 4603 %}
 4604 
 4605 // Float reg-reg operation
 4606 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4607 %{
 4608     instruction_count(3);
 4609     dst    : S4(write);
 4610     src1   : S3(read);
 4611     src2   : S3(read);
 4612     DECODE : S0(3);     // any 3 decoders
 4613     FPU    : S3(2);
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4618 %{
 4619     instruction_count(4);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     src3   : S3(read);
 4624     DECODE : S0(4);     // any 3 decoders
 4625     FPU    : S3(2);
 4626 %}
 4627 
 4628 // Float reg-reg operation
 4629 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4630 %{
 4631     instruction_count(4);
 4632     dst    : S4(write);
 4633     src1   : S3(read);
 4634     src2   : S3(read);
 4635     src3   : S3(read);
 4636     DECODE : S1(3);     // any 3 decoders
 4637     D0     : S0;        // Big decoder only
 4638     FPU    : S3(2);
 4639     MEM    : S3;
 4640 %}
 4641 
 4642 // Float reg-mem operation
 4643 pipe_class fpu_reg_mem(regD dst, memory mem)
 4644 %{
 4645     instruction_count(2);
 4646     dst    : S5(write);
 4647     mem    : S3(read);
 4648     D0     : S0;        // big decoder only
 4649     DECODE : S1;        // any decoder for FPU POP
 4650     FPU    : S4;
 4651     MEM    : S3;        // any mem
 4652 %}
 4653 
 4654 // Float reg-mem operation
 4655 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4656 %{
 4657     instruction_count(3);
 4658     dst    : S5(write);
 4659     src1   : S3(read);
 4660     mem    : S3(read);
 4661     D0     : S0;        // big decoder only
 4662     DECODE : S1(2);     // any decoder for FPU POP
 4663     FPU    : S4;
 4664     MEM    : S3;        // any mem
 4665 %}
 4666 
 4667 // Float mem-reg operation
 4668 pipe_class fpu_mem_reg(memory mem, regD src)
 4669 %{
 4670     instruction_count(2);
 4671     src    : S5(read);
 4672     mem    : S3(read);
 4673     DECODE : S0;        // any decoder for FPU PUSH
 4674     D0     : S1;        // big decoder only
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4680 %{
 4681     instruction_count(3);
 4682     src1   : S3(read);
 4683     src2   : S3(read);
 4684     mem    : S3(read);
 4685     DECODE : S0(2);     // any decoder for FPU PUSH
 4686     D0     : S1;        // big decoder only
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4692 %{
 4693     instruction_count(3);
 4694     src1   : S3(read);
 4695     src2   : S3(read);
 4696     mem    : S4(read);
 4697     DECODE : S0;        // any decoder for FPU PUSH
 4698     D0     : S0(2);     // big decoder only
 4699     FPU    : S4;
 4700     MEM    : S3(2);     // any mem
 4701 %}
 4702 
 4703 pipe_class fpu_mem_mem(memory dst, memory src1)
 4704 %{
 4705     instruction_count(2);
 4706     src1   : S3(read);
 4707     dst    : S4(read);
 4708     D0     : S0(2);     // big decoder only
 4709     MEM    : S3(2);     // any mem
 4710 %}
 4711 
 4712 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4713 %{
 4714     instruction_count(3);
 4715     src1   : S3(read);
 4716     src2   : S3(read);
 4717     dst    : S4(read);
 4718     D0     : S0(3);     // big decoder only
 4719     FPU    : S4;
 4720     MEM    : S3(3);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S4(read);
 4727     mem    : S4(read);
 4728     DECODE : S0;        // any decoder for FPU PUSH
 4729     D0     : S0(2);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(2);     // any mem
 4732 %}
 4733 
 4734 // Float load constant
 4735 pipe_class fpu_reg_con(regD dst)
 4736 %{
 4737     instruction_count(2);
 4738     dst    : S5(write);
 4739     D0     : S0;        // big decoder only for the load
 4740     DECODE : S1;        // any decoder for FPU POP
 4741     FPU    : S4;
 4742     MEM    : S3;        // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4747 %{
 4748     instruction_count(3);
 4749     dst    : S5(write);
 4750     src    : S3(read);
 4751     D0     : S0;        // big decoder only for the load
 4752     DECODE : S1(2);     // any decoder for FPU POP
 4753     FPU    : S4;
 4754     MEM    : S3;        // any mem
 4755 %}
 4756 
 4757 // UnConditional branch
 4758 pipe_class pipe_jmp(label labl)
 4759 %{
 4760     single_instruction;
 4761     BR   : S3;
 4762 %}
 4763 
 4764 // Conditional branch
 4765 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4766 %{
 4767     single_instruction;
 4768     cr    : S1(read);
 4769     BR    : S3;
 4770 %}
 4771 
 4772 // Allocation idiom
 4773 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4774 %{
 4775     instruction_count(1); force_serialization;
 4776     fixed_latency(6);
 4777     heap_ptr : S3(read);
 4778     DECODE   : S0(3);
 4779     D0       : S2;
 4780     MEM      : S3;
 4781     ALU      : S3(2);
 4782     dst      : S5(write);
 4783     BR       : S5;
 4784 %}
 4785 
 4786 // Generic big/slow expanded idiom
 4787 pipe_class pipe_slow()
 4788 %{
 4789     instruction_count(10); multiple_bundles; force_serialization;
 4790     fixed_latency(100);
 4791     D0  : S0(2);
 4792     MEM : S3(2);
 4793 %}
 4794 
 4795 // The real do-nothing guy
 4796 pipe_class empty()
 4797 %{
 4798     instruction_count(0);
 4799 %}
 4800 
 4801 // Define the class for the Nop node
 4802 define
 4803 %{
 4804    MachNop = empty;
 4805 %}
 4806 
 4807 %}
 4808 
 4809 //----------INSTRUCTIONS-------------------------------------------------------
 4810 //
 4811 // match      -- States which machine-independent subtree may be replaced
 4812 //               by this instruction.
 4813 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4814 //               selection to identify a minimum cost tree of machine
 4815 //               instructions that matches a tree of machine-independent
 4816 //               instructions.
 4817 // format     -- A string providing the disassembly for this instruction.
 4818 //               The value of an instruction's operand may be inserted
 4819 //               by referring to it with a '$' prefix.
 4820 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4821 //               to within an encode class as $primary, $secondary, and $tertiary
 4822 //               rrspectively.  The primary opcode is commonly used to
 4823 //               indicate the type of machine instruction, while secondary
 4824 //               and tertiary are often used for prefix options or addressing
 4825 //               modes.
 4826 // ins_encode -- A list of encode classes with parameters. The encode class
 4827 //               name must have been defined in an 'enc_class' specification
 4828 //               in the encode section of the architecture description.
 4829 
 4830 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4831 // Load Float
 4832 instruct MoveF2VL(vlRegF dst, regF src) %{
 4833   match(Set dst src);
 4834   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4835   ins_encode %{
 4836     ShouldNotReachHere();
 4837   %}
 4838   ins_pipe( fpu_reg_reg );
 4839 %}
 4840 
 4841 // Load Float
 4842 instruct MoveF2LEG(legRegF dst, regF src) %{
 4843   match(Set dst src);
 4844   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4845   ins_encode %{
 4846     ShouldNotReachHere();
 4847   %}
 4848   ins_pipe( fpu_reg_reg );
 4849 %}
 4850 
 4851 // Load Float
 4852 instruct MoveVL2F(regF dst, vlRegF src) %{
 4853   match(Set dst src);
 4854   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4855   ins_encode %{
 4856     ShouldNotReachHere();
 4857   %}
 4858   ins_pipe( fpu_reg_reg );
 4859 %}
 4860 
 4861 // Load Float
 4862 instruct MoveLEG2F(regF dst, legRegF src) %{
 4863   match(Set dst src);
 4864   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4865   ins_encode %{
 4866     ShouldNotReachHere();
 4867   %}
 4868   ins_pipe( fpu_reg_reg );
 4869 %}
 4870 
 4871 // Load Double
 4872 instruct MoveD2VL(vlRegD dst, regD src) %{
 4873   match(Set dst src);
 4874   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4875   ins_encode %{
 4876     ShouldNotReachHere();
 4877   %}
 4878   ins_pipe( fpu_reg_reg );
 4879 %}
 4880 
 4881 // Load Double
 4882 instruct MoveD2LEG(legRegD dst, regD src) %{
 4883   match(Set dst src);
 4884   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4885   ins_encode %{
 4886     ShouldNotReachHere();
 4887   %}
 4888   ins_pipe( fpu_reg_reg );
 4889 %}
 4890 
 4891 // Load Double
 4892 instruct MoveVL2D(regD dst, vlRegD src) %{
 4893   match(Set dst src);
 4894   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4895   ins_encode %{
 4896     ShouldNotReachHere();
 4897   %}
 4898   ins_pipe( fpu_reg_reg );
 4899 %}
 4900 
 4901 // Load Double
 4902 instruct MoveLEG2D(regD dst, legRegD src) %{
 4903   match(Set dst src);
 4904   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4905   ins_encode %{
 4906     ShouldNotReachHere();
 4907   %}
 4908   ins_pipe( fpu_reg_reg );
 4909 %}
 4910 
 4911 //----------Load/Store/Move Instructions---------------------------------------
 4912 //----------Load Instructions--------------------------------------------------
 4913 
 4914 // Load Byte (8 bit signed)
 4915 instruct loadB(rRegI dst, memory mem)
 4916 %{
 4917   match(Set dst (LoadB mem));
 4918 
 4919   ins_cost(125);
 4920   format %{ "movsbl  $dst, $mem\t# byte" %}
 4921 
 4922   ins_encode %{
 4923     __ movsbl($dst$$Register, $mem$$Address);
 4924   %}
 4925 
 4926   ins_pipe(ialu_reg_mem);
 4927 %}
 4928 
 4929 // Load Byte (8 bit signed) into Long Register
 4930 instruct loadB2L(rRegL dst, memory mem)
 4931 %{
 4932   match(Set dst (ConvI2L (LoadB mem)));
 4933 
 4934   ins_cost(125);
 4935   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4936 
 4937   ins_encode %{
 4938     __ movsbq($dst$$Register, $mem$$Address);
 4939   %}
 4940 
 4941   ins_pipe(ialu_reg_mem);
 4942 %}
 4943 
 4944 // Load Unsigned Byte (8 bit UNsigned)
 4945 instruct loadUB(rRegI dst, memory mem)
 4946 %{
 4947   match(Set dst (LoadUB mem));
 4948 
 4949   ins_cost(125);
 4950   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4951 
 4952   ins_encode %{
 4953     __ movzbl($dst$$Register, $mem$$Address);
 4954   %}
 4955 
 4956   ins_pipe(ialu_reg_mem);
 4957 %}
 4958 
 4959 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4960 instruct loadUB2L(rRegL dst, memory mem)
 4961 %{
 4962   match(Set dst (ConvI2L (LoadUB mem)));
 4963 
 4964   ins_cost(125);
 4965   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4966 
 4967   ins_encode %{
 4968     __ movzbq($dst$$Register, $mem$$Address);
 4969   %}
 4970 
 4971   ins_pipe(ialu_reg_mem);
 4972 %}
 4973 
 4974 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4975 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4976   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4977   effect(KILL cr);
 4978 
 4979   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4980             "andl    $dst, right_n_bits($mask, 8)" %}
 4981   ins_encode %{
 4982     Register Rdst = $dst$$Register;
 4983     __ movzbq(Rdst, $mem$$Address);
 4984     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4985   %}
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 // Load Short (16 bit signed)
 4990 instruct loadS(rRegI dst, memory mem)
 4991 %{
 4992   match(Set dst (LoadS mem));
 4993 
 4994   ins_cost(125);
 4995   format %{ "movswl $dst, $mem\t# short" %}
 4996 
 4997   ins_encode %{
 4998     __ movswl($dst$$Register, $mem$$Address);
 4999   %}
 5000 
 5001   ins_pipe(ialu_reg_mem);
 5002 %}
 5003 
 5004 // Load Short (16 bit signed) to Byte (8 bit signed)
 5005 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5006   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5007 
 5008   ins_cost(125);
 5009   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5010   ins_encode %{
 5011     __ movsbl($dst$$Register, $mem$$Address);
 5012   %}
 5013   ins_pipe(ialu_reg_mem);
 5014 %}
 5015 
 5016 // Load Short (16 bit signed) into Long Register
 5017 instruct loadS2L(rRegL dst, memory mem)
 5018 %{
 5019   match(Set dst (ConvI2L (LoadS mem)));
 5020 
 5021   ins_cost(125);
 5022   format %{ "movswq $dst, $mem\t# short -> long" %}
 5023 
 5024   ins_encode %{
 5025     __ movswq($dst$$Register, $mem$$Address);
 5026   %}
 5027 
 5028   ins_pipe(ialu_reg_mem);
 5029 %}
 5030 
 5031 // Load Unsigned Short/Char (16 bit UNsigned)
 5032 instruct loadUS(rRegI dst, memory mem)
 5033 %{
 5034   match(Set dst (LoadUS mem));
 5035 
 5036   ins_cost(125);
 5037   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5038 
 5039   ins_encode %{
 5040     __ movzwl($dst$$Register, $mem$$Address);
 5041   %}
 5042 
 5043   ins_pipe(ialu_reg_mem);
 5044 %}
 5045 
 5046 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5047 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5048   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5049 
 5050   ins_cost(125);
 5051   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5052   ins_encode %{
 5053     __ movsbl($dst$$Register, $mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_reg_mem);
 5056 %}
 5057 
 5058 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5059 instruct loadUS2L(rRegL dst, memory mem)
 5060 %{
 5061   match(Set dst (ConvI2L (LoadUS mem)));
 5062 
 5063   ins_cost(125);
 5064   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5065 
 5066   ins_encode %{
 5067     __ movzwq($dst$$Register, $mem$$Address);
 5068   %}
 5069 
 5070   ins_pipe(ialu_reg_mem);
 5071 %}
 5072 
 5073 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5074 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5075   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5076 
 5077   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5078   ins_encode %{
 5079     __ movzbq($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5085 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087   effect(KILL cr);
 5088 
 5089   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5090             "andl    $dst, right_n_bits($mask, 16)" %}
 5091   ins_encode %{
 5092     Register Rdst = $dst$$Register;
 5093     __ movzwq(Rdst, $mem$$Address);
 5094     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5095   %}
 5096   ins_pipe(ialu_reg_mem);
 5097 %}
 5098 
 5099 // Load Integer
 5100 instruct loadI(rRegI dst, memory mem)
 5101 %{
 5102   match(Set dst (LoadI mem));
 5103 
 5104   ins_cost(125);
 5105   format %{ "movl    $dst, $mem\t# int" %}
 5106 
 5107   ins_encode %{
 5108     __ movl($dst$$Register, $mem$$Address);
 5109   %}
 5110 
 5111   ins_pipe(ialu_reg_mem);
 5112 %}
 5113 
 5114 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5115 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5116   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5117 
 5118   ins_cost(125);
 5119   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5120   ins_encode %{
 5121     __ movsbl($dst$$Register, $mem$$Address);
 5122   %}
 5123   ins_pipe(ialu_reg_mem);
 5124 %}
 5125 
 5126 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5127 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5128   match(Set dst (AndI (LoadI mem) mask));
 5129 
 5130   ins_cost(125);
 5131   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5132   ins_encode %{
 5133     __ movzbl($dst$$Register, $mem$$Address);
 5134   %}
 5135   ins_pipe(ialu_reg_mem);
 5136 %}
 5137 
 5138 // Load Integer (32 bit signed) to Short (16 bit signed)
 5139 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5140   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5141 
 5142   ins_cost(125);
 5143   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5144   ins_encode %{
 5145     __ movswl($dst$$Register, $mem$$Address);
 5146   %}
 5147   ins_pipe(ialu_reg_mem);
 5148 %}
 5149 
 5150 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5151 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5152   match(Set dst (AndI (LoadI mem) mask));
 5153 
 5154   ins_cost(125);
 5155   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5156   ins_encode %{
 5157     __ movzwl($dst$$Register, $mem$$Address);
 5158   %}
 5159   ins_pipe(ialu_reg_mem);
 5160 %}
 5161 
 5162 // Load Integer into Long Register
 5163 instruct loadI2L(rRegL dst, memory mem)
 5164 %{
 5165   match(Set dst (ConvI2L (LoadI mem)));
 5166 
 5167   ins_cost(125);
 5168   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5169 
 5170   ins_encode %{
 5171     __ movslq($dst$$Register, $mem$$Address);
 5172   %}
 5173 
 5174   ins_pipe(ialu_reg_mem);
 5175 %}
 5176 
 5177 // Load Integer with mask 0xFF into Long Register
 5178 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5179   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5180 
 5181   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5182   ins_encode %{
 5183     __ movzbq($dst$$Register, $mem$$Address);
 5184   %}
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFFFF into Long Register
 5189 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5193   ins_encode %{
 5194     __ movzwq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with a 31-bit mask into Long Register
 5200 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202   effect(KILL cr);
 5203 
 5204   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5205             "andl    $dst, $mask" %}
 5206   ins_encode %{
 5207     Register Rdst = $dst$$Register;
 5208     __ movl(Rdst, $mem$$Address);
 5209     __ andl(Rdst, $mask$$constant);
 5210   %}
 5211   ins_pipe(ialu_reg_mem);
 5212 %}
 5213 
 5214 // Load Unsigned Integer into Long Register
 5215 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5216 %{
 5217   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5218 
 5219   ins_cost(125);
 5220   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5221 
 5222   ins_encode %{
 5223     __ movl($dst$$Register, $mem$$Address);
 5224   %}
 5225 
 5226   ins_pipe(ialu_reg_mem);
 5227 %}
 5228 
 5229 // Load Long
 5230 instruct loadL(rRegL dst, memory mem)
 5231 %{
 5232   match(Set dst (LoadL mem));
 5233 
 5234   ins_cost(125);
 5235   format %{ "movq    $dst, $mem\t# long" %}
 5236 
 5237   ins_encode %{
 5238     __ movq($dst$$Register, $mem$$Address);
 5239   %}
 5240 
 5241   ins_pipe(ialu_reg_mem); // XXX
 5242 %}
 5243 
 5244 // Load Range
 5245 instruct loadRange(rRegI dst, memory mem)
 5246 %{
 5247   match(Set dst (LoadRange mem));
 5248 
 5249   ins_cost(125); // XXX
 5250   format %{ "movl    $dst, $mem\t# range" %}
 5251   ins_encode %{
 5252     __ movl($dst$$Register, $mem$$Address);
 5253   %}
 5254   ins_pipe(ialu_reg_mem);
 5255 %}
 5256 
 5257 // Load Pointer
 5258 instruct loadP(rRegP dst, memory mem)
 5259 %{
 5260   match(Set dst (LoadP mem));
 5261   predicate(n->as_Load()->barrier_data() == 0);
 5262 
 5263   ins_cost(125); // XXX
 5264   format %{ "movq    $dst, $mem\t# ptr" %}
 5265   ins_encode %{
 5266     __ movq($dst$$Register, $mem$$Address);
 5267   %}
 5268   ins_pipe(ialu_reg_mem); // XXX
 5269 %}
 5270 
 5271 // Load Compressed Pointer
 5272 instruct loadN(rRegN dst, memory mem)
 5273 %{
 5274    match(Set dst (LoadN mem));
 5275 
 5276    ins_cost(125); // XXX
 5277    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5278    ins_encode %{
 5279      __ movl($dst$$Register, $mem$$Address);
 5280    %}
 5281    ins_pipe(ialu_reg_mem); // XXX
 5282 %}
 5283 
 5284 
 5285 // Load Klass Pointer
 5286 instruct loadKlass(rRegP dst, memory mem)
 5287 %{
 5288   match(Set dst (LoadKlass mem));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movq    $dst, $mem\t# class" %}
 5292   ins_encode %{
 5293     __ movq($dst$$Register, $mem$$Address);
 5294   %}
 5295   ins_pipe(ialu_reg_mem); // XXX
 5296 %}
 5297 
 5298 // Load narrow Klass Pointer
 5299 instruct loadNKlass(rRegN dst, memory mem)
 5300 %{
 5301   match(Set dst (LoadNKlass mem));
 5302 
 5303   ins_cost(125); // XXX
 5304   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5305   ins_encode %{
 5306     __ movl($dst$$Register, $mem$$Address);
 5307   %}
 5308   ins_pipe(ialu_reg_mem); // XXX
 5309 %}
 5310 
 5311 // Load Float
 5312 instruct loadF(regF dst, memory mem)
 5313 %{
 5314   match(Set dst (LoadF mem));
 5315 
 5316   ins_cost(145); // XXX
 5317   format %{ "movss   $dst, $mem\t# float" %}
 5318   ins_encode %{
 5319     __ movflt($dst$$XMMRegister, $mem$$Address);
 5320   %}
 5321   ins_pipe(pipe_slow); // XXX
 5322 %}
 5323 
 5324 // Load Double
 5325 instruct loadD_partial(regD dst, memory mem)
 5326 %{
 5327   predicate(!UseXmmLoadAndClearUpper);
 5328   match(Set dst (LoadD mem));
 5329 
 5330   ins_cost(145); // XXX
 5331   format %{ "movlpd  $dst, $mem\t# double" %}
 5332   ins_encode %{
 5333     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5334   %}
 5335   ins_pipe(pipe_slow); // XXX
 5336 %}
 5337 
 5338 instruct loadD(regD dst, memory mem)
 5339 %{
 5340   predicate(UseXmmLoadAndClearUpper);
 5341   match(Set dst (LoadD mem));
 5342 
 5343   ins_cost(145); // XXX
 5344   format %{ "movsd   $dst, $mem\t# double" %}
 5345   ins_encode %{
 5346     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5347   %}
 5348   ins_pipe(pipe_slow); // XXX
 5349 %}
 5350 
 5351 
 5352 // Following pseudo code describes the algorithm for max[FD]:
 5353 // Min algorithm is on similar lines
 5354 //  btmp = (b < +0.0) ? a : b
 5355 //  atmp = (b < +0.0) ? b : a
 5356 //  Tmp  = Max_Float(atmp , btmp)
 5357 //  Res  = (atmp == NaN) ? atmp : Tmp
 5358 
 5359 // max = java.lang.Math.max(float a, float b)
 5360 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5361   predicate(UseAVX > 0 && !n->is_reduction());
 5362   match(Set dst (MaxF a b));
 5363   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5364   format %{
 5365      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5366      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5367      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5368      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5369      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5370   %}
 5371   ins_encode %{
 5372     int vector_len = Assembler::AVX_128bit;
 5373     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5374     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5375     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5376     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5377     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5378  %}
 5379   ins_pipe( pipe_slow );
 5380 %}
 5381 
 5382 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5383   predicate(UseAVX > 0 && n->is_reduction());
 5384   match(Set dst (MaxF a b));
 5385   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5386 
 5387   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5388   ins_encode %{
 5389     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5390                     false /*min*/, true /*single*/);
 5391   %}
 5392   ins_pipe( pipe_slow );
 5393 %}
 5394 
 5395 // max = java.lang.Math.max(double a, double b)
 5396 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5397   predicate(UseAVX > 0 && !n->is_reduction());
 5398   match(Set dst (MaxD a b));
 5399   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5400   format %{
 5401      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5402      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5403      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5404      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5405      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5406   %}
 5407   ins_encode %{
 5408     int vector_len = Assembler::AVX_128bit;
 5409     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5410     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5411     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5412     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5413     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5414   %}
 5415   ins_pipe( pipe_slow );
 5416 %}
 5417 
 5418 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5419   predicate(UseAVX > 0 && n->is_reduction());
 5420   match(Set dst (MaxD a b));
 5421   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5422 
 5423   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5424   ins_encode %{
 5425     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5426                     false /*min*/, false /*single*/);
 5427   %}
 5428   ins_pipe( pipe_slow );
 5429 %}
 5430 
 5431 // min = java.lang.Math.min(float a, float b)
 5432 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5433   predicate(UseAVX > 0 && !n->is_reduction());
 5434   match(Set dst (MinF a b));
 5435   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5436   format %{
 5437      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5438      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5439      "vminss           $tmp,$atmp,$btmp           \n\t"
 5440      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5441      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5442   %}
 5443   ins_encode %{
 5444     int vector_len = Assembler::AVX_128bit;
 5445     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5446     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5447     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5448     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5449     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5450   %}
 5451   ins_pipe( pipe_slow );
 5452 %}
 5453 
 5454 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5455   predicate(UseAVX > 0 && n->is_reduction());
 5456   match(Set dst (MinF a b));
 5457   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5458 
 5459   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5460   ins_encode %{
 5461     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5462                     true /*min*/, true /*single*/);
 5463   %}
 5464   ins_pipe( pipe_slow );
 5465 %}
 5466 
 5467 // min = java.lang.Math.min(double a, double b)
 5468 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5469   predicate(UseAVX > 0 && !n->is_reduction());
 5470   match(Set dst (MinD a b));
 5471   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5472   format %{
 5473      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5474      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5475      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5476      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5477      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5478   %}
 5479   ins_encode %{
 5480     int vector_len = Assembler::AVX_128bit;
 5481     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5482     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5483     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5484     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5485     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5486   %}
 5487   ins_pipe( pipe_slow );
 5488 %}
 5489 
 5490 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5491   predicate(UseAVX > 0 && n->is_reduction());
 5492   match(Set dst (MinD a b));
 5493   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5494 
 5495   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5496   ins_encode %{
 5497     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5498                     true /*min*/, false /*single*/);
 5499   %}
 5500   ins_pipe( pipe_slow );
 5501 %}
 5502 
 5503 // Load Effective Address
 5504 instruct leaP8(rRegP dst, indOffset8 mem)
 5505 %{
 5506   match(Set dst mem);
 5507 
 5508   ins_cost(110); // XXX
 5509   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5510   ins_encode %{
 5511     __ leaq($dst$$Register, $mem$$Address);
 5512   %}
 5513   ins_pipe(ialu_reg_reg_fat);
 5514 %}
 5515 
 5516 instruct leaP32(rRegP dst, indOffset32 mem)
 5517 %{
 5518   match(Set dst mem);
 5519 
 5520   ins_cost(110);
 5521   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5522   ins_encode %{
 5523     __ leaq($dst$$Register, $mem$$Address);
 5524   %}
 5525   ins_pipe(ialu_reg_reg_fat);
 5526 %}
 5527 
 5528 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5529 %{
 5530   match(Set dst mem);
 5531 
 5532   ins_cost(110);
 5533   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5534   ins_encode %{
 5535     __ leaq($dst$$Register, $mem$$Address);
 5536   %}
 5537   ins_pipe(ialu_reg_reg_fat);
 5538 %}
 5539 
 5540 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5541 %{
 5542   match(Set dst mem);
 5543 
 5544   ins_cost(110);
 5545   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5546   ins_encode %{
 5547     __ leaq($dst$$Register, $mem$$Address);
 5548   %}
 5549   ins_pipe(ialu_reg_reg_fat);
 5550 %}
 5551 
 5552 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5553 %{
 5554   match(Set dst mem);
 5555 
 5556   ins_cost(110);
 5557   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5558   ins_encode %{
 5559     __ leaq($dst$$Register, $mem$$Address);
 5560   %}
 5561   ins_pipe(ialu_reg_reg_fat);
 5562 %}
 5563 
 5564 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5565 %{
 5566   match(Set dst mem);
 5567 
 5568   ins_cost(110);
 5569   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5570   ins_encode %{
 5571     __ leaq($dst$$Register, $mem$$Address);
 5572   %}
 5573   ins_pipe(ialu_reg_reg_fat);
 5574 %}
 5575 
 5576 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5577 %{
 5578   match(Set dst mem);
 5579 
 5580   ins_cost(110);
 5581   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5582   ins_encode %{
 5583     __ leaq($dst$$Register, $mem$$Address);
 5584   %}
 5585   ins_pipe(ialu_reg_reg_fat);
 5586 %}
 5587 
 5588 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5589 %{
 5590   match(Set dst mem);
 5591 
 5592   ins_cost(110);
 5593   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5594   ins_encode %{
 5595     __ leaq($dst$$Register, $mem$$Address);
 5596   %}
 5597   ins_pipe(ialu_reg_reg_fat);
 5598 %}
 5599 
 5600 // Load Effective Address which uses Narrow (32-bits) oop
 5601 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5602 %{
 5603   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5604   match(Set dst mem);
 5605 
 5606   ins_cost(110);
 5607   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5608   ins_encode %{
 5609     __ leaq($dst$$Register, $mem$$Address);
 5610   %}
 5611   ins_pipe(ialu_reg_reg_fat);
 5612 %}
 5613 
 5614 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5615 %{
 5616   predicate(CompressedOops::shift() == 0);
 5617   match(Set dst mem);
 5618 
 5619   ins_cost(110); // XXX
 5620   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5621   ins_encode %{
 5622     __ leaq($dst$$Register, $mem$$Address);
 5623   %}
 5624   ins_pipe(ialu_reg_reg_fat);
 5625 %}
 5626 
 5627 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5628 %{
 5629   predicate(CompressedOops::shift() == 0);
 5630   match(Set dst mem);
 5631 
 5632   ins_cost(110);
 5633   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5634   ins_encode %{
 5635     __ leaq($dst$$Register, $mem$$Address);
 5636   %}
 5637   ins_pipe(ialu_reg_reg_fat);
 5638 %}
 5639 
 5640 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5641 %{
 5642   predicate(CompressedOops::shift() == 0);
 5643   match(Set dst mem);
 5644 
 5645   ins_cost(110);
 5646   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5647   ins_encode %{
 5648     __ leaq($dst$$Register, $mem$$Address);
 5649   %}
 5650   ins_pipe(ialu_reg_reg_fat);
 5651 %}
 5652 
 5653 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5654 %{
 5655   predicate(CompressedOops::shift() == 0);
 5656   match(Set dst mem);
 5657 
 5658   ins_cost(110);
 5659   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5660   ins_encode %{
 5661     __ leaq($dst$$Register, $mem$$Address);
 5662   %}
 5663   ins_pipe(ialu_reg_reg_fat);
 5664 %}
 5665 
 5666 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5667 %{
 5668   predicate(CompressedOops::shift() == 0);
 5669   match(Set dst mem);
 5670 
 5671   ins_cost(110);
 5672   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5673   ins_encode %{
 5674     __ leaq($dst$$Register, $mem$$Address);
 5675   %}
 5676   ins_pipe(ialu_reg_reg_fat);
 5677 %}
 5678 
 5679 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5680 %{
 5681   predicate(CompressedOops::shift() == 0);
 5682   match(Set dst mem);
 5683 
 5684   ins_cost(110);
 5685   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5686   ins_encode %{
 5687     __ leaq($dst$$Register, $mem$$Address);
 5688   %}
 5689   ins_pipe(ialu_reg_reg_fat);
 5690 %}
 5691 
 5692 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5693 %{
 5694   predicate(CompressedOops::shift() == 0);
 5695   match(Set dst mem);
 5696 
 5697   ins_cost(110);
 5698   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5699   ins_encode %{
 5700     __ leaq($dst$$Register, $mem$$Address);
 5701   %}
 5702   ins_pipe(ialu_reg_reg_fat);
 5703 %}
 5704 
 5705 instruct loadConI(rRegI dst, immI src)
 5706 %{
 5707   match(Set dst src);
 5708 
 5709   format %{ "movl    $dst, $src\t# int" %}
 5710   ins_encode %{
 5711     __ movl($dst$$Register, $src$$constant);
 5712   %}
 5713   ins_pipe(ialu_reg_fat); // XXX
 5714 %}
 5715 
 5716 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5717 %{
 5718   match(Set dst src);
 5719   effect(KILL cr);
 5720 
 5721   ins_cost(50);
 5722   format %{ "xorl    $dst, $dst\t# int" %}
 5723   ins_encode %{
 5724     __ xorl($dst$$Register, $dst$$Register);
 5725   %}
 5726   ins_pipe(ialu_reg);
 5727 %}
 5728 
 5729 instruct loadConL(rRegL dst, immL src)
 5730 %{
 5731   match(Set dst src);
 5732 
 5733   ins_cost(150);
 5734   format %{ "movq    $dst, $src\t# long" %}
 5735   ins_encode %{
 5736     __ mov64($dst$$Register, $src$$constant);
 5737   %}
 5738   ins_pipe(ialu_reg);
 5739 %}
 5740 
 5741 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5742 %{
 5743   match(Set dst src);
 5744   effect(KILL cr);
 5745 
 5746   ins_cost(50);
 5747   format %{ "xorl    $dst, $dst\t# long" %}
 5748   ins_encode %{
 5749     __ xorl($dst$$Register, $dst$$Register);
 5750   %}
 5751   ins_pipe(ialu_reg); // XXX
 5752 %}
 5753 
 5754 instruct loadConUL32(rRegL dst, immUL32 src)
 5755 %{
 5756   match(Set dst src);
 5757 
 5758   ins_cost(60);
 5759   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5760   ins_encode %{
 5761     __ movl($dst$$Register, $src$$constant);
 5762   %}
 5763   ins_pipe(ialu_reg);
 5764 %}
 5765 
 5766 instruct loadConL32(rRegL dst, immL32 src)
 5767 %{
 5768   match(Set dst src);
 5769 
 5770   ins_cost(70);
 5771   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5772   ins_encode %{
 5773     __ movq($dst$$Register, $src$$constant);
 5774   %}
 5775   ins_pipe(ialu_reg);
 5776 %}
 5777 
 5778 instruct loadConP(rRegP dst, immP con) %{
 5779   match(Set dst con);
 5780 
 5781   format %{ "movq    $dst, $con\t# ptr" %}
 5782   ins_encode %{
 5783     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5784   %}
 5785   ins_pipe(ialu_reg_fat); // XXX
 5786 %}
 5787 
 5788 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5789 %{
 5790   match(Set dst src);
 5791   effect(KILL cr);
 5792 
 5793   ins_cost(50);
 5794   format %{ "xorl    $dst, $dst\t# ptr" %}
 5795   ins_encode %{
 5796     __ xorl($dst$$Register, $dst$$Register);
 5797   %}
 5798   ins_pipe(ialu_reg);
 5799 %}
 5800 
 5801 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5802 %{
 5803   match(Set dst src);
 5804   effect(KILL cr);
 5805 
 5806   ins_cost(60);
 5807   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5808   ins_encode %{
 5809     __ movl($dst$$Register, $src$$constant);
 5810   %}
 5811   ins_pipe(ialu_reg);
 5812 %}
 5813 
 5814 instruct loadConF(regF dst, immF con) %{
 5815   match(Set dst con);
 5816   ins_cost(125);
 5817   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5818   ins_encode %{
 5819     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5820   %}
 5821   ins_pipe(pipe_slow);
 5822 %}
 5823 
 5824 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5825   match(Set dst src);
 5826   effect(KILL cr);
 5827   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5828   ins_encode %{
 5829     __ xorq($dst$$Register, $dst$$Register);
 5830   %}
 5831   ins_pipe(ialu_reg);
 5832 %}
 5833 
 5834 instruct loadConN(rRegN dst, immN src) %{
 5835   match(Set dst src);
 5836 
 5837   ins_cost(125);
 5838   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5839   ins_encode %{
 5840     address con = (address)$src$$constant;
 5841     if (con == NULL) {
 5842       ShouldNotReachHere();
 5843     } else {
 5844       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5845     }
 5846   %}
 5847   ins_pipe(ialu_reg_fat); // XXX
 5848 %}
 5849 
 5850 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5851   match(Set dst src);
 5852 
 5853   ins_cost(125);
 5854   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5855   ins_encode %{
 5856     address con = (address)$src$$constant;
 5857     if (con == NULL) {
 5858       ShouldNotReachHere();
 5859     } else {
 5860       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5861     }
 5862   %}
 5863   ins_pipe(ialu_reg_fat); // XXX
 5864 %}
 5865 
 5866 instruct loadConF0(regF dst, immF0 src)
 5867 %{
 5868   match(Set dst src);
 5869   ins_cost(100);
 5870 
 5871   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5872   ins_encode %{
 5873     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5874   %}
 5875   ins_pipe(pipe_slow);
 5876 %}
 5877 
 5878 // Use the same format since predicate() can not be used here.
 5879 instruct loadConD(regD dst, immD con) %{
 5880   match(Set dst con);
 5881   ins_cost(125);
 5882   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5883   ins_encode %{
 5884     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5885   %}
 5886   ins_pipe(pipe_slow);
 5887 %}
 5888 
 5889 instruct loadConD0(regD dst, immD0 src)
 5890 %{
 5891   match(Set dst src);
 5892   ins_cost(100);
 5893 
 5894   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5895   ins_encode %{
 5896     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5897   %}
 5898   ins_pipe(pipe_slow);
 5899 %}
 5900 
 5901 instruct loadSSI(rRegI dst, stackSlotI src)
 5902 %{
 5903   match(Set dst src);
 5904 
 5905   ins_cost(125);
 5906   format %{ "movl    $dst, $src\t# int stk" %}
 5907   opcode(0x8B);
 5908   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5909   ins_pipe(ialu_reg_mem);
 5910 %}
 5911 
 5912 instruct loadSSL(rRegL dst, stackSlotL src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movq    $dst, $src\t# long stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSP(rRegP dst, stackSlotP src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movq    $dst, $src\t# ptr stk" %}
 5929   opcode(0x8B);
 5930   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5931   ins_pipe(ialu_reg_mem);
 5932 %}
 5933 
 5934 instruct loadSSF(regF dst, stackSlotF src)
 5935 %{
 5936   match(Set dst src);
 5937 
 5938   ins_cost(125);
 5939   format %{ "movss   $dst, $src\t# float stk" %}
 5940   ins_encode %{
 5941     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5942   %}
 5943   ins_pipe(pipe_slow); // XXX
 5944 %}
 5945 
 5946 // Use the same format since predicate() can not be used here.
 5947 instruct loadSSD(regD dst, stackSlotD src)
 5948 %{
 5949   match(Set dst src);
 5950 
 5951   ins_cost(125);
 5952   format %{ "movsd   $dst, $src\t# double stk" %}
 5953   ins_encode  %{
 5954     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5955   %}
 5956   ins_pipe(pipe_slow); // XXX
 5957 %}
 5958 
 5959 // Prefetch instructions for allocation.
 5960 // Must be safe to execute with invalid address (cannot fault).
 5961 
 5962 instruct prefetchAlloc( memory mem ) %{
 5963   predicate(AllocatePrefetchInstr==3);
 5964   match(PrefetchAllocation mem);
 5965   ins_cost(125);
 5966 
 5967   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5968   ins_encode %{
 5969     __ prefetchw($mem$$Address);
 5970   %}
 5971   ins_pipe(ialu_mem);
 5972 %}
 5973 
 5974 instruct prefetchAllocNTA( memory mem ) %{
 5975   predicate(AllocatePrefetchInstr==0);
 5976   match(PrefetchAllocation mem);
 5977   ins_cost(125);
 5978 
 5979   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5980   ins_encode %{
 5981     __ prefetchnta($mem$$Address);
 5982   %}
 5983   ins_pipe(ialu_mem);
 5984 %}
 5985 
 5986 instruct prefetchAllocT0( memory mem ) %{
 5987   predicate(AllocatePrefetchInstr==1);
 5988   match(PrefetchAllocation mem);
 5989   ins_cost(125);
 5990 
 5991   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5992   ins_encode %{
 5993     __ prefetcht0($mem$$Address);
 5994   %}
 5995   ins_pipe(ialu_mem);
 5996 %}
 5997 
 5998 instruct prefetchAllocT2( memory mem ) %{
 5999   predicate(AllocatePrefetchInstr==2);
 6000   match(PrefetchAllocation mem);
 6001   ins_cost(125);
 6002 
 6003   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6004   ins_encode %{
 6005     __ prefetcht2($mem$$Address);
 6006   %}
 6007   ins_pipe(ialu_mem);
 6008 %}
 6009 
 6010 //----------Store Instructions-------------------------------------------------
 6011 
 6012 // Store Byte
 6013 instruct storeB(memory mem, rRegI src)
 6014 %{
 6015   match(Set mem (StoreB mem src));
 6016 
 6017   ins_cost(125); // XXX
 6018   format %{ "movb    $mem, $src\t# byte" %}
 6019   ins_encode %{
 6020     __ movb($mem$$Address, $src$$Register);
 6021   %}
 6022   ins_pipe(ialu_mem_reg);
 6023 %}
 6024 
 6025 // Store Char/Short
 6026 instruct storeC(memory mem, rRegI src)
 6027 %{
 6028   match(Set mem (StoreC mem src));
 6029 
 6030   ins_cost(125); // XXX
 6031   format %{ "movw    $mem, $src\t# char/short" %}
 6032   ins_encode %{
 6033     __ movw($mem$$Address, $src$$Register);
 6034   %}
 6035   ins_pipe(ialu_mem_reg);
 6036 %}
 6037 
 6038 // Store Integer
 6039 instruct storeI(memory mem, rRegI src)
 6040 %{
 6041   match(Set mem (StoreI mem src));
 6042 
 6043   ins_cost(125); // XXX
 6044   format %{ "movl    $mem, $src\t# int" %}
 6045   ins_encode %{
 6046     __ movl($mem$$Address, $src$$Register);
 6047   %}
 6048   ins_pipe(ialu_mem_reg);
 6049 %}
 6050 
 6051 // Store Long
 6052 instruct storeL(memory mem, rRegL src)
 6053 %{
 6054   match(Set mem (StoreL mem src));
 6055 
 6056   ins_cost(125); // XXX
 6057   format %{ "movq    $mem, $src\t# long" %}
 6058   ins_encode %{
 6059     __ movq($mem$$Address, $src$$Register);
 6060   %}
 6061   ins_pipe(ialu_mem_reg); // XXX
 6062 %}
 6063 
 6064 // Store Pointer
 6065 instruct storeP(memory mem, any_RegP src)
 6066 %{
 6067   match(Set mem (StoreP mem src));
 6068 
 6069   ins_cost(125); // XXX
 6070   format %{ "movq    $mem, $src\t# ptr" %}
 6071   ins_encode %{
 6072     __ movq($mem$$Address, $src$$Register);
 6073   %}
 6074   ins_pipe(ialu_mem_reg);
 6075 %}
 6076 
 6077 instruct storeImmP0(memory mem, immP0 zero)
 6078 %{
 6079   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6080   match(Set mem (StoreP mem zero));
 6081 
 6082   ins_cost(125); // XXX
 6083   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6084   ins_encode %{
 6085     __ movq($mem$$Address, r12);
 6086   %}
 6087   ins_pipe(ialu_mem_reg);
 6088 %}
 6089 
 6090 // Store NULL Pointer, mark word, or other simple pointer constant.
 6091 instruct storeImmP(memory mem, immP31 src)
 6092 %{
 6093   match(Set mem (StoreP mem src));
 6094 
 6095   ins_cost(150); // XXX
 6096   format %{ "movq    $mem, $src\t# ptr" %}
 6097   ins_encode %{
 6098     __ movq($mem$$Address, $src$$constant);
 6099   %}
 6100   ins_pipe(ialu_mem_imm);
 6101 %}
 6102 
 6103 // Store Compressed Pointer
 6104 instruct storeN(memory mem, rRegN src)
 6105 %{
 6106   match(Set mem (StoreN mem src));
 6107 
 6108   ins_cost(125); // XXX
 6109   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6110   ins_encode %{
 6111     __ movl($mem$$Address, $src$$Register);
 6112   %}
 6113   ins_pipe(ialu_mem_reg);
 6114 %}
 6115 
 6116 instruct storeNKlass(memory mem, rRegN src)
 6117 %{
 6118   match(Set mem (StoreNKlass mem src));
 6119 
 6120   ins_cost(125); // XXX
 6121   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6122   ins_encode %{
 6123     __ movl($mem$$Address, $src$$Register);
 6124   %}
 6125   ins_pipe(ialu_mem_reg);
 6126 %}
 6127 
 6128 instruct storeImmN0(memory mem, immN0 zero)
 6129 %{
 6130   predicate(CompressedOops::base() == NULL);
 6131   match(Set mem (StoreN mem zero));
 6132 
 6133   ins_cost(125); // XXX
 6134   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6135   ins_encode %{
 6136     __ movl($mem$$Address, r12);
 6137   %}
 6138   ins_pipe(ialu_mem_reg);
 6139 %}
 6140 
 6141 instruct storeImmN(memory mem, immN src)
 6142 %{
 6143   match(Set mem (StoreN mem src));
 6144 
 6145   ins_cost(150); // XXX
 6146   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6147   ins_encode %{
 6148     address con = (address)$src$$constant;
 6149     if (con == NULL) {
 6150       __ movl($mem$$Address, (int32_t)0);
 6151     } else {
 6152       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6153     }
 6154   %}
 6155   ins_pipe(ialu_mem_imm);
 6156 %}
 6157 
 6158 instruct storeImmNKlass(memory mem, immNKlass src)
 6159 %{
 6160   match(Set mem (StoreNKlass mem src));
 6161 
 6162   ins_cost(150); // XXX
 6163   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6164   ins_encode %{
 6165     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6166   %}
 6167   ins_pipe(ialu_mem_imm);
 6168 %}
 6169 
 6170 // Store Integer Immediate
 6171 instruct storeImmI0(memory mem, immI_0 zero)
 6172 %{
 6173   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6174   match(Set mem (StoreI mem zero));
 6175 
 6176   ins_cost(125); // XXX
 6177   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6178   ins_encode %{
 6179     __ movl($mem$$Address, r12);
 6180   %}
 6181   ins_pipe(ialu_mem_reg);
 6182 %}
 6183 
 6184 instruct storeImmI(memory mem, immI src)
 6185 %{
 6186   match(Set mem (StoreI mem src));
 6187 
 6188   ins_cost(150);
 6189   format %{ "movl    $mem, $src\t# int" %}
 6190   ins_encode %{
 6191     __ movl($mem$$Address, $src$$constant);
 6192   %}
 6193   ins_pipe(ialu_mem_imm);
 6194 %}
 6195 
 6196 // Store Long Immediate
 6197 instruct storeImmL0(memory mem, immL0 zero)
 6198 %{
 6199   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6200   match(Set mem (StoreL mem zero));
 6201 
 6202   ins_cost(125); // XXX
 6203   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6204   ins_encode %{
 6205     __ movq($mem$$Address, r12);
 6206   %}
 6207   ins_pipe(ialu_mem_reg);
 6208 %}
 6209 
 6210 instruct storeImmL(memory mem, immL32 src)
 6211 %{
 6212   match(Set mem (StoreL mem src));
 6213 
 6214   ins_cost(150);
 6215   format %{ "movq    $mem, $src\t# long" %}
 6216   ins_encode %{
 6217     __ movq($mem$$Address, $src$$constant);
 6218   %}
 6219   ins_pipe(ialu_mem_imm);
 6220 %}
 6221 
 6222 // Store Short/Char Immediate
 6223 instruct storeImmC0(memory mem, immI_0 zero)
 6224 %{
 6225   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6226   match(Set mem (StoreC mem zero));
 6227 
 6228   ins_cost(125); // XXX
 6229   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6230   ins_encode %{
 6231     __ movw($mem$$Address, r12);
 6232   %}
 6233   ins_pipe(ialu_mem_reg);
 6234 %}
 6235 
 6236 instruct storeImmI16(memory mem, immI16 src)
 6237 %{
 6238   predicate(UseStoreImmI16);
 6239   match(Set mem (StoreC mem src));
 6240 
 6241   ins_cost(150);
 6242   format %{ "movw    $mem, $src\t# short/char" %}
 6243   ins_encode %{
 6244     __ movw($mem$$Address, $src$$constant);
 6245   %}
 6246   ins_pipe(ialu_mem_imm);
 6247 %}
 6248 
 6249 // Store Byte Immediate
 6250 instruct storeImmB0(memory mem, immI_0 zero)
 6251 %{
 6252   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6253   match(Set mem (StoreB mem zero));
 6254 
 6255   ins_cost(125); // XXX
 6256   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6257   ins_encode %{
 6258     __ movb($mem$$Address, r12);
 6259   %}
 6260   ins_pipe(ialu_mem_reg);
 6261 %}
 6262 
 6263 instruct storeImmB(memory mem, immI8 src)
 6264 %{
 6265   match(Set mem (StoreB mem src));
 6266 
 6267   ins_cost(150); // XXX
 6268   format %{ "movb    $mem, $src\t# byte" %}
 6269   ins_encode %{
 6270     __ movb($mem$$Address, $src$$constant);
 6271   %}
 6272   ins_pipe(ialu_mem_imm);
 6273 %}
 6274 
 6275 // Store CMS card-mark Immediate
 6276 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6277 %{
 6278   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6279   match(Set mem (StoreCM mem zero));
 6280 
 6281   ins_cost(125); // XXX
 6282   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6283   ins_encode %{
 6284     __ movb($mem$$Address, r12);
 6285   %}
 6286   ins_pipe(ialu_mem_reg);
 6287 %}
 6288 
 6289 instruct storeImmCM0(memory mem, immI_0 src)
 6290 %{
 6291   match(Set mem (StoreCM mem src));
 6292 
 6293   ins_cost(150); // XXX
 6294   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6295   ins_encode %{
 6296     __ movb($mem$$Address, $src$$constant);
 6297   %}
 6298   ins_pipe(ialu_mem_imm);
 6299 %}
 6300 
 6301 // Store Float
 6302 instruct storeF(memory mem, regF src)
 6303 %{
 6304   match(Set mem (StoreF mem src));
 6305 
 6306   ins_cost(95); // XXX
 6307   format %{ "movss   $mem, $src\t# float" %}
 6308   ins_encode %{
 6309     __ movflt($mem$$Address, $src$$XMMRegister);
 6310   %}
 6311   ins_pipe(pipe_slow); // XXX
 6312 %}
 6313 
 6314 // Store immediate Float value (it is faster than store from XMM register)
 6315 instruct storeF0(memory mem, immF0 zero)
 6316 %{
 6317   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6318   match(Set mem (StoreF mem zero));
 6319 
 6320   ins_cost(25); // XXX
 6321   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6322   ins_encode %{
 6323     __ movl($mem$$Address, r12);
 6324   %}
 6325   ins_pipe(ialu_mem_reg);
 6326 %}
 6327 
 6328 instruct storeF_imm(memory mem, immF src)
 6329 %{
 6330   match(Set mem (StoreF mem src));
 6331 
 6332   ins_cost(50);
 6333   format %{ "movl    $mem, $src\t# float" %}
 6334   ins_encode %{
 6335     __ movl($mem$$Address, jint_cast($src$$constant));
 6336   %}
 6337   ins_pipe(ialu_mem_imm);
 6338 %}
 6339 
 6340 // Store Double
 6341 instruct storeD(memory mem, regD src)
 6342 %{
 6343   match(Set mem (StoreD mem src));
 6344 
 6345   ins_cost(95); // XXX
 6346   format %{ "movsd   $mem, $src\t# double" %}
 6347   ins_encode %{
 6348     __ movdbl($mem$$Address, $src$$XMMRegister);
 6349   %}
 6350   ins_pipe(pipe_slow); // XXX
 6351 %}
 6352 
 6353 // Store immediate double 0.0 (it is faster than store from XMM register)
 6354 instruct storeD0_imm(memory mem, immD0 src)
 6355 %{
 6356   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6357   match(Set mem (StoreD mem src));
 6358 
 6359   ins_cost(50);
 6360   format %{ "movq    $mem, $src\t# double 0." %}
 6361   ins_encode %{
 6362     __ movq($mem$$Address, $src$$constant);
 6363   %}
 6364   ins_pipe(ialu_mem_imm);
 6365 %}
 6366 
 6367 instruct storeD0(memory mem, immD0 zero)
 6368 %{
 6369   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6370   match(Set mem (StoreD mem zero));
 6371 
 6372   ins_cost(25); // XXX
 6373   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6374   ins_encode %{
 6375     __ movq($mem$$Address, r12);
 6376   %}
 6377   ins_pipe(ialu_mem_reg);
 6378 %}
 6379 
 6380 instruct storeSSI(stackSlotI dst, rRegI src)
 6381 %{
 6382   match(Set dst src);
 6383 
 6384   ins_cost(100);
 6385   format %{ "movl    $dst, $src\t# int stk" %}
 6386   opcode(0x89);
 6387   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6388   ins_pipe( ialu_mem_reg );
 6389 %}
 6390 
 6391 instruct storeSSL(stackSlotL dst, rRegL src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movq    $dst, $src\t# long stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe(ialu_mem_reg);
 6400 %}
 6401 
 6402 instruct storeSSP(stackSlotP dst, rRegP src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(100);
 6407   format %{ "movq    $dst, $src\t# ptr stk" %}
 6408   opcode(0x89);
 6409   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6410   ins_pipe(ialu_mem_reg);
 6411 %}
 6412 
 6413 instruct storeSSF(stackSlotF dst, regF src)
 6414 %{
 6415   match(Set dst src);
 6416 
 6417   ins_cost(95); // XXX
 6418   format %{ "movss   $dst, $src\t# float stk" %}
 6419   ins_encode %{
 6420     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6421   %}
 6422   ins_pipe(pipe_slow); // XXX
 6423 %}
 6424 
 6425 instruct storeSSD(stackSlotD dst, regD src)
 6426 %{
 6427   match(Set dst src);
 6428 
 6429   ins_cost(95); // XXX
 6430   format %{ "movsd   $dst, $src\t# double stk" %}
 6431   ins_encode %{
 6432     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6433   %}
 6434   ins_pipe(pipe_slow); // XXX
 6435 %}
 6436 
 6437 instruct cacheWB(indirect addr)
 6438 %{
 6439   predicate(VM_Version::supports_data_cache_line_flush());
 6440   match(CacheWB addr);
 6441 
 6442   ins_cost(100);
 6443   format %{"cache wb $addr" %}
 6444   ins_encode %{
 6445     assert($addr->index_position() < 0, "should be");
 6446     assert($addr$$disp == 0, "should be");
 6447     __ cache_wb(Address($addr$$base$$Register, 0));
 6448   %}
 6449   ins_pipe(pipe_slow); // XXX
 6450 %}
 6451 
 6452 instruct cacheWBPreSync()
 6453 %{
 6454   predicate(VM_Version::supports_data_cache_line_flush());
 6455   match(CacheWBPreSync);
 6456 
 6457   ins_cost(100);
 6458   format %{"cache wb presync" %}
 6459   ins_encode %{
 6460     __ cache_wbsync(true);
 6461   %}
 6462   ins_pipe(pipe_slow); // XXX
 6463 %}
 6464 
 6465 instruct cacheWBPostSync()
 6466 %{
 6467   predicate(VM_Version::supports_data_cache_line_flush());
 6468   match(CacheWBPostSync);
 6469 
 6470   ins_cost(100);
 6471   format %{"cache wb postsync" %}
 6472   ins_encode %{
 6473     __ cache_wbsync(false);
 6474   %}
 6475   ins_pipe(pipe_slow); // XXX
 6476 %}
 6477 
 6478 //----------BSWAP Instructions-------------------------------------------------
 6479 instruct bytes_reverse_int(rRegI dst) %{
 6480   match(Set dst (ReverseBytesI dst));
 6481 
 6482   format %{ "bswapl  $dst" %}
 6483   ins_encode %{
 6484     __ bswapl($dst$$Register);
 6485   %}
 6486   ins_pipe( ialu_reg );
 6487 %}
 6488 
 6489 instruct bytes_reverse_long(rRegL dst) %{
 6490   match(Set dst (ReverseBytesL dst));
 6491 
 6492   format %{ "bswapq  $dst" %}
 6493   ins_encode %{
 6494     __ bswapq($dst$$Register);
 6495   %}
 6496   ins_pipe( ialu_reg);
 6497 %}
 6498 
 6499 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6500   match(Set dst (ReverseBytesUS dst));
 6501   effect(KILL cr);
 6502 
 6503   format %{ "bswapl  $dst\n\t"
 6504             "shrl    $dst,16\n\t" %}
 6505   ins_encode %{
 6506     __ bswapl($dst$$Register);
 6507     __ shrl($dst$$Register, 16);
 6508   %}
 6509   ins_pipe( ialu_reg );
 6510 %}
 6511 
 6512 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6513   match(Set dst (ReverseBytesS dst));
 6514   effect(KILL cr);
 6515 
 6516   format %{ "bswapl  $dst\n\t"
 6517             "sar     $dst,16\n\t" %}
 6518   ins_encode %{
 6519     __ bswapl($dst$$Register);
 6520     __ sarl($dst$$Register, 16);
 6521   %}
 6522   ins_pipe( ialu_reg );
 6523 %}
 6524 
 6525 //---------- Zeros Count Instructions ------------------------------------------
 6526 
 6527 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6528   predicate(UseCountLeadingZerosInstruction);
 6529   match(Set dst (CountLeadingZerosI src));
 6530   effect(KILL cr);
 6531 
 6532   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6533   ins_encode %{
 6534     __ lzcntl($dst$$Register, $src$$Register);
 6535   %}
 6536   ins_pipe(ialu_reg);
 6537 %}
 6538 
 6539 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6540   predicate(!UseCountLeadingZerosInstruction);
 6541   match(Set dst (CountLeadingZerosI src));
 6542   effect(KILL cr);
 6543 
 6544   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6545             "jnz     skip\n\t"
 6546             "movl    $dst, -1\n"
 6547       "skip:\n\t"
 6548             "negl    $dst\n\t"
 6549             "addl    $dst, 31" %}
 6550   ins_encode %{
 6551     Register Rdst = $dst$$Register;
 6552     Register Rsrc = $src$$Register;
 6553     Label skip;
 6554     __ bsrl(Rdst, Rsrc);
 6555     __ jccb(Assembler::notZero, skip);
 6556     __ movl(Rdst, -1);
 6557     __ bind(skip);
 6558     __ negl(Rdst);
 6559     __ addl(Rdst, BitsPerInt - 1);
 6560   %}
 6561   ins_pipe(ialu_reg);
 6562 %}
 6563 
 6564 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6565   predicate(UseCountLeadingZerosInstruction);
 6566   match(Set dst (CountLeadingZerosL src));
 6567   effect(KILL cr);
 6568 
 6569   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6570   ins_encode %{
 6571     __ lzcntq($dst$$Register, $src$$Register);
 6572   %}
 6573   ins_pipe(ialu_reg);
 6574 %}
 6575 
 6576 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6577   predicate(!UseCountLeadingZerosInstruction);
 6578   match(Set dst (CountLeadingZerosL src));
 6579   effect(KILL cr);
 6580 
 6581   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6582             "jnz     skip\n\t"
 6583             "movl    $dst, -1\n"
 6584       "skip:\n\t"
 6585             "negl    $dst\n\t"
 6586             "addl    $dst, 63" %}
 6587   ins_encode %{
 6588     Register Rdst = $dst$$Register;
 6589     Register Rsrc = $src$$Register;
 6590     Label skip;
 6591     __ bsrq(Rdst, Rsrc);
 6592     __ jccb(Assembler::notZero, skip);
 6593     __ movl(Rdst, -1);
 6594     __ bind(skip);
 6595     __ negl(Rdst);
 6596     __ addl(Rdst, BitsPerLong - 1);
 6597   %}
 6598   ins_pipe(ialu_reg);
 6599 %}
 6600 
 6601 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6602   predicate(UseCountTrailingZerosInstruction);
 6603   match(Set dst (CountTrailingZerosI src));
 6604   effect(KILL cr);
 6605 
 6606   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6607   ins_encode %{
 6608     __ tzcntl($dst$$Register, $src$$Register);
 6609   %}
 6610   ins_pipe(ialu_reg);
 6611 %}
 6612 
 6613 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6614   predicate(!UseCountTrailingZerosInstruction);
 6615   match(Set dst (CountTrailingZerosI src));
 6616   effect(KILL cr);
 6617 
 6618   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6619             "jnz     done\n\t"
 6620             "movl    $dst, 32\n"
 6621       "done:" %}
 6622   ins_encode %{
 6623     Register Rdst = $dst$$Register;
 6624     Label done;
 6625     __ bsfl(Rdst, $src$$Register);
 6626     __ jccb(Assembler::notZero, done);
 6627     __ movl(Rdst, BitsPerInt);
 6628     __ bind(done);
 6629   %}
 6630   ins_pipe(ialu_reg);
 6631 %}
 6632 
 6633 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6634   predicate(UseCountTrailingZerosInstruction);
 6635   match(Set dst (CountTrailingZerosL src));
 6636   effect(KILL cr);
 6637 
 6638   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6639   ins_encode %{
 6640     __ tzcntq($dst$$Register, $src$$Register);
 6641   %}
 6642   ins_pipe(ialu_reg);
 6643 %}
 6644 
 6645 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6646   predicate(!UseCountTrailingZerosInstruction);
 6647   match(Set dst (CountTrailingZerosL src));
 6648   effect(KILL cr);
 6649 
 6650   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6651             "jnz     done\n\t"
 6652             "movl    $dst, 64\n"
 6653       "done:" %}
 6654   ins_encode %{
 6655     Register Rdst = $dst$$Register;
 6656     Label done;
 6657     __ bsfq(Rdst, $src$$Register);
 6658     __ jccb(Assembler::notZero, done);
 6659     __ movl(Rdst, BitsPerLong);
 6660     __ bind(done);
 6661   %}
 6662   ins_pipe(ialu_reg);
 6663 %}
 6664 
 6665 
 6666 //---------- Population Count Instructions -------------------------------------
 6667 
 6668 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6669   predicate(UsePopCountInstruction);
 6670   match(Set dst (PopCountI src));
 6671   effect(KILL cr);
 6672 
 6673   format %{ "popcnt  $dst, $src" %}
 6674   ins_encode %{
 6675     __ popcntl($dst$$Register, $src$$Register);
 6676   %}
 6677   ins_pipe(ialu_reg);
 6678 %}
 6679 
 6680 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6681   predicate(UsePopCountInstruction);
 6682   match(Set dst (PopCountI (LoadI mem)));
 6683   effect(KILL cr);
 6684 
 6685   format %{ "popcnt  $dst, $mem" %}
 6686   ins_encode %{
 6687     __ popcntl($dst$$Register, $mem$$Address);
 6688   %}
 6689   ins_pipe(ialu_reg);
 6690 %}
 6691 
 6692 // Note: Long.bitCount(long) returns an int.
 6693 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6694   predicate(UsePopCountInstruction);
 6695   match(Set dst (PopCountL src));
 6696   effect(KILL cr);
 6697 
 6698   format %{ "popcnt  $dst, $src" %}
 6699   ins_encode %{
 6700     __ popcntq($dst$$Register, $src$$Register);
 6701   %}
 6702   ins_pipe(ialu_reg);
 6703 %}
 6704 
 6705 // Note: Long.bitCount(long) returns an int.
 6706 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6707   predicate(UsePopCountInstruction);
 6708   match(Set dst (PopCountL (LoadL mem)));
 6709   effect(KILL cr);
 6710 
 6711   format %{ "popcnt  $dst, $mem" %}
 6712   ins_encode %{
 6713     __ popcntq($dst$$Register, $mem$$Address);
 6714   %}
 6715   ins_pipe(ialu_reg);
 6716 %}
 6717 
 6718 
 6719 //----------MemBar Instructions-----------------------------------------------
 6720 // Memory barrier flavors
 6721 
 6722 instruct membar_acquire()
 6723 %{
 6724   match(MemBarAcquire);
 6725   match(LoadFence);
 6726   ins_cost(0);
 6727 
 6728   size(0);
 6729   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6730   ins_encode();
 6731   ins_pipe(empty);
 6732 %}
 6733 
 6734 instruct membar_acquire_lock()
 6735 %{
 6736   match(MemBarAcquireLock);
 6737   ins_cost(0);
 6738 
 6739   size(0);
 6740   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6741   ins_encode();
 6742   ins_pipe(empty);
 6743 %}
 6744 
 6745 instruct membar_release()
 6746 %{
 6747   match(MemBarRelease);
 6748   match(StoreFence);
 6749   ins_cost(0);
 6750 
 6751   size(0);
 6752   format %{ "MEMBAR-release ! (empty encoding)" %}
 6753   ins_encode();
 6754   ins_pipe(empty);
 6755 %}
 6756 
 6757 instruct membar_release_lock()
 6758 %{
 6759   match(MemBarReleaseLock);
 6760   ins_cost(0);
 6761 
 6762   size(0);
 6763   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6764   ins_encode();
 6765   ins_pipe(empty);
 6766 %}
 6767 
 6768 instruct membar_volatile(rFlagsReg cr) %{
 6769   match(MemBarVolatile);
 6770   effect(KILL cr);
 6771   ins_cost(400);
 6772 
 6773   format %{
 6774     $$template
 6775     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6776   %}
 6777   ins_encode %{
 6778     __ membar(Assembler::StoreLoad);
 6779   %}
 6780   ins_pipe(pipe_slow);
 6781 %}
 6782 
 6783 instruct unnecessary_membar_volatile()
 6784 %{
 6785   match(MemBarVolatile);
 6786   predicate(Matcher::post_store_load_barrier(n));
 6787   ins_cost(0);
 6788 
 6789   size(0);
 6790   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6791   ins_encode();
 6792   ins_pipe(empty);
 6793 %}
 6794 
 6795 instruct membar_storestore() %{
 6796   match(MemBarStoreStore);
 6797   ins_cost(0);
 6798 
 6799   size(0);
 6800   format %{ "MEMBAR-storestore (empty encoding)" %}
 6801   ins_encode( );
 6802   ins_pipe(empty);
 6803 %}
 6804 
 6805 //----------Move Instructions--------------------------------------------------
 6806 
 6807 instruct castX2P(rRegP dst, rRegL src)
 6808 %{
 6809   match(Set dst (CastX2P src));
 6810 
 6811   format %{ "movq    $dst, $src\t# long->ptr" %}
 6812   ins_encode %{
 6813     if ($dst$$reg != $src$$reg) {
 6814       __ movptr($dst$$Register, $src$$Register);
 6815     }
 6816   %}
 6817   ins_pipe(ialu_reg_reg); // XXX
 6818 %}
 6819 
 6820 instruct castP2X(rRegL dst, rRegP src)
 6821 %{
 6822   match(Set dst (CastP2X src));
 6823 
 6824   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6825   ins_encode %{
 6826     if ($dst$$reg != $src$$reg) {
 6827       __ movptr($dst$$Register, $src$$Register);
 6828     }
 6829   %}
 6830   ins_pipe(ialu_reg_reg); // XXX
 6831 %}
 6832 
 6833 // Convert oop into int for vectors alignment masking
 6834 instruct convP2I(rRegI dst, rRegP src)
 6835 %{
 6836   match(Set dst (ConvL2I (CastP2X src)));
 6837 
 6838   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6839   ins_encode %{
 6840     __ movl($dst$$Register, $src$$Register);
 6841   %}
 6842   ins_pipe(ialu_reg_reg); // XXX
 6843 %}
 6844 
 6845 // Convert compressed oop into int for vectors alignment masking
 6846 // in case of 32bit oops (heap < 4Gb).
 6847 instruct convN2I(rRegI dst, rRegN src)
 6848 %{
 6849   predicate(CompressedOops::shift() == 0);
 6850   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6851 
 6852   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6853   ins_encode %{
 6854     __ movl($dst$$Register, $src$$Register);
 6855   %}
 6856   ins_pipe(ialu_reg_reg); // XXX
 6857 %}
 6858 
 6859 // Convert oop pointer into compressed form
 6860 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6861   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6862   match(Set dst (EncodeP src));
 6863   effect(KILL cr);
 6864   format %{ "encode_heap_oop $dst,$src" %}
 6865   ins_encode %{
 6866     Register s = $src$$Register;
 6867     Register d = $dst$$Register;
 6868     if (s != d) {
 6869       __ movq(d, s);
 6870     }
 6871     __ encode_heap_oop(d);
 6872   %}
 6873   ins_pipe(ialu_reg_long);
 6874 %}
 6875 
 6876 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6877   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6878   match(Set dst (EncodeP src));
 6879   effect(KILL cr);
 6880   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6881   ins_encode %{
 6882     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6883   %}
 6884   ins_pipe(ialu_reg_long);
 6885 %}
 6886 
 6887 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6888   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6889             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6890   match(Set dst (DecodeN src));
 6891   effect(KILL cr);
 6892   format %{ "decode_heap_oop $dst,$src" %}
 6893   ins_encode %{
 6894     Register s = $src$$Register;
 6895     Register d = $dst$$Register;
 6896     if (s != d) {
 6897       __ movq(d, s);
 6898     }
 6899     __ decode_heap_oop(d);
 6900   %}
 6901   ins_pipe(ialu_reg_long);
 6902 %}
 6903 
 6904 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6905   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6906             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6907   match(Set dst (DecodeN src));
 6908   effect(KILL cr);
 6909   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6910   ins_encode %{
 6911     Register s = $src$$Register;
 6912     Register d = $dst$$Register;
 6913     if (s != d) {
 6914       __ decode_heap_oop_not_null(d, s);
 6915     } else {
 6916       __ decode_heap_oop_not_null(d);
 6917     }
 6918   %}
 6919   ins_pipe(ialu_reg_long);
 6920 %}
 6921 
 6922 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6923   match(Set dst (EncodePKlass src));
 6924   effect(TEMP dst, KILL cr);
 6925   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6926   ins_encode %{
 6927     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6928   %}
 6929   ins_pipe(ialu_reg_long);
 6930 %}
 6931 
 6932 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6933   match(Set dst (DecodeNKlass src));
 6934   effect(TEMP dst, KILL cr);
 6935   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6936   ins_encode %{
 6937     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6938   %}
 6939   ins_pipe(ialu_reg_long);
 6940 %}
 6941 
 6942 //----------Conditional Move---------------------------------------------------
 6943 // Jump
 6944 // dummy instruction for generating temp registers
 6945 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6946   match(Jump (LShiftL switch_val shift));
 6947   ins_cost(350);
 6948   predicate(false);
 6949   effect(TEMP dest);
 6950 
 6951   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6952             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6953   ins_encode %{
 6954     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6955     // to do that and the compiler is using that register as one it can allocate.
 6956     // So we build it all by hand.
 6957     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6958     // ArrayAddress dispatch(table, index);
 6959     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6960     __ lea($dest$$Register, $constantaddress);
 6961     __ jmp(dispatch);
 6962   %}
 6963   ins_pipe(pipe_jmp);
 6964 %}
 6965 
 6966 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6967   match(Jump (AddL (LShiftL switch_val shift) offset));
 6968   ins_cost(350);
 6969   effect(TEMP dest);
 6970 
 6971   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6972             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6973   ins_encode %{
 6974     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6975     // to do that and the compiler is using that register as one it can allocate.
 6976     // So we build it all by hand.
 6977     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6978     // ArrayAddress dispatch(table, index);
 6979     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6980     __ lea($dest$$Register, $constantaddress);
 6981     __ jmp(dispatch);
 6982   %}
 6983   ins_pipe(pipe_jmp);
 6984 %}
 6985 
 6986 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6987   match(Jump switch_val);
 6988   ins_cost(350);
 6989   effect(TEMP dest);
 6990 
 6991   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6992             "jmp     [$dest + $switch_val]\n\t" %}
 6993   ins_encode %{
 6994     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6995     // to do that and the compiler is using that register as one it can allocate.
 6996     // So we build it all by hand.
 6997     // Address index(noreg, switch_reg, Address::times_1);
 6998     // ArrayAddress dispatch(table, index);
 6999     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7000     __ lea($dest$$Register, $constantaddress);
 7001     __ jmp(dispatch);
 7002   %}
 7003   ins_pipe(pipe_jmp);
 7004 %}
 7005 
 7006 // Conditional move
 7007 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7008 %{
 7009   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7010 
 7011   ins_cost(200); // XXX
 7012   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7013   ins_encode %{
 7014     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7015   %}
 7016   ins_pipe(pipe_cmov_reg);
 7017 %}
 7018 
 7019 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7020   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7021 
 7022   ins_cost(200); // XXX
 7023   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7024   ins_encode %{
 7025     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7026   %}
 7027   ins_pipe(pipe_cmov_reg);
 7028 %}
 7029 
 7030 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7031   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7032   ins_cost(200);
 7033   expand %{
 7034     cmovI_regU(cop, cr, dst, src);
 7035   %}
 7036 %}
 7037 
 7038 // Conditional move
 7039 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7040   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7041 
 7042   ins_cost(250); // XXX
 7043   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7044   ins_encode %{
 7045     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7046   %}
 7047   ins_pipe(pipe_cmov_mem);
 7048 %}
 7049 
 7050 // Conditional move
 7051 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7052 %{
 7053   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7054 
 7055   ins_cost(250); // XXX
 7056   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7057   ins_encode %{
 7058     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7059   %}
 7060   ins_pipe(pipe_cmov_mem);
 7061 %}
 7062 
 7063 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7064   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7065   ins_cost(250);
 7066   expand %{
 7067     cmovI_memU(cop, cr, dst, src);
 7068   %}
 7069 %}
 7070 
 7071 // Conditional move
 7072 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7073 %{
 7074   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7075 
 7076   ins_cost(200); // XXX
 7077   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7078   ins_encode %{
 7079     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7080   %}
 7081   ins_pipe(pipe_cmov_reg);
 7082 %}
 7083 
 7084 // Conditional move
 7085 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7086 %{
 7087   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7088 
 7089   ins_cost(200); // XXX
 7090   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7091   ins_encode %{
 7092     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7093   %}
 7094   ins_pipe(pipe_cmov_reg);
 7095 %}
 7096 
 7097 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7098   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7099   ins_cost(200);
 7100   expand %{
 7101     cmovN_regU(cop, cr, dst, src);
 7102   %}
 7103 %}
 7104 
 7105 // Conditional move
 7106 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7107 %{
 7108   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7109 
 7110   ins_cost(200); // XXX
 7111   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7112   ins_encode %{
 7113     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7114   %}
 7115   ins_pipe(pipe_cmov_reg);  // XXX
 7116 %}
 7117 
 7118 // Conditional move
 7119 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7120 %{
 7121   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7122 
 7123   ins_cost(200); // XXX
 7124   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7125   ins_encode %{
 7126     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7127   %}
 7128   ins_pipe(pipe_cmov_reg); // XXX
 7129 %}
 7130 
 7131 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7132   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7133   ins_cost(200);
 7134   expand %{
 7135     cmovP_regU(cop, cr, dst, src);
 7136   %}
 7137 %}
 7138 
 7139 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7140 // correctly meets the two pointer arguments; one is an incoming
 7141 // register but the other is a memory operand.  ALSO appears to
 7142 // be buggy with implicit null checks.
 7143 //
 7144 //// Conditional move
 7145 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7146 //%{
 7147 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7148 //  ins_cost(250);
 7149 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7150 //  opcode(0x0F,0x40);
 7151 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7152 //  ins_pipe( pipe_cmov_mem );
 7153 //%}
 7154 //
 7155 //// Conditional move
 7156 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7157 //%{
 7158 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7159 //  ins_cost(250);
 7160 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7161 //  opcode(0x0F,0x40);
 7162 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7163 //  ins_pipe( pipe_cmov_mem );
 7164 //%}
 7165 
 7166 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7167 %{
 7168   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7169 
 7170   ins_cost(200); // XXX
 7171   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7172   ins_encode %{
 7173     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7174   %}
 7175   ins_pipe(pipe_cmov_reg);  // XXX
 7176 %}
 7177 
 7178 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7179 %{
 7180   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7181 
 7182   ins_cost(200); // XXX
 7183   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7184   ins_encode %{
 7185     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7186   %}
 7187   ins_pipe(pipe_cmov_mem);  // XXX
 7188 %}
 7189 
 7190 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7191 %{
 7192   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7193 
 7194   ins_cost(200); // XXX
 7195   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7196   ins_encode %{
 7197     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7198   %}
 7199   ins_pipe(pipe_cmov_reg); // XXX
 7200 %}
 7201 
 7202 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7203   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7204   ins_cost(200);
 7205   expand %{
 7206     cmovL_regU(cop, cr, dst, src);
 7207   %}
 7208 %}
 7209 
 7210 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7211 %{
 7212   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7213 
 7214   ins_cost(200); // XXX
 7215   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7216   ins_encode %{
 7217     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7218   %}
 7219   ins_pipe(pipe_cmov_mem); // XXX
 7220 %}
 7221 
 7222 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7223   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7224   ins_cost(200);
 7225   expand %{
 7226     cmovL_memU(cop, cr, dst, src);
 7227   %}
 7228 %}
 7229 
 7230 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7231 %{
 7232   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7233 
 7234   ins_cost(200); // XXX
 7235   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7236             "movss     $dst, $src\n"
 7237     "skip:" %}
 7238   ins_encode %{
 7239     Label Lskip;
 7240     // Invert sense of branch from sense of CMOV
 7241     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7242     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7243     __ bind(Lskip);
 7244   %}
 7245   ins_pipe(pipe_slow);
 7246 %}
 7247 
 7248 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7249 // %{
 7250 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7251 
 7252 //   ins_cost(200); // XXX
 7253 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7254 //             "movss     $dst, $src\n"
 7255 //     "skip:" %}
 7256 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7257 //   ins_pipe(pipe_slow);
 7258 // %}
 7259 
 7260 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7261 %{
 7262   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7263 
 7264   ins_cost(200); // XXX
 7265   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7266             "movss     $dst, $src\n"
 7267     "skip:" %}
 7268   ins_encode %{
 7269     Label Lskip;
 7270     // Invert sense of branch from sense of CMOV
 7271     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7272     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7273     __ bind(Lskip);
 7274   %}
 7275   ins_pipe(pipe_slow);
 7276 %}
 7277 
 7278 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7279   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7280   ins_cost(200);
 7281   expand %{
 7282     cmovF_regU(cop, cr, dst, src);
 7283   %}
 7284 %}
 7285 
 7286 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7287 %{
 7288   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7289 
 7290   ins_cost(200); // XXX
 7291   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7292             "movsd     $dst, $src\n"
 7293     "skip:" %}
 7294   ins_encode %{
 7295     Label Lskip;
 7296     // Invert sense of branch from sense of CMOV
 7297     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7298     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7299     __ bind(Lskip);
 7300   %}
 7301   ins_pipe(pipe_slow);
 7302 %}
 7303 
 7304 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7305 %{
 7306   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7307 
 7308   ins_cost(200); // XXX
 7309   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7310             "movsd     $dst, $src\n"
 7311     "skip:" %}
 7312   ins_encode %{
 7313     Label Lskip;
 7314     // Invert sense of branch from sense of CMOV
 7315     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7316     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7317     __ bind(Lskip);
 7318   %}
 7319   ins_pipe(pipe_slow);
 7320 %}
 7321 
 7322 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7323   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7324   ins_cost(200);
 7325   expand %{
 7326     cmovD_regU(cop, cr, dst, src);
 7327   %}
 7328 %}
 7329 
 7330 //----------Arithmetic Instructions--------------------------------------------
 7331 //----------Addition Instructions----------------------------------------------
 7332 
 7333 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7334 %{
 7335   match(Set dst (AddI dst src));
 7336   effect(KILL cr);
 7337 
 7338   format %{ "addl    $dst, $src\t# int" %}
 7339   ins_encode %{
 7340     __ addl($dst$$Register, $src$$Register);
 7341   %}
 7342   ins_pipe(ialu_reg_reg);
 7343 %}
 7344 
 7345 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7346 %{
 7347   match(Set dst (AddI dst src));
 7348   effect(KILL cr);
 7349 
 7350   format %{ "addl    $dst, $src\t# int" %}
 7351   ins_encode %{
 7352     __ addl($dst$$Register, $src$$constant);
 7353   %}
 7354   ins_pipe( ialu_reg );
 7355 %}
 7356 
 7357 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7358 %{
 7359   match(Set dst (AddI dst (LoadI src)));
 7360   effect(KILL cr);
 7361 
 7362   ins_cost(125); // XXX
 7363   format %{ "addl    $dst, $src\t# int" %}
 7364   ins_encode %{
 7365     __ addl($dst$$Register, $src$$Address);
 7366   %}
 7367   ins_pipe(ialu_reg_mem);
 7368 %}
 7369 
 7370 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7371 %{
 7372   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7373   effect(KILL cr);
 7374 
 7375   ins_cost(150); // XXX
 7376   format %{ "addl    $dst, $src\t# int" %}
 7377   ins_encode %{
 7378     __ addl($dst$$Address, $src$$Register);
 7379   %}
 7380   ins_pipe(ialu_mem_reg);
 7381 %}
 7382 
 7383 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7384 %{
 7385   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7386   effect(KILL cr);
 7387 
 7388   ins_cost(125); // XXX
 7389   format %{ "addl    $dst, $src\t# int" %}
 7390   ins_encode %{
 7391     __ addl($dst$$Address, $src$$constant);
 7392   %}
 7393   ins_pipe(ialu_mem_imm);
 7394 %}
 7395 
 7396 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7397 %{
 7398   predicate(UseIncDec);
 7399   match(Set dst (AddI dst src));
 7400   effect(KILL cr);
 7401 
 7402   format %{ "incl    $dst\t# int" %}
 7403   ins_encode %{
 7404     __ incrementl($dst$$Register);
 7405   %}
 7406   ins_pipe(ialu_reg);
 7407 %}
 7408 
 7409 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7410 %{
 7411   predicate(UseIncDec);
 7412   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7413   effect(KILL cr);
 7414 
 7415   ins_cost(125); // XXX
 7416   format %{ "incl    $dst\t# int" %}
 7417   ins_encode %{
 7418     __ incrementl($dst$$Address);
 7419   %}
 7420   ins_pipe(ialu_mem_imm);
 7421 %}
 7422 
 7423 // XXX why does that use AddI
 7424 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7425 %{
 7426   predicate(UseIncDec);
 7427   match(Set dst (AddI dst src));
 7428   effect(KILL cr);
 7429 
 7430   format %{ "decl    $dst\t# int" %}
 7431   ins_encode %{
 7432     __ decrementl($dst$$Register);
 7433   %}
 7434   ins_pipe(ialu_reg);
 7435 %}
 7436 
 7437 // XXX why does that use AddI
 7438 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7439 %{
 7440   predicate(UseIncDec);
 7441   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7442   effect(KILL cr);
 7443 
 7444   ins_cost(125); // XXX
 7445   format %{ "decl    $dst\t# int" %}
 7446   ins_encode %{
 7447     __ decrementl($dst$$Address);
 7448   %}
 7449   ins_pipe(ialu_mem_imm);
 7450 %}
 7451 
 7452 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
 7453 %{
 7454   match(Set dst (AddI src0 src1));
 7455 
 7456   ins_cost(110);
 7457   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
 7458   ins_encode %{
 7459     __ leal($dst$$Register, Address($src0$$Register, $src1$$constant));
 7460   %}
 7461   ins_pipe(ialu_reg_reg);
 7462 %}
 7463 
 7464 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7465 %{
 7466   match(Set dst (AddL dst src));
 7467   effect(KILL cr);
 7468 
 7469   format %{ "addq    $dst, $src\t# long" %}
 7470   ins_encode %{
 7471     __ addq($dst$$Register, $src$$Register);
 7472   %}
 7473   ins_pipe(ialu_reg_reg);
 7474 %}
 7475 
 7476 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7477 %{
 7478   match(Set dst (AddL dst src));
 7479   effect(KILL cr);
 7480 
 7481   format %{ "addq    $dst, $src\t# long" %}
 7482   ins_encode %{
 7483     __ addq($dst$$Register, $src$$constant);
 7484   %}
 7485   ins_pipe( ialu_reg );
 7486 %}
 7487 
 7488 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7489 %{
 7490   match(Set dst (AddL dst (LoadL src)));
 7491   effect(KILL cr);
 7492 
 7493   ins_cost(125); // XXX
 7494   format %{ "addq    $dst, $src\t# long" %}
 7495   ins_encode %{
 7496     __ addq($dst$$Register, $src$$Address);
 7497   %}
 7498   ins_pipe(ialu_reg_mem);
 7499 %}
 7500 
 7501 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7502 %{
 7503   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7504   effect(KILL cr);
 7505 
 7506   ins_cost(150); // XXX
 7507   format %{ "addq    $dst, $src\t# long" %}
 7508   ins_encode %{
 7509     __ addq($dst$$Address, $src$$Register);
 7510   %}
 7511   ins_pipe(ialu_mem_reg);
 7512 %}
 7513 
 7514 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7515 %{
 7516   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7517   effect(KILL cr);
 7518 
 7519   ins_cost(125); // XXX
 7520   format %{ "addq    $dst, $src\t# long" %}
 7521   ins_encode %{
 7522     __ addq($dst$$Address, $src$$constant);
 7523   %}
 7524   ins_pipe(ialu_mem_imm);
 7525 %}
 7526 
 7527 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7528 %{
 7529   predicate(UseIncDec);
 7530   match(Set dst (AddL dst src));
 7531   effect(KILL cr);
 7532 
 7533   format %{ "incq    $dst\t# long" %}
 7534   ins_encode %{
 7535     __ incrementq($dst$$Register);
 7536   %}
 7537   ins_pipe(ialu_reg);
 7538 %}
 7539 
 7540 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7541 %{
 7542   predicate(UseIncDec);
 7543   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7544   effect(KILL cr);
 7545 
 7546   ins_cost(125); // XXX
 7547   format %{ "incq    $dst\t# long" %}
 7548   ins_encode %{
 7549     __ incrementq($dst$$Address);
 7550   %}
 7551   ins_pipe(ialu_mem_imm);
 7552 %}
 7553 
 7554 // XXX why does that use AddL
 7555 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7556 %{
 7557   predicate(UseIncDec);
 7558   match(Set dst (AddL dst src));
 7559   effect(KILL cr);
 7560 
 7561   format %{ "decq    $dst\t# long" %}
 7562   ins_encode %{
 7563     __ decrementq($dst$$Register);
 7564   %}
 7565   ins_pipe(ialu_reg);
 7566 %}
 7567 
 7568 // XXX why does that use AddL
 7569 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7570 %{
 7571   predicate(UseIncDec);
 7572   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7573   effect(KILL cr);
 7574 
 7575   ins_cost(125); // XXX
 7576   format %{ "decq    $dst\t# long" %}
 7577   ins_encode %{
 7578     __ decrementq($dst$$Address);
 7579   %}
 7580   ins_pipe(ialu_mem_imm);
 7581 %}
 7582 
 7583 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
 7584 %{
 7585   match(Set dst (AddL src0 src1));
 7586 
 7587   ins_cost(110);
 7588   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
 7589   ins_encode %{
 7590     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7591   %}
 7592   ins_pipe(ialu_reg_reg);
 7593 %}
 7594 
 7595 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7596 %{
 7597   match(Set dst (AddP dst src));
 7598   effect(KILL cr);
 7599 
 7600   format %{ "addq    $dst, $src\t# ptr" %}
 7601   ins_encode %{
 7602     __ addq($dst$$Register, $src$$Register);
 7603   %}
 7604   ins_pipe(ialu_reg_reg);
 7605 %}
 7606 
 7607 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7608 %{
 7609   match(Set dst (AddP dst src));
 7610   effect(KILL cr);
 7611 
 7612   format %{ "addq    $dst, $src\t# ptr" %}
 7613   ins_encode %{
 7614     __ addq($dst$$Register, $src$$constant);
 7615   %}
 7616   ins_pipe( ialu_reg );
 7617 %}
 7618 
 7619 // XXX addP mem ops ????
 7620 
 7621 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
 7622 %{
 7623   match(Set dst (AddP src0 src1));
 7624 
 7625   ins_cost(110);
 7626   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
 7627   ins_encode %{
 7628     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7629   %}
 7630   ins_pipe(ialu_reg_reg);
 7631 %}
 7632 
 7633 instruct checkCastPP(rRegP dst)
 7634 %{
 7635   match(Set dst (CheckCastPP dst));
 7636 
 7637   size(0);
 7638   format %{ "# checkcastPP of $dst" %}
 7639   ins_encode(/* empty encoding */);
 7640   ins_pipe(empty);
 7641 %}
 7642 
 7643 instruct castPP(rRegP dst)
 7644 %{
 7645   match(Set dst (CastPP dst));
 7646 
 7647   size(0);
 7648   format %{ "# castPP of $dst" %}
 7649   ins_encode(/* empty encoding */);
 7650   ins_pipe(empty);
 7651 %}
 7652 
 7653 instruct castII(rRegI dst)
 7654 %{
 7655   match(Set dst (CastII dst));
 7656 
 7657   size(0);
 7658   format %{ "# castII of $dst" %}
 7659   ins_encode(/* empty encoding */);
 7660   ins_cost(0);
 7661   ins_pipe(empty);
 7662 %}
 7663 
 7664 instruct castLL(rRegL dst)
 7665 %{
 7666   match(Set dst (CastLL dst));
 7667 
 7668   size(0);
 7669   format %{ "# castLL of $dst" %}
 7670   ins_encode(/* empty encoding */);
 7671   ins_cost(0);
 7672   ins_pipe(empty);
 7673 %}
 7674 
 7675 instruct castFF(regF dst)
 7676 %{
 7677   match(Set dst (CastFF dst));
 7678 
 7679   size(0);
 7680   format %{ "# castFF of $dst" %}
 7681   ins_encode(/* empty encoding */);
 7682   ins_cost(0);
 7683   ins_pipe(empty);
 7684 %}
 7685 
 7686 instruct castDD(regD dst)
 7687 %{
 7688   match(Set dst (CastDD dst));
 7689 
 7690   size(0);
 7691   format %{ "# castDD of $dst" %}
 7692   ins_encode(/* empty encoding */);
 7693   ins_cost(0);
 7694   ins_pipe(empty);
 7695 %}
 7696 
 7697 // LoadP-locked same as a regular LoadP when used with compare-swap
 7698 instruct loadPLocked(rRegP dst, memory mem)
 7699 %{
 7700   match(Set dst (LoadPLocked mem));
 7701 
 7702   ins_cost(125); // XXX
 7703   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7704   ins_encode %{
 7705     __ movq($dst$$Register, $mem$$Address);
 7706   %}
 7707   ins_pipe(ialu_reg_mem); // XXX
 7708 %}
 7709 
 7710 // Conditional-store of the updated heap-top.
 7711 // Used during allocation of the shared heap.
 7712 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7713 
 7714 instruct storePConditional(memory heap_top_ptr,
 7715                            rax_RegP oldval, rRegP newval,
 7716                            rFlagsReg cr)
 7717 %{
 7718   predicate(n->as_LoadStore()->barrier_data() == 0);
 7719   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7720 
 7721   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7722             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7723   ins_encode %{
 7724     __ lock();
 7725     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7726   %}
 7727   ins_pipe(pipe_cmpxchg);
 7728 %}
 7729 
 7730 // Conditional-store of an int value.
 7731 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7732 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7733 %{
 7734   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7735   effect(KILL oldval);
 7736 
 7737   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7738   opcode(0x0F, 0xB1);
 7739   ins_encode(lock_prefix,
 7740              REX_reg_mem(newval, mem),
 7741              OpcP, OpcS,
 7742              reg_mem(newval, mem));
 7743   ins_pipe(pipe_cmpxchg);
 7744 %}
 7745 
 7746 // Conditional-store of a long value.
 7747 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7748 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7749 %{
 7750   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7751   effect(KILL oldval);
 7752 
 7753   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7754   ins_encode %{
 7755     __ lock();
 7756     __ cmpxchgq($newval$$Register, $mem$$Address);
 7757   %}
 7758   ins_pipe(pipe_cmpxchg);
 7759 %}
 7760 
 7761 
 7762 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7763 instruct compareAndSwapP(rRegI res,
 7764                          memory mem_ptr,
 7765                          rax_RegP oldval, rRegP newval,
 7766                          rFlagsReg cr)
 7767 %{
 7768   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7769   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7770   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7771   effect(KILL cr, KILL oldval);
 7772 
 7773   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7774             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7775             "sete    $res\n\t"
 7776             "movzbl  $res, $res" %}
 7777   ins_encode %{
 7778     __ lock();
 7779     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7780     __ sete($res$$Register);
 7781     __ movzbl($res$$Register, $res$$Register);
 7782   %}
 7783   ins_pipe( pipe_cmpxchg );
 7784 %}
 7785 
 7786 instruct compareAndSwapL(rRegI res,
 7787                          memory mem_ptr,
 7788                          rax_RegL oldval, rRegL newval,
 7789                          rFlagsReg cr)
 7790 %{
 7791   predicate(VM_Version::supports_cx8());
 7792   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7793   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7794   effect(KILL cr, KILL oldval);
 7795 
 7796   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7797             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7798             "sete    $res\n\t"
 7799             "movzbl  $res, $res" %}
 7800   ins_encode %{
 7801     __ lock();
 7802     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7803     __ sete($res$$Register);
 7804     __ movzbl($res$$Register, $res$$Register);
 7805   %}
 7806   ins_pipe( pipe_cmpxchg );
 7807 %}
 7808 
 7809 instruct compareAndSwapI(rRegI res,
 7810                          memory mem_ptr,
 7811                          rax_RegI oldval, rRegI newval,
 7812                          rFlagsReg cr)
 7813 %{
 7814   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7815   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7816   effect(KILL cr, KILL oldval);
 7817 
 7818   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7819             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7820             "sete    $res\n\t"
 7821             "movzbl  $res, $res" %}
 7822   ins_encode %{
 7823     __ lock();
 7824     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7825     __ sete($res$$Register);
 7826     __ movzbl($res$$Register, $res$$Register);
 7827   %}
 7828   ins_pipe( pipe_cmpxchg );
 7829 %}
 7830 
 7831 instruct compareAndSwapB(rRegI res,
 7832                          memory mem_ptr,
 7833                          rax_RegI oldval, rRegI newval,
 7834                          rFlagsReg cr)
 7835 %{
 7836   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7837   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7838   effect(KILL cr, KILL oldval);
 7839 
 7840   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7841             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7842             "sete    $res\n\t"
 7843             "movzbl  $res, $res" %}
 7844   ins_encode %{
 7845     __ lock();
 7846     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7847     __ sete($res$$Register);
 7848     __ movzbl($res$$Register, $res$$Register);
 7849   %}
 7850   ins_pipe( pipe_cmpxchg );
 7851 %}
 7852 
 7853 instruct compareAndSwapS(rRegI res,
 7854                          memory mem_ptr,
 7855                          rax_RegI oldval, rRegI newval,
 7856                          rFlagsReg cr)
 7857 %{
 7858   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7859   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7860   effect(KILL cr, KILL oldval);
 7861 
 7862   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7863             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7864             "sete    $res\n\t"
 7865             "movzbl  $res, $res" %}
 7866   ins_encode %{
 7867     __ lock();
 7868     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7869     __ sete($res$$Register);
 7870     __ movzbl($res$$Register, $res$$Register);
 7871   %}
 7872   ins_pipe( pipe_cmpxchg );
 7873 %}
 7874 
 7875 instruct compareAndSwapN(rRegI res,
 7876                           memory mem_ptr,
 7877                           rax_RegN oldval, rRegN newval,
 7878                           rFlagsReg cr) %{
 7879   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7880   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7881   effect(KILL cr, KILL oldval);
 7882 
 7883   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7884             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7885             "sete    $res\n\t"
 7886             "movzbl  $res, $res" %}
 7887   ins_encode %{
 7888     __ lock();
 7889     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7890     __ sete($res$$Register);
 7891     __ movzbl($res$$Register, $res$$Register);
 7892   %}
 7893   ins_pipe( pipe_cmpxchg );
 7894 %}
 7895 
 7896 instruct compareAndExchangeB(
 7897                          memory mem_ptr,
 7898                          rax_RegI oldval, rRegI newval,
 7899                          rFlagsReg cr)
 7900 %{
 7901   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7902   effect(KILL cr);
 7903 
 7904   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7905             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7906   ins_encode %{
 7907     __ lock();
 7908     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7909   %}
 7910   ins_pipe( pipe_cmpxchg );
 7911 %}
 7912 
 7913 instruct compareAndExchangeS(
 7914                          memory mem_ptr,
 7915                          rax_RegI oldval, rRegI newval,
 7916                          rFlagsReg cr)
 7917 %{
 7918   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7919   effect(KILL cr);
 7920 
 7921   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7922             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7923   ins_encode %{
 7924     __ lock();
 7925     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7926   %}
 7927   ins_pipe( pipe_cmpxchg );
 7928 %}
 7929 
 7930 instruct compareAndExchangeI(
 7931                          memory mem_ptr,
 7932                          rax_RegI oldval, rRegI newval,
 7933                          rFlagsReg cr)
 7934 %{
 7935   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7936   effect(KILL cr);
 7937 
 7938   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7939             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7940   ins_encode %{
 7941     __ lock();
 7942     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7943   %}
 7944   ins_pipe( pipe_cmpxchg );
 7945 %}
 7946 
 7947 instruct compareAndExchangeL(
 7948                          memory mem_ptr,
 7949                          rax_RegL oldval, rRegL newval,
 7950                          rFlagsReg cr)
 7951 %{
 7952   predicate(VM_Version::supports_cx8());
 7953   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7954   effect(KILL cr);
 7955 
 7956   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7957             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7958   ins_encode %{
 7959     __ lock();
 7960     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7961   %}
 7962   ins_pipe( pipe_cmpxchg );
 7963 %}
 7964 
 7965 instruct compareAndExchangeN(
 7966                           memory mem_ptr,
 7967                           rax_RegN oldval, rRegN newval,
 7968                           rFlagsReg cr) %{
 7969   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7970   effect(KILL cr);
 7971 
 7972   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7973             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7974   ins_encode %{
 7975     __ lock();
 7976     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7977   %}
 7978   ins_pipe( pipe_cmpxchg );
 7979 %}
 7980 
 7981 instruct compareAndExchangeP(
 7982                          memory mem_ptr,
 7983                          rax_RegP oldval, rRegP newval,
 7984                          rFlagsReg cr)
 7985 %{
 7986   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7987   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7988   effect(KILL cr);
 7989 
 7990   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7991             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7992   ins_encode %{
 7993     __ lock();
 7994     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7995   %}
 7996   ins_pipe( pipe_cmpxchg );
 7997 %}
 7998 
 7999 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8000   predicate(n->as_LoadStore()->result_not_used());
 8001   match(Set dummy (GetAndAddB mem add));
 8002   effect(KILL cr);
 8003   format %{ "ADDB  [$mem],$add" %}
 8004   ins_encode %{
 8005     __ lock();
 8006     __ addb($mem$$Address, $add$$constant);
 8007   %}
 8008   ins_pipe( pipe_cmpxchg );
 8009 %}
 8010 
 8011 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8012   match(Set newval (GetAndAddB mem newval));
 8013   effect(KILL cr);
 8014   format %{ "XADDB  [$mem],$newval" %}
 8015   ins_encode %{
 8016     __ lock();
 8017     __ xaddb($mem$$Address, $newval$$Register);
 8018   %}
 8019   ins_pipe( pipe_cmpxchg );
 8020 %}
 8021 
 8022 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8023   predicate(n->as_LoadStore()->result_not_used());
 8024   match(Set dummy (GetAndAddS mem add));
 8025   effect(KILL cr);
 8026   format %{ "ADDW  [$mem],$add" %}
 8027   ins_encode %{
 8028     __ lock();
 8029     __ addw($mem$$Address, $add$$constant);
 8030   %}
 8031   ins_pipe( pipe_cmpxchg );
 8032 %}
 8033 
 8034 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8035   match(Set newval (GetAndAddS mem newval));
 8036   effect(KILL cr);
 8037   format %{ "XADDW  [$mem],$newval" %}
 8038   ins_encode %{
 8039     __ lock();
 8040     __ xaddw($mem$$Address, $newval$$Register);
 8041   %}
 8042   ins_pipe( pipe_cmpxchg );
 8043 %}
 8044 
 8045 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8046   predicate(n->as_LoadStore()->result_not_used());
 8047   match(Set dummy (GetAndAddI mem add));
 8048   effect(KILL cr);
 8049   format %{ "ADDL  [$mem],$add" %}
 8050   ins_encode %{
 8051     __ lock();
 8052     __ addl($mem$$Address, $add$$constant);
 8053   %}
 8054   ins_pipe( pipe_cmpxchg );
 8055 %}
 8056 
 8057 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8058   match(Set newval (GetAndAddI mem newval));
 8059   effect(KILL cr);
 8060   format %{ "XADDL  [$mem],$newval" %}
 8061   ins_encode %{
 8062     __ lock();
 8063     __ xaddl($mem$$Address, $newval$$Register);
 8064   %}
 8065   ins_pipe( pipe_cmpxchg );
 8066 %}
 8067 
 8068 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8069   predicate(n->as_LoadStore()->result_not_used());
 8070   match(Set dummy (GetAndAddL mem add));
 8071   effect(KILL cr);
 8072   format %{ "ADDQ  [$mem],$add" %}
 8073   ins_encode %{
 8074     __ lock();
 8075     __ addq($mem$$Address, $add$$constant);
 8076   %}
 8077   ins_pipe( pipe_cmpxchg );
 8078 %}
 8079 
 8080 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8081   match(Set newval (GetAndAddL mem newval));
 8082   effect(KILL cr);
 8083   format %{ "XADDQ  [$mem],$newval" %}
 8084   ins_encode %{
 8085     __ lock();
 8086     __ xaddq($mem$$Address, $newval$$Register);
 8087   %}
 8088   ins_pipe( pipe_cmpxchg );
 8089 %}
 8090 
 8091 instruct xchgB( memory mem, rRegI newval) %{
 8092   match(Set newval (GetAndSetB mem newval));
 8093   format %{ "XCHGB  $newval,[$mem]" %}
 8094   ins_encode %{
 8095     __ xchgb($newval$$Register, $mem$$Address);
 8096   %}
 8097   ins_pipe( pipe_cmpxchg );
 8098 %}
 8099 
 8100 instruct xchgS( memory mem, rRegI newval) %{
 8101   match(Set newval (GetAndSetS mem newval));
 8102   format %{ "XCHGW  $newval,[$mem]" %}
 8103   ins_encode %{
 8104     __ xchgw($newval$$Register, $mem$$Address);
 8105   %}
 8106   ins_pipe( pipe_cmpxchg );
 8107 %}
 8108 
 8109 instruct xchgI( memory mem, rRegI newval) %{
 8110   match(Set newval (GetAndSetI mem newval));
 8111   format %{ "XCHGL  $newval,[$mem]" %}
 8112   ins_encode %{
 8113     __ xchgl($newval$$Register, $mem$$Address);
 8114   %}
 8115   ins_pipe( pipe_cmpxchg );
 8116 %}
 8117 
 8118 instruct xchgL( memory mem, rRegL newval) %{
 8119   match(Set newval (GetAndSetL mem newval));
 8120   format %{ "XCHGL  $newval,[$mem]" %}
 8121   ins_encode %{
 8122     __ xchgq($newval$$Register, $mem$$Address);
 8123   %}
 8124   ins_pipe( pipe_cmpxchg );
 8125 %}
 8126 
 8127 instruct xchgP( memory mem, rRegP newval) %{
 8128   match(Set newval (GetAndSetP mem newval));
 8129   predicate(n->as_LoadStore()->barrier_data() == 0);
 8130   format %{ "XCHGQ  $newval,[$mem]" %}
 8131   ins_encode %{
 8132     __ xchgq($newval$$Register, $mem$$Address);
 8133   %}
 8134   ins_pipe( pipe_cmpxchg );
 8135 %}
 8136 
 8137 instruct xchgN( memory mem, rRegN newval) %{
 8138   match(Set newval (GetAndSetN mem newval));
 8139   format %{ "XCHGL  $newval,$mem]" %}
 8140   ins_encode %{
 8141     __ xchgl($newval$$Register, $mem$$Address);
 8142   %}
 8143   ins_pipe( pipe_cmpxchg );
 8144 %}
 8145 
 8146 //----------Abs Instructions-------------------------------------------
 8147 
 8148 // Integer Absolute Instructions
 8149 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8150 %{
 8151   match(Set dst (AbsI src));
 8152   effect(TEMP dst, TEMP tmp, KILL cr);
 8153   format %{ "movl $tmp, $src\n\t"
 8154             "sarl $tmp, 31\n\t"
 8155             "movl $dst, $src\n\t"
 8156             "xorl $dst, $tmp\n\t"
 8157             "subl $dst, $tmp\n"
 8158           %}
 8159   ins_encode %{
 8160     __ movl($tmp$$Register, $src$$Register);
 8161     __ sarl($tmp$$Register, 31);
 8162     __ movl($dst$$Register, $src$$Register);
 8163     __ xorl($dst$$Register, $tmp$$Register);
 8164     __ subl($dst$$Register, $tmp$$Register);
 8165   %}
 8166 
 8167   ins_pipe(ialu_reg_reg);
 8168 %}
 8169 
 8170 // Long Absolute Instructions
 8171 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8172 %{
 8173   match(Set dst (AbsL src));
 8174   effect(TEMP dst, TEMP tmp, KILL cr);
 8175   format %{ "movq $tmp, $src\n\t"
 8176             "sarq $tmp, 63\n\t"
 8177             "movq $dst, $src\n\t"
 8178             "xorq $dst, $tmp\n\t"
 8179             "subq $dst, $tmp\n"
 8180           %}
 8181   ins_encode %{
 8182     __ movq($tmp$$Register, $src$$Register);
 8183     __ sarq($tmp$$Register, 63);
 8184     __ movq($dst$$Register, $src$$Register);
 8185     __ xorq($dst$$Register, $tmp$$Register);
 8186     __ subq($dst$$Register, $tmp$$Register);
 8187   %}
 8188 
 8189   ins_pipe(ialu_reg_reg);
 8190 %}
 8191 
 8192 //----------Subtraction Instructions-------------------------------------------
 8193 
 8194 // Integer Subtraction Instructions
 8195 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8196 %{
 8197   match(Set dst (SubI dst src));
 8198   effect(KILL cr);
 8199 
 8200   format %{ "subl    $dst, $src\t# int" %}
 8201   ins_encode %{
 8202     __ subl($dst$$Register, $src$$Register);
 8203   %}
 8204   ins_pipe(ialu_reg_reg);
 8205 %}
 8206 
 8207 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8208 %{
 8209   match(Set dst (SubI dst src));
 8210   effect(KILL cr);
 8211 
 8212   format %{ "subl    $dst, $src\t# int" %}
 8213   ins_encode %{
 8214     __ subl($dst$$Register, $src$$constant);
 8215   %}
 8216   ins_pipe(ialu_reg);
 8217 %}
 8218 
 8219 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8220 %{
 8221   match(Set dst (SubI dst (LoadI src)));
 8222   effect(KILL cr);
 8223 
 8224   ins_cost(125);
 8225   format %{ "subl    $dst, $src\t# int" %}
 8226   ins_encode %{
 8227     __ subl($dst$$Register, $src$$Address);
 8228   %}
 8229   ins_pipe(ialu_reg_mem);
 8230 %}
 8231 
 8232 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8233 %{
 8234   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8235   effect(KILL cr);
 8236 
 8237   ins_cost(150);
 8238   format %{ "subl    $dst, $src\t# int" %}
 8239   ins_encode %{
 8240     __ subl($dst$$Address, $src$$Register);
 8241   %}
 8242   ins_pipe(ialu_mem_reg);
 8243 %}
 8244 
 8245 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8246 %{
 8247   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8248   effect(KILL cr);
 8249 
 8250   ins_cost(125); // XXX
 8251   format %{ "subl    $dst, $src\t# int" %}
 8252   ins_encode %{
 8253     __ subl($dst$$Address, $src$$constant);
 8254   %}
 8255   ins_pipe(ialu_mem_imm);
 8256 %}
 8257 
 8258 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8259 %{
 8260   match(Set dst (SubL dst src));
 8261   effect(KILL cr);
 8262 
 8263   format %{ "subq    $dst, $src\t# long" %}
 8264   ins_encode %{
 8265     __ subq($dst$$Register, $src$$Register);
 8266   %}
 8267   ins_pipe(ialu_reg_reg);
 8268 %}
 8269 
 8270 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8271 %{
 8272   match(Set dst (SubL dst src));
 8273   effect(KILL cr);
 8274 
 8275   format %{ "subq    $dst, $src\t# long" %}
 8276   ins_encode %{
 8277     __ subq($dst$$Register, $src$$constant);
 8278   %}
 8279   ins_pipe(ialu_reg);
 8280 %}
 8281 
 8282 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8283 %{
 8284   match(Set dst (SubL dst (LoadL src)));
 8285   effect(KILL cr);
 8286 
 8287   ins_cost(125);
 8288   format %{ "subq    $dst, $src\t# long" %}
 8289   ins_encode %{
 8290     __ subq($dst$$Register, $src$$Address);
 8291   %}
 8292   ins_pipe(ialu_reg_mem);
 8293 %}
 8294 
 8295 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8296 %{
 8297   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8298   effect(KILL cr);
 8299 
 8300   ins_cost(150);
 8301   format %{ "subq    $dst, $src\t# long" %}
 8302   ins_encode %{
 8303     __ subq($dst$$Address, $src$$Register);
 8304   %}
 8305   ins_pipe(ialu_mem_reg);
 8306 %}
 8307 
 8308 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8309 %{
 8310   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8311   effect(KILL cr);
 8312 
 8313   ins_cost(125); // XXX
 8314   format %{ "subq    $dst, $src\t# long" %}
 8315   ins_encode %{
 8316     __ subq($dst$$Address, $src$$constant);
 8317   %}
 8318   ins_pipe(ialu_mem_imm);
 8319 %}
 8320 
 8321 // Subtract from a pointer
 8322 // XXX hmpf???
 8323 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8324 %{
 8325   match(Set dst (AddP dst (SubI zero src)));
 8326   effect(KILL cr);
 8327 
 8328   format %{ "subq    $dst, $src\t# ptr - int" %}
 8329   opcode(0x2B);
 8330   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8331   ins_pipe(ialu_reg_reg);
 8332 %}
 8333 
 8334 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8335 %{
 8336   match(Set dst (SubI zero dst));
 8337   effect(KILL cr);
 8338 
 8339   format %{ "negl    $dst\t# int" %}
 8340   ins_encode %{
 8341     __ negl($dst$$Register);
 8342   %}
 8343   ins_pipe(ialu_reg);
 8344 %}
 8345 
 8346 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8347 %{
 8348   match(Set dst (NegI dst));
 8349   effect(KILL cr);
 8350 
 8351   format %{ "negl    $dst\t# int" %}
 8352   ins_encode %{
 8353     __ negl($dst$$Register);
 8354   %}
 8355   ins_pipe(ialu_reg);
 8356 %}
 8357 
 8358 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8359 %{
 8360   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8361   effect(KILL cr);
 8362 
 8363   format %{ "negl    $dst\t# int" %}
 8364   ins_encode %{
 8365     __ negl($dst$$Address);
 8366   %}
 8367   ins_pipe(ialu_reg);
 8368 %}
 8369 
 8370 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8371 %{
 8372   match(Set dst (SubL zero dst));
 8373   effect(KILL cr);
 8374 
 8375   format %{ "negq    $dst\t# long" %}
 8376   ins_encode %{
 8377     __ negq($dst$$Register);
 8378   %}
 8379   ins_pipe(ialu_reg);
 8380 %}
 8381 
 8382 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8383 %{
 8384   match(Set dst (NegL dst));
 8385   effect(KILL cr);
 8386 
 8387   format %{ "negq    $dst\t# int" %}
 8388   ins_encode %{
 8389     __ negq($dst$$Register);
 8390   %}
 8391   ins_pipe(ialu_reg);
 8392 %}
 8393 
 8394 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8395 %{
 8396   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8397   effect(KILL cr);
 8398 
 8399   format %{ "negq    $dst\t# long" %}
 8400   ins_encode %{
 8401     __ negq($dst$$Address);
 8402   %}
 8403   ins_pipe(ialu_reg);
 8404 %}
 8405 
 8406 //----------Multiplication/Division Instructions-------------------------------
 8407 // Integer Multiplication Instructions
 8408 // Multiply Register
 8409 
 8410 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8411 %{
 8412   match(Set dst (MulI dst src));
 8413   effect(KILL cr);
 8414 
 8415   ins_cost(300);
 8416   format %{ "imull   $dst, $src\t# int" %}
 8417   ins_encode %{
 8418     __ imull($dst$$Register, $src$$Register);
 8419   %}
 8420   ins_pipe(ialu_reg_reg_alu0);
 8421 %}
 8422 
 8423 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8424 %{
 8425   match(Set dst (MulI src imm));
 8426   effect(KILL cr);
 8427 
 8428   ins_cost(300);
 8429   format %{ "imull   $dst, $src, $imm\t# int" %}
 8430   ins_encode %{
 8431     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8432   %}
 8433   ins_pipe(ialu_reg_reg_alu0);
 8434 %}
 8435 
 8436 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8437 %{
 8438   match(Set dst (MulI dst (LoadI src)));
 8439   effect(KILL cr);
 8440 
 8441   ins_cost(350);
 8442   format %{ "imull   $dst, $src\t# int" %}
 8443   ins_encode %{
 8444     __ imull($dst$$Register, $src$$Address);
 8445   %}
 8446   ins_pipe(ialu_reg_mem_alu0);
 8447 %}
 8448 
 8449 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8450 %{
 8451   match(Set dst (MulI (LoadI src) imm));
 8452   effect(KILL cr);
 8453 
 8454   ins_cost(300);
 8455   format %{ "imull   $dst, $src, $imm\t# int" %}
 8456   ins_encode %{
 8457     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8458   %}
 8459   ins_pipe(ialu_reg_mem_alu0);
 8460 %}
 8461 
 8462 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8463 %{
 8464   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8465   effect(KILL cr, KILL src2);
 8466 
 8467   expand %{ mulI_rReg(dst, src1, cr);
 8468            mulI_rReg(src2, src3, cr);
 8469            addI_rReg(dst, src2, cr); %}
 8470 %}
 8471 
 8472 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8473 %{
 8474   match(Set dst (MulL dst src));
 8475   effect(KILL cr);
 8476 
 8477   ins_cost(300);
 8478   format %{ "imulq   $dst, $src\t# long" %}
 8479   ins_encode %{
 8480     __ imulq($dst$$Register, $src$$Register);
 8481   %}
 8482   ins_pipe(ialu_reg_reg_alu0);
 8483 %}
 8484 
 8485 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8486 %{
 8487   match(Set dst (MulL src imm));
 8488   effect(KILL cr);
 8489 
 8490   ins_cost(300);
 8491   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8492   ins_encode %{
 8493     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8494   %}
 8495   ins_pipe(ialu_reg_reg_alu0);
 8496 %}
 8497 
 8498 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8499 %{
 8500   match(Set dst (MulL dst (LoadL src)));
 8501   effect(KILL cr);
 8502 
 8503   ins_cost(350);
 8504   format %{ "imulq   $dst, $src\t# long" %}
 8505   ins_encode %{
 8506     __ imulq($dst$$Register, $src$$Address);
 8507   %}
 8508   ins_pipe(ialu_reg_mem_alu0);
 8509 %}
 8510 
 8511 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8512 %{
 8513   match(Set dst (MulL (LoadL src) imm));
 8514   effect(KILL cr);
 8515 
 8516   ins_cost(300);
 8517   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8518   ins_encode %{
 8519     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8520   %}
 8521   ins_pipe(ialu_reg_mem_alu0);
 8522 %}
 8523 
 8524 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8525 %{
 8526   match(Set dst (MulHiL src rax));
 8527   effect(USE_KILL rax, KILL cr);
 8528 
 8529   ins_cost(300);
 8530   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8531   ins_encode %{
 8532     __ imulq($src$$Register);
 8533   %}
 8534   ins_pipe(ialu_reg_reg_alu0);
 8535 %}
 8536 
 8537 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8538                    rFlagsReg cr)
 8539 %{
 8540   match(Set rax (DivI rax div));
 8541   effect(KILL rdx, KILL cr);
 8542 
 8543   ins_cost(30*100+10*100); // XXX
 8544   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8545             "jne,s   normal\n\t"
 8546             "xorl    rdx, rdx\n\t"
 8547             "cmpl    $div, -1\n\t"
 8548             "je,s    done\n"
 8549     "normal: cdql\n\t"
 8550             "idivl   $div\n"
 8551     "done:"        %}
 8552   ins_encode(cdql_enc(div));
 8553   ins_pipe(ialu_reg_reg_alu0);
 8554 %}
 8555 
 8556 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8557                    rFlagsReg cr)
 8558 %{
 8559   match(Set rax (DivL rax div));
 8560   effect(KILL rdx, KILL cr);
 8561 
 8562   ins_cost(30*100+10*100); // XXX
 8563   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8564             "cmpq    rax, rdx\n\t"
 8565             "jne,s   normal\n\t"
 8566             "xorl    rdx, rdx\n\t"
 8567             "cmpq    $div, -1\n\t"
 8568             "je,s    done\n"
 8569     "normal: cdqq\n\t"
 8570             "idivq   $div\n"
 8571     "done:"        %}
 8572   ins_encode(cdqq_enc(div));
 8573   ins_pipe(ialu_reg_reg_alu0);
 8574 %}
 8575 
 8576 // Integer DIVMOD with Register, both quotient and mod results
 8577 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8578                              rFlagsReg cr)
 8579 %{
 8580   match(DivModI rax div);
 8581   effect(KILL cr);
 8582 
 8583   ins_cost(30*100+10*100); // XXX
 8584   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8585             "jne,s   normal\n\t"
 8586             "xorl    rdx, rdx\n\t"
 8587             "cmpl    $div, -1\n\t"
 8588             "je,s    done\n"
 8589     "normal: cdql\n\t"
 8590             "idivl   $div\n"
 8591     "done:"        %}
 8592   ins_encode(cdql_enc(div));
 8593   ins_pipe(pipe_slow);
 8594 %}
 8595 
 8596 // Long DIVMOD with Register, both quotient and mod results
 8597 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8598                              rFlagsReg cr)
 8599 %{
 8600   match(DivModL rax div);
 8601   effect(KILL cr);
 8602 
 8603   ins_cost(30*100+10*100); // XXX
 8604   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8605             "cmpq    rax, rdx\n\t"
 8606             "jne,s   normal\n\t"
 8607             "xorl    rdx, rdx\n\t"
 8608             "cmpq    $div, -1\n\t"
 8609             "je,s    done\n"
 8610     "normal: cdqq\n\t"
 8611             "idivq   $div\n"
 8612     "done:"        %}
 8613   ins_encode(cdqq_enc(div));
 8614   ins_pipe(pipe_slow);
 8615 %}
 8616 
 8617 //----------- DivL-By-Constant-Expansions--------------------------------------
 8618 // DivI cases are handled by the compiler
 8619 
 8620 // Magic constant, reciprocal of 10
 8621 instruct loadConL_0x6666666666666667(rRegL dst)
 8622 %{
 8623   effect(DEF dst);
 8624 
 8625   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 8626   ins_encode(load_immL(dst, 0x6666666666666667));
 8627   ins_pipe(ialu_reg);
 8628 %}
 8629 
 8630 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8631 %{
 8632   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 8633 
 8634   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 8635   ins_encode %{
 8636     __ imulq($src$$Register);
 8637   %}
 8638   ins_pipe(ialu_reg_reg_alu0);
 8639 %}
 8640 
 8641 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 8642 %{
 8643   effect(USE_DEF dst, KILL cr);
 8644 
 8645   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 8646   ins_encode %{
 8647     __ sarq($dst$$Register, 63);
 8648   %}
 8649   ins_pipe(ialu_reg);
 8650 %}
 8651 
 8652 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 8653 %{
 8654   effect(USE_DEF dst, KILL cr);
 8655 
 8656   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 8657   ins_encode %{
 8658     __ sarq($dst$$Register, 2);
 8659   %}
 8660   ins_pipe(ialu_reg);
 8661 %}
 8662 
 8663 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 8664 %{
 8665   match(Set dst (DivL src div));
 8666 
 8667   ins_cost((5+8)*100);
 8668   expand %{
 8669     rax_RegL rax;                     // Killed temp
 8670     rFlagsReg cr;                     // Killed
 8671     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 8672     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 8673     sarL_rReg_63(src, cr);            // sarq  src, 63
 8674     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 8675     subL_rReg(dst, src, cr);          // subl  rdx, src
 8676   %}
 8677 %}
 8678 
 8679 //-----------------------------------------------------------------------------
 8680 
 8681 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8682                    rFlagsReg cr)
 8683 %{
 8684   match(Set rdx (ModI rax div));
 8685   effect(KILL rax, KILL cr);
 8686 
 8687   ins_cost(300); // XXX
 8688   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8689             "jne,s   normal\n\t"
 8690             "xorl    rdx, rdx\n\t"
 8691             "cmpl    $div, -1\n\t"
 8692             "je,s    done\n"
 8693     "normal: cdql\n\t"
 8694             "idivl   $div\n"
 8695     "done:"        %}
 8696   ins_encode(cdql_enc(div));
 8697   ins_pipe(ialu_reg_reg_alu0);
 8698 %}
 8699 
 8700 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8701                    rFlagsReg cr)
 8702 %{
 8703   match(Set rdx (ModL rax div));
 8704   effect(KILL rax, KILL cr);
 8705 
 8706   ins_cost(300); // XXX
 8707   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8708             "cmpq    rax, rdx\n\t"
 8709             "jne,s   normal\n\t"
 8710             "xorl    rdx, rdx\n\t"
 8711             "cmpq    $div, -1\n\t"
 8712             "je,s    done\n"
 8713     "normal: cdqq\n\t"
 8714             "idivq   $div\n"
 8715     "done:"        %}
 8716   ins_encode(cdqq_enc(div));
 8717   ins_pipe(ialu_reg_reg_alu0);
 8718 %}
 8719 
 8720 // Integer Shift Instructions
 8721 // Shift Left by one
 8722 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8723 %{
 8724   match(Set dst (LShiftI dst shift));
 8725   effect(KILL cr);
 8726 
 8727   format %{ "sall    $dst, $shift" %}
 8728   ins_encode %{
 8729     __ sall($dst$$Register, $shift$$constant);
 8730   %}
 8731   ins_pipe(ialu_reg);
 8732 %}
 8733 
 8734 // Shift Left by one
 8735 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8736 %{
 8737   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8738   effect(KILL cr);
 8739 
 8740   format %{ "sall    $dst, $shift\t" %}
 8741   ins_encode %{
 8742     __ sall($dst$$Address, $shift$$constant);
 8743   %}
 8744   ins_pipe(ialu_mem_imm);
 8745 %}
 8746 
 8747 // Shift Left by 8-bit immediate
 8748 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8749 %{
 8750   match(Set dst (LShiftI dst shift));
 8751   effect(KILL cr);
 8752 
 8753   format %{ "sall    $dst, $shift" %}
 8754   ins_encode %{
 8755     __ sall($dst$$Register, $shift$$constant);
 8756   %}
 8757   ins_pipe(ialu_reg);
 8758 %}
 8759 
 8760 // Shift Left by 8-bit immediate
 8761 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8762 %{
 8763   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8764   effect(KILL cr);
 8765 
 8766   format %{ "sall    $dst, $shift" %}
 8767   ins_encode %{
 8768     __ sall($dst$$Address, $shift$$constant);
 8769   %}
 8770   ins_pipe(ialu_mem_imm);
 8771 %}
 8772 
 8773 // Shift Left by variable
 8774 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8775 %{
 8776   match(Set dst (LShiftI dst shift));
 8777   effect(KILL cr);
 8778 
 8779   format %{ "sall    $dst, $shift" %}
 8780   ins_encode %{
 8781     __ sall($dst$$Register);
 8782   %}
 8783   ins_pipe(ialu_reg_reg);
 8784 %}
 8785 
 8786 // Shift Left by variable
 8787 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8788 %{
 8789   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8790   effect(KILL cr);
 8791 
 8792   format %{ "sall    $dst, $shift" %}
 8793   ins_encode %{
 8794     __ sall($dst$$Address);
 8795   %}
 8796   ins_pipe(ialu_mem_reg);
 8797 %}
 8798 
 8799 // Arithmetic shift right by one
 8800 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8801 %{
 8802   match(Set dst (RShiftI dst shift));
 8803   effect(KILL cr);
 8804 
 8805   format %{ "sarl    $dst, $shift" %}
 8806   ins_encode %{
 8807     __ sarl($dst$$Register, $shift$$constant);
 8808   %}
 8809   ins_pipe(ialu_reg);
 8810 %}
 8811 
 8812 // Arithmetic shift right by one
 8813 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8814 %{
 8815   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8816   effect(KILL cr);
 8817 
 8818   format %{ "sarl    $dst, $shift" %}
 8819   ins_encode %{
 8820     __ sarl($dst$$Address, $shift$$constant);
 8821   %}
 8822   ins_pipe(ialu_mem_imm);
 8823 %}
 8824 
 8825 // Arithmetic Shift Right by 8-bit immediate
 8826 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8827 %{
 8828   match(Set dst (RShiftI dst shift));
 8829   effect(KILL cr);
 8830 
 8831   format %{ "sarl    $dst, $shift" %}
 8832   ins_encode %{
 8833     __ sarl($dst$$Register, $shift$$constant);
 8834   %}
 8835   ins_pipe(ialu_mem_imm);
 8836 %}
 8837 
 8838 // Arithmetic Shift Right by 8-bit immediate
 8839 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8840 %{
 8841   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8842   effect(KILL cr);
 8843 
 8844   format %{ "sarl    $dst, $shift" %}
 8845   ins_encode %{
 8846     __ sarl($dst$$Address, $shift$$constant);
 8847   %}
 8848   ins_pipe(ialu_mem_imm);
 8849 %}
 8850 
 8851 // Arithmetic Shift Right by variable
 8852 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8853 %{
 8854   match(Set dst (RShiftI dst shift));
 8855   effect(KILL cr);
 8856   format %{ "sarl    $dst, $shift" %}
 8857   ins_encode %{
 8858     __ sarl($dst$$Register);
 8859   %}
 8860   ins_pipe(ialu_reg_reg);
 8861 %}
 8862 
 8863 // Arithmetic Shift Right by variable
 8864 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8865 %{
 8866   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8867   effect(KILL cr);
 8868 
 8869   format %{ "sarl    $dst, $shift" %}
 8870   ins_encode %{
 8871     __ sarl($dst$$Address);
 8872   %}
 8873   ins_pipe(ialu_mem_reg);
 8874 %}
 8875 
 8876 // Logical shift right by one
 8877 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8878 %{
 8879   match(Set dst (URShiftI dst shift));
 8880   effect(KILL cr);
 8881 
 8882   format %{ "shrl    $dst, $shift" %}
 8883   ins_encode %{
 8884     __ shrl($dst$$Register, $shift$$constant);
 8885   %}
 8886   ins_pipe(ialu_reg);
 8887 %}
 8888 
 8889 // Logical shift right by one
 8890 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8891 %{
 8892   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8893   effect(KILL cr);
 8894 
 8895   format %{ "shrl    $dst, $shift" %}
 8896   ins_encode %{
 8897     __ shrl($dst$$Address, $shift$$constant);
 8898   %}
 8899   ins_pipe(ialu_mem_imm);
 8900 %}
 8901 
 8902 // Logical Shift Right by 8-bit immediate
 8903 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8904 %{
 8905   match(Set dst (URShiftI dst shift));
 8906   effect(KILL cr);
 8907 
 8908   format %{ "shrl    $dst, $shift" %}
 8909   ins_encode %{
 8910     __ shrl($dst$$Register, $shift$$constant);
 8911   %}
 8912   ins_pipe(ialu_reg);
 8913 %}
 8914 
 8915 // Logical Shift Right by 8-bit immediate
 8916 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8917 %{
 8918   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8919   effect(KILL cr);
 8920 
 8921   format %{ "shrl    $dst, $shift" %}
 8922   ins_encode %{
 8923     __ shrl($dst$$Address, $shift$$constant);
 8924   %}
 8925   ins_pipe(ialu_mem_imm);
 8926 %}
 8927 
 8928 // Logical Shift Right by variable
 8929 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8930 %{
 8931   match(Set dst (URShiftI dst shift));
 8932   effect(KILL cr);
 8933 
 8934   format %{ "shrl    $dst, $shift" %}
 8935   ins_encode %{
 8936     __ shrl($dst$$Register);
 8937   %}
 8938   ins_pipe(ialu_reg_reg);
 8939 %}
 8940 
 8941 // Logical Shift Right by variable
 8942 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8943 %{
 8944   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8945   effect(KILL cr);
 8946 
 8947   format %{ "shrl    $dst, $shift" %}
 8948   ins_encode %{
 8949     __ shrl($dst$$Address);
 8950   %}
 8951   ins_pipe(ialu_mem_reg);
 8952 %}
 8953 
 8954 // Long Shift Instructions
 8955 // Shift Left by one
 8956 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 8957 %{
 8958   match(Set dst (LShiftL dst shift));
 8959   effect(KILL cr);
 8960 
 8961   format %{ "salq    $dst, $shift" %}
 8962   ins_encode %{
 8963     __ salq($dst$$Register, $shift$$constant);
 8964   %}
 8965   ins_pipe(ialu_reg);
 8966 %}
 8967 
 8968 // Shift Left by one
 8969 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8970 %{
 8971   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8972   effect(KILL cr);
 8973 
 8974   format %{ "salq    $dst, $shift" %}
 8975   ins_encode %{
 8976     __ salq($dst$$Address, $shift$$constant);
 8977   %}
 8978   ins_pipe(ialu_mem_imm);
 8979 %}
 8980 
 8981 // Shift Left by 8-bit immediate
 8982 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8983 %{
 8984   match(Set dst (LShiftL dst shift));
 8985   effect(KILL cr);
 8986 
 8987   format %{ "salq    $dst, $shift" %}
 8988   ins_encode %{
 8989     __ salq($dst$$Register, $shift$$constant);
 8990   %}
 8991   ins_pipe(ialu_reg);
 8992 %}
 8993 
 8994 // Shift Left by 8-bit immediate
 8995 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8996 %{
 8997   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8998   effect(KILL cr);
 8999 
 9000   format %{ "salq    $dst, $shift" %}
 9001   ins_encode %{
 9002     __ salq($dst$$Address, $shift$$constant);
 9003   %}
 9004   ins_pipe(ialu_mem_imm);
 9005 %}
 9006 
 9007 // Shift Left by variable
 9008 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9009 %{
 9010   match(Set dst (LShiftL dst shift));
 9011   effect(KILL cr);
 9012 
 9013   format %{ "salq    $dst, $shift" %}
 9014   ins_encode %{
 9015     __ salq($dst$$Register);
 9016   %}
 9017   ins_pipe(ialu_reg_reg);
 9018 %}
 9019 
 9020 // Shift Left by variable
 9021 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9022 %{
 9023   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9024   effect(KILL cr);
 9025 
 9026   format %{ "salq    $dst, $shift" %}
 9027   ins_encode %{
 9028     __ salq($dst$$Address);
 9029   %}
 9030   ins_pipe(ialu_mem_reg);
 9031 %}
 9032 
 9033 // Arithmetic shift right by one
 9034 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9035 %{
 9036   match(Set dst (RShiftL dst shift));
 9037   effect(KILL cr);
 9038 
 9039   format %{ "sarq    $dst, $shift" %}
 9040   ins_encode %{
 9041     __ sarq($dst$$Register, $shift$$constant);
 9042   %}
 9043   ins_pipe(ialu_reg);
 9044 %}
 9045 
 9046 // Arithmetic shift right by one
 9047 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9048 %{
 9049   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9050   effect(KILL cr);
 9051 
 9052   format %{ "sarq    $dst, $shift" %}
 9053   ins_encode %{
 9054     __ sarq($dst$$Address, $shift$$constant);
 9055   %}
 9056   ins_pipe(ialu_mem_imm);
 9057 %}
 9058 
 9059 // Arithmetic Shift Right by 8-bit immediate
 9060 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9061 %{
 9062   match(Set dst (RShiftL dst shift));
 9063   effect(KILL cr);
 9064 
 9065   format %{ "sarq    $dst, $shift" %}
 9066   ins_encode %{
 9067     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9068   %}
 9069   ins_pipe(ialu_mem_imm);
 9070 %}
 9071 
 9072 // Arithmetic Shift Right by 8-bit immediate
 9073 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9074 %{
 9075   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9076   effect(KILL cr);
 9077 
 9078   format %{ "sarq    $dst, $shift" %}
 9079   ins_encode %{
 9080     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9081   %}
 9082   ins_pipe(ialu_mem_imm);
 9083 %}
 9084 
 9085 // Arithmetic Shift Right by variable
 9086 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9087 %{
 9088   match(Set dst (RShiftL dst shift));
 9089   effect(KILL cr);
 9090 
 9091   format %{ "sarq    $dst, $shift" %}
 9092   ins_encode %{
 9093     __ sarq($dst$$Register);
 9094   %}
 9095   ins_pipe(ialu_reg_reg);
 9096 %}
 9097 
 9098 // Arithmetic Shift Right by variable
 9099 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9100 %{
 9101   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9102   effect(KILL cr);
 9103 
 9104   format %{ "sarq    $dst, $shift" %}
 9105   ins_encode %{
 9106     __ sarq($dst$$Address);
 9107   %}
 9108   ins_pipe(ialu_mem_reg);
 9109 %}
 9110 
 9111 // Logical shift right by one
 9112 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9113 %{
 9114   match(Set dst (URShiftL dst shift));
 9115   effect(KILL cr);
 9116 
 9117   format %{ "shrq    $dst, $shift" %}
 9118   ins_encode %{
 9119     __ shrq($dst$$Register, $shift$$constant);
 9120   %}
 9121   ins_pipe(ialu_reg);
 9122 %}
 9123 
 9124 // Logical shift right by one
 9125 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9126 %{
 9127   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9128   effect(KILL cr);
 9129 
 9130   format %{ "shrq    $dst, $shift" %}
 9131   ins_encode %{
 9132     __ shrq($dst$$Address, $shift$$constant);
 9133   %}
 9134   ins_pipe(ialu_mem_imm);
 9135 %}
 9136 
 9137 // Logical Shift Right by 8-bit immediate
 9138 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9139 %{
 9140   match(Set dst (URShiftL dst shift));
 9141   effect(KILL cr);
 9142 
 9143   format %{ "shrq    $dst, $shift" %}
 9144   ins_encode %{
 9145     __ shrq($dst$$Register, $shift$$constant);
 9146   %}
 9147   ins_pipe(ialu_reg);
 9148 %}
 9149 
 9150 // Logical Shift Right by 8-bit immediate
 9151 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9152 %{
 9153   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9154   effect(KILL cr);
 9155 
 9156   format %{ "shrq    $dst, $shift" %}
 9157   ins_encode %{
 9158     __ shrq($dst$$Address, $shift$$constant);
 9159   %}
 9160   ins_pipe(ialu_mem_imm);
 9161 %}
 9162 
 9163 // Logical Shift Right by variable
 9164 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9165 %{
 9166   match(Set dst (URShiftL dst shift));
 9167   effect(KILL cr);
 9168 
 9169   format %{ "shrq    $dst, $shift" %}
 9170   ins_encode %{
 9171     __ shrq($dst$$Register);
 9172   %}
 9173   ins_pipe(ialu_reg_reg);
 9174 %}
 9175 
 9176 // Logical Shift Right by variable
 9177 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9178 %{
 9179   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9180   effect(KILL cr);
 9181 
 9182   format %{ "shrq    $dst, $shift" %}
 9183   ins_encode %{
 9184     __ shrq($dst$$Address);
 9185   %}
 9186   ins_pipe(ialu_mem_reg);
 9187 %}
 9188 
 9189 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9190 // This idiom is used by the compiler for the i2b bytecode.
 9191 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9192 %{
 9193   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9194 
 9195   format %{ "movsbl  $dst, $src\t# i2b" %}
 9196   ins_encode %{
 9197     __ movsbl($dst$$Register, $src$$Register);
 9198   %}
 9199   ins_pipe(ialu_reg_reg);
 9200 %}
 9201 
 9202 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9203 // This idiom is used by the compiler the i2s bytecode.
 9204 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9205 %{
 9206   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9207 
 9208   format %{ "movswl  $dst, $src\t# i2s" %}
 9209   ins_encode %{
 9210     __ movswl($dst$$Register, $src$$Register);
 9211   %}
 9212   ins_pipe(ialu_reg_reg);
 9213 %}
 9214 
 9215 // ROL/ROR instructions
 9216 
 9217 // Rotate left by constant.
 9218 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9219 %{
 9220   predicate(n->bottom_type()->basic_type() == T_INT);
 9221   match(Set dst (RotateLeft dst shift));
 9222   effect(KILL cr);
 9223   format %{ "roll    $dst, $shift" %}
 9224   ins_encode %{
 9225     __ roll($dst$$Register, $shift$$constant);
 9226   %}
 9227   ins_pipe(ialu_reg);
 9228 %}
 9229 
 9230 // Rotate Left by variable
 9231 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9232 %{
 9233   predicate(n->bottom_type()->basic_type() == T_INT);
 9234   match(Set dst (RotateLeft dst shift));
 9235   effect(KILL cr);
 9236   format %{ "roll    $dst, $shift" %}
 9237   ins_encode %{
 9238     __ roll($dst$$Register);
 9239   %}
 9240   ins_pipe(ialu_reg_reg);
 9241 %}
 9242 
 9243 // Rotate Right by constant.
 9244 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9245 %{
 9246   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9247   match(Set dst (RotateRight dst shift));
 9248   effect(KILL cr);
 9249   format %{ "rorl    $dst, $shift" %}
 9250   ins_encode %{
 9251     __ rorl($dst$$Register, $shift$$constant);
 9252   %}
 9253   ins_pipe(ialu_reg);
 9254 %}
 9255 
 9256 // Rotate Right by constant.
 9257 instruct rorI_immI8(rRegI dst, immI8 shift)
 9258 %{
 9259   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9260   match(Set dst (RotateRight dst shift));
 9261   format %{ "rorxd     $dst, $shift" %}
 9262   ins_encode %{
 9263     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9264   %}
 9265   ins_pipe(ialu_reg_reg);
 9266 %}
 9267 
 9268 // Rotate Right by variable
 9269 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9270 %{
 9271   predicate(n->bottom_type()->basic_type() == T_INT);
 9272   match(Set dst (RotateRight dst shift));
 9273   effect(KILL cr);
 9274   format %{ "rorl    $dst, $shift" %}
 9275   ins_encode %{
 9276     __ rorl($dst$$Register);
 9277   %}
 9278   ins_pipe(ialu_reg_reg);
 9279 %}
 9280 
 9281 
 9282 // Rotate Left by constant.
 9283 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9284 %{
 9285   predicate(n->bottom_type()->basic_type() == T_LONG);
 9286   match(Set dst (RotateLeft dst shift));
 9287   effect(KILL cr);
 9288   format %{ "rolq    $dst, $shift" %}
 9289   ins_encode %{
 9290     __ rolq($dst$$Register, $shift$$constant);
 9291   %}
 9292   ins_pipe(ialu_reg);
 9293 %}
 9294 
 9295 // Rotate Left by variable
 9296 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9297 %{
 9298   predicate(n->bottom_type()->basic_type() == T_LONG);
 9299   match(Set dst (RotateLeft dst shift));
 9300   effect(KILL cr);
 9301   format %{ "rolq    $dst, $shift" %}
 9302   ins_encode %{
 9303     __ rolq($dst$$Register);
 9304   %}
 9305   ins_pipe(ialu_reg_reg);
 9306 %}
 9307 
 9308 
 9309 // Rotate Right by constant.
 9310 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9311 %{
 9312   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9313   match(Set dst (RotateRight dst shift));
 9314   effect(KILL cr);
 9315   format %{ "rorq    $dst, $shift" %}
 9316   ins_encode %{
 9317     __ rorq($dst$$Register, $shift$$constant);
 9318   %}
 9319   ins_pipe(ialu_reg);
 9320 %}
 9321 
 9322 
 9323 // Rotate Right by constant
 9324 instruct rorL_immI8(rRegL dst, immI8 shift)
 9325 %{
 9326   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9327   match(Set dst (RotateRight dst shift));
 9328   format %{ "rorxq    $dst, $shift" %}
 9329   ins_encode %{
 9330     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9331   %}
 9332   ins_pipe(ialu_reg_reg);
 9333 %}
 9334 
 9335 // Rotate Right by variable
 9336 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9337 %{
 9338   predicate(n->bottom_type()->basic_type() == T_LONG);
 9339   match(Set dst (RotateRight dst shift));
 9340   effect(KILL cr);
 9341   format %{ "rorq    $dst, $shift" %}
 9342   ins_encode %{
 9343     __ rorq($dst$$Register);
 9344   %}
 9345   ins_pipe(ialu_reg_reg);
 9346 %}
 9347 
 9348 
 9349 // Logical Instructions
 9350 
 9351 // Integer Logical Instructions
 9352 
 9353 // And Instructions
 9354 // And Register with Register
 9355 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9356 %{
 9357   match(Set dst (AndI dst src));
 9358   effect(KILL cr);
 9359 
 9360   format %{ "andl    $dst, $src\t# int" %}
 9361   ins_encode %{
 9362     __ andl($dst$$Register, $src$$Register);
 9363   %}
 9364   ins_pipe(ialu_reg_reg);
 9365 %}
 9366 
 9367 // And Register with Immediate 255
 9368 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9369 %{
 9370   match(Set dst (AndI dst src));
 9371 
 9372   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9373   ins_encode %{
 9374     __ movzbl($dst$$Register, $dst$$Register);
 9375   %}
 9376   ins_pipe(ialu_reg);
 9377 %}
 9378 
 9379 // And Register with Immediate 255 and promote to long
 9380 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9381 %{
 9382   match(Set dst (ConvI2L (AndI src mask)));
 9383 
 9384   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9385   ins_encode %{
 9386     __ movzbl($dst$$Register, $src$$Register);
 9387   %}
 9388   ins_pipe(ialu_reg);
 9389 %}
 9390 
 9391 // And Register with Immediate 65535
 9392 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9393 %{
 9394   match(Set dst (AndI dst src));
 9395 
 9396   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9397   ins_encode %{
 9398     __ movzwl($dst$$Register, $dst$$Register);
 9399   %}
 9400   ins_pipe(ialu_reg);
 9401 %}
 9402 
 9403 // And Register with Immediate 65535 and promote to long
 9404 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9405 %{
 9406   match(Set dst (ConvI2L (AndI src mask)));
 9407 
 9408   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9409   ins_encode %{
 9410     __ movzwl($dst$$Register, $src$$Register);
 9411   %}
 9412   ins_pipe(ialu_reg);
 9413 %}
 9414 
 9415 // Can skip int2long conversions after AND with small bitmask
 9416 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9417 %{
 9418   predicate(VM_Version::supports_bmi2());
 9419   ins_cost(125);
 9420   effect(TEMP tmp, KILL cr);
 9421   match(Set dst (ConvI2L (AndI src mask)));
 9422   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9423   ins_encode %{
 9424     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9425     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9426   %}
 9427   ins_pipe(ialu_reg_reg);
 9428 %}
 9429 
 9430 // And Register with Immediate
 9431 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9432 %{
 9433   match(Set dst (AndI dst src));
 9434   effect(KILL cr);
 9435 
 9436   format %{ "andl    $dst, $src\t# int" %}
 9437   ins_encode %{
 9438     __ andl($dst$$Register, $src$$constant);
 9439   %}
 9440   ins_pipe(ialu_reg);
 9441 %}
 9442 
 9443 // And Register with Memory
 9444 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9445 %{
 9446   match(Set dst (AndI dst (LoadI src)));
 9447   effect(KILL cr);
 9448 
 9449   ins_cost(125);
 9450   format %{ "andl    $dst, $src\t# int" %}
 9451   ins_encode %{
 9452     __ andl($dst$$Register, $src$$Address);
 9453   %}
 9454   ins_pipe(ialu_reg_mem);
 9455 %}
 9456 
 9457 // And Memory with Register
 9458 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9459 %{
 9460   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9461   effect(KILL cr);
 9462 
 9463   ins_cost(150);
 9464   format %{ "andb    $dst, $src\t# byte" %}
 9465   ins_encode %{
 9466     __ andb($dst$$Address, $src$$Register);
 9467   %}
 9468   ins_pipe(ialu_mem_reg);
 9469 %}
 9470 
 9471 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9472 %{
 9473   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9474   effect(KILL cr);
 9475 
 9476   ins_cost(150);
 9477   format %{ "andl    $dst, $src\t# int" %}
 9478   ins_encode %{
 9479     __ andl($dst$$Address, $src$$Register);
 9480   %}
 9481   ins_pipe(ialu_mem_reg);
 9482 %}
 9483 
 9484 // And Memory with Immediate
 9485 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9486 %{
 9487   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9488   effect(KILL cr);
 9489 
 9490   ins_cost(125);
 9491   format %{ "andl    $dst, $src\t# int" %}
 9492   ins_encode %{
 9493     __ andl($dst$$Address, $src$$constant);
 9494   %}
 9495   ins_pipe(ialu_mem_imm);
 9496 %}
 9497 
 9498 // BMI1 instructions
 9499 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9500   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9501   predicate(UseBMI1Instructions);
 9502   effect(KILL cr);
 9503 
 9504   ins_cost(125);
 9505   format %{ "andnl  $dst, $src1, $src2" %}
 9506 
 9507   ins_encode %{
 9508     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9509   %}
 9510   ins_pipe(ialu_reg_mem);
 9511 %}
 9512 
 9513 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9514   match(Set dst (AndI (XorI src1 minus_1) src2));
 9515   predicate(UseBMI1Instructions);
 9516   effect(KILL cr);
 9517 
 9518   format %{ "andnl  $dst, $src1, $src2" %}
 9519 
 9520   ins_encode %{
 9521     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9522   %}
 9523   ins_pipe(ialu_reg);
 9524 %}
 9525 
 9526 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9527   match(Set dst (AndI (SubI imm_zero src) src));
 9528   predicate(UseBMI1Instructions);
 9529   effect(KILL cr);
 9530 
 9531   format %{ "blsil  $dst, $src" %}
 9532 
 9533   ins_encode %{
 9534     __ blsil($dst$$Register, $src$$Register);
 9535   %}
 9536   ins_pipe(ialu_reg);
 9537 %}
 9538 
 9539 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9540   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9541   predicate(UseBMI1Instructions);
 9542   effect(KILL cr);
 9543 
 9544   ins_cost(125);
 9545   format %{ "blsil  $dst, $src" %}
 9546 
 9547   ins_encode %{
 9548     __ blsil($dst$$Register, $src$$Address);
 9549   %}
 9550   ins_pipe(ialu_reg_mem);
 9551 %}
 9552 
 9553 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9554 %{
 9555   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9556   predicate(UseBMI1Instructions);
 9557   effect(KILL cr);
 9558 
 9559   ins_cost(125);
 9560   format %{ "blsmskl $dst, $src" %}
 9561 
 9562   ins_encode %{
 9563     __ blsmskl($dst$$Register, $src$$Address);
 9564   %}
 9565   ins_pipe(ialu_reg_mem);
 9566 %}
 9567 
 9568 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9569 %{
 9570   match(Set dst (XorI (AddI src minus_1) src));
 9571   predicate(UseBMI1Instructions);
 9572   effect(KILL cr);
 9573 
 9574   format %{ "blsmskl $dst, $src" %}
 9575 
 9576   ins_encode %{
 9577     __ blsmskl($dst$$Register, $src$$Register);
 9578   %}
 9579 
 9580   ins_pipe(ialu_reg);
 9581 %}
 9582 
 9583 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9584 %{
 9585   match(Set dst (AndI (AddI src minus_1) src) );
 9586   predicate(UseBMI1Instructions);
 9587   effect(KILL cr);
 9588 
 9589   format %{ "blsrl  $dst, $src" %}
 9590 
 9591   ins_encode %{
 9592     __ blsrl($dst$$Register, $src$$Register);
 9593   %}
 9594 
 9595   ins_pipe(ialu_reg_mem);
 9596 %}
 9597 
 9598 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9599 %{
 9600   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9601   predicate(UseBMI1Instructions);
 9602   effect(KILL cr);
 9603 
 9604   ins_cost(125);
 9605   format %{ "blsrl  $dst, $src" %}
 9606 
 9607   ins_encode %{
 9608     __ blsrl($dst$$Register, $src$$Address);
 9609   %}
 9610 
 9611   ins_pipe(ialu_reg);
 9612 %}
 9613 
 9614 // Or Instructions
 9615 // Or Register with Register
 9616 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9617 %{
 9618   match(Set dst (OrI dst src));
 9619   effect(KILL cr);
 9620 
 9621   format %{ "orl     $dst, $src\t# int" %}
 9622   ins_encode %{
 9623     __ orl($dst$$Register, $src$$Register);
 9624   %}
 9625   ins_pipe(ialu_reg_reg);
 9626 %}
 9627 
 9628 // Or Register with Immediate
 9629 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9630 %{
 9631   match(Set dst (OrI dst src));
 9632   effect(KILL cr);
 9633 
 9634   format %{ "orl     $dst, $src\t# int" %}
 9635   ins_encode %{
 9636     __ orl($dst$$Register, $src$$constant);
 9637   %}
 9638   ins_pipe(ialu_reg);
 9639 %}
 9640 
 9641 // Or Register with Memory
 9642 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9643 %{
 9644   match(Set dst (OrI dst (LoadI src)));
 9645   effect(KILL cr);
 9646 
 9647   ins_cost(125);
 9648   format %{ "orl     $dst, $src\t# int" %}
 9649   ins_encode %{
 9650     __ orl($dst$$Register, $src$$Address);
 9651   %}
 9652   ins_pipe(ialu_reg_mem);
 9653 %}
 9654 
 9655 // Or Memory with Register
 9656 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9657 %{
 9658   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9659   effect(KILL cr);
 9660 
 9661   ins_cost(150);
 9662   format %{ "orb    $dst, $src\t# byte" %}
 9663   ins_encode %{
 9664     __ orb($dst$$Address, $src$$Register);
 9665   %}
 9666   ins_pipe(ialu_mem_reg);
 9667 %}
 9668 
 9669 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9670 %{
 9671   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9672   effect(KILL cr);
 9673 
 9674   ins_cost(150);
 9675   format %{ "orl     $dst, $src\t# int" %}
 9676   ins_encode %{
 9677     __ orl($dst$$Address, $src$$Register);
 9678   %}
 9679   ins_pipe(ialu_mem_reg);
 9680 %}
 9681 
 9682 // Or Memory with Immediate
 9683 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9684 %{
 9685   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9686   effect(KILL cr);
 9687 
 9688   ins_cost(125);
 9689   format %{ "orl     $dst, $src\t# int" %}
 9690   ins_encode %{
 9691     __ orl($dst$$Address, $src$$constant);
 9692   %}
 9693   ins_pipe(ialu_mem_imm);
 9694 %}
 9695 
 9696 // Xor Instructions
 9697 // Xor Register with Register
 9698 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9699 %{
 9700   match(Set dst (XorI dst src));
 9701   effect(KILL cr);
 9702 
 9703   format %{ "xorl    $dst, $src\t# int" %}
 9704   ins_encode %{
 9705     __ xorl($dst$$Register, $src$$Register);
 9706   %}
 9707   ins_pipe(ialu_reg_reg);
 9708 %}
 9709 
 9710 // Xor Register with Immediate -1
 9711 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9712   match(Set dst (XorI dst imm));
 9713 
 9714   format %{ "not    $dst" %}
 9715   ins_encode %{
 9716      __ notl($dst$$Register);
 9717   %}
 9718   ins_pipe(ialu_reg);
 9719 %}
 9720 
 9721 // Xor Register with Immediate
 9722 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9723 %{
 9724   match(Set dst (XorI dst src));
 9725   effect(KILL cr);
 9726 
 9727   format %{ "xorl    $dst, $src\t# int" %}
 9728   ins_encode %{
 9729     __ xorl($dst$$Register, $src$$constant);
 9730   %}
 9731   ins_pipe(ialu_reg);
 9732 %}
 9733 
 9734 // Xor Register with Memory
 9735 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9736 %{
 9737   match(Set dst (XorI dst (LoadI src)));
 9738   effect(KILL cr);
 9739 
 9740   ins_cost(125);
 9741   format %{ "xorl    $dst, $src\t# int" %}
 9742   ins_encode %{
 9743     __ xorl($dst$$Register, $src$$Address);
 9744   %}
 9745   ins_pipe(ialu_reg_mem);
 9746 %}
 9747 
 9748 // Xor Memory with Register
 9749 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9750 %{
 9751   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9752   effect(KILL cr);
 9753 
 9754   ins_cost(150);
 9755   format %{ "xorb    $dst, $src\t# byte" %}
 9756   ins_encode %{
 9757     __ xorb($dst$$Address, $src$$Register);
 9758   %}
 9759   ins_pipe(ialu_mem_reg);
 9760 %}
 9761 
 9762 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9763 %{
 9764   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9765   effect(KILL cr);
 9766 
 9767   ins_cost(150);
 9768   format %{ "xorl    $dst, $src\t# int" %}
 9769   ins_encode %{
 9770     __ xorl($dst$$Address, $src$$Register);
 9771   %}
 9772   ins_pipe(ialu_mem_reg);
 9773 %}
 9774 
 9775 // Xor Memory with Immediate
 9776 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9777 %{
 9778   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9779   effect(KILL cr);
 9780 
 9781   ins_cost(125);
 9782   format %{ "xorl    $dst, $src\t# int" %}
 9783   ins_encode %{
 9784     __ xorl($dst$$Address, $src$$constant);
 9785   %}
 9786   ins_pipe(ialu_mem_imm);
 9787 %}
 9788 
 9789 
 9790 // Long Logical Instructions
 9791 
 9792 // And Instructions
 9793 // And Register with Register
 9794 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9795 %{
 9796   match(Set dst (AndL dst src));
 9797   effect(KILL cr);
 9798 
 9799   format %{ "andq    $dst, $src\t# long" %}
 9800   ins_encode %{
 9801     __ andq($dst$$Register, $src$$Register);
 9802   %}
 9803   ins_pipe(ialu_reg_reg);
 9804 %}
 9805 
 9806 // And Register with Immediate 255
 9807 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9808 %{
 9809   match(Set dst (AndL dst src));
 9810 
 9811   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9812   ins_encode %{
 9813     __ movzbq($dst$$Register, $dst$$Register);
 9814   %}
 9815   ins_pipe(ialu_reg);
 9816 %}
 9817 
 9818 // And Register with Immediate 65535
 9819 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9820 %{
 9821   match(Set dst (AndL dst src));
 9822 
 9823   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9824   ins_encode %{
 9825     __ movzwq($dst$$Register, $dst$$Register);
 9826   %}
 9827   ins_pipe(ialu_reg);
 9828 %}
 9829 
 9830 // And Register with Immediate
 9831 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9832 %{
 9833   match(Set dst (AndL dst src));
 9834   effect(KILL cr);
 9835 
 9836   format %{ "andq    $dst, $src\t# long" %}
 9837   ins_encode %{
 9838     __ andq($dst$$Register, $src$$constant);
 9839   %}
 9840   ins_pipe(ialu_reg);
 9841 %}
 9842 
 9843 // And Register with Memory
 9844 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9845 %{
 9846   match(Set dst (AndL dst (LoadL src)));
 9847   effect(KILL cr);
 9848 
 9849   ins_cost(125);
 9850   format %{ "andq    $dst, $src\t# long" %}
 9851   ins_encode %{
 9852     __ andq($dst$$Register, $src$$Address);
 9853   %}
 9854   ins_pipe(ialu_reg_mem);
 9855 %}
 9856 
 9857 // And Memory with Register
 9858 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9859 %{
 9860   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9861   effect(KILL cr);
 9862 
 9863   ins_cost(150);
 9864   format %{ "andq    $dst, $src\t# long" %}
 9865   ins_encode %{
 9866     __ andq($dst$$Address, $src$$Register);
 9867   %}
 9868   ins_pipe(ialu_mem_reg);
 9869 %}
 9870 
 9871 // And Memory with Immediate
 9872 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9873 %{
 9874   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9875   effect(KILL cr);
 9876 
 9877   ins_cost(125);
 9878   format %{ "andq    $dst, $src\t# long" %}
 9879   ins_encode %{
 9880     __ andq($dst$$Address, $src$$constant);
 9881   %}
 9882   ins_pipe(ialu_mem_imm);
 9883 %}
 9884 
 9885 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9886 %{
 9887   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9888   // because AND/OR works well enough for 8/32-bit values.
 9889   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9890 
 9891   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9892   effect(KILL cr);
 9893 
 9894   ins_cost(125);
 9895   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9896   ins_encode %{
 9897     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9898   %}
 9899   ins_pipe(ialu_mem_imm);
 9900 %}
 9901 
 9902 // BMI1 instructions
 9903 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9904   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9905   predicate(UseBMI1Instructions);
 9906   effect(KILL cr);
 9907 
 9908   ins_cost(125);
 9909   format %{ "andnq  $dst, $src1, $src2" %}
 9910 
 9911   ins_encode %{
 9912     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9913   %}
 9914   ins_pipe(ialu_reg_mem);
 9915 %}
 9916 
 9917 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9918   match(Set dst (AndL (XorL src1 minus_1) src2));
 9919   predicate(UseBMI1Instructions);
 9920   effect(KILL cr);
 9921 
 9922   format %{ "andnq  $dst, $src1, $src2" %}
 9923 
 9924   ins_encode %{
 9925   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9926   %}
 9927   ins_pipe(ialu_reg_mem);
 9928 %}
 9929 
 9930 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9931   match(Set dst (AndL (SubL imm_zero src) src));
 9932   predicate(UseBMI1Instructions);
 9933   effect(KILL cr);
 9934 
 9935   format %{ "blsiq  $dst, $src" %}
 9936 
 9937   ins_encode %{
 9938     __ blsiq($dst$$Register, $src$$Register);
 9939   %}
 9940   ins_pipe(ialu_reg);
 9941 %}
 9942 
 9943 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9944   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9945   predicate(UseBMI1Instructions);
 9946   effect(KILL cr);
 9947 
 9948   ins_cost(125);
 9949   format %{ "blsiq  $dst, $src" %}
 9950 
 9951   ins_encode %{
 9952     __ blsiq($dst$$Register, $src$$Address);
 9953   %}
 9954   ins_pipe(ialu_reg_mem);
 9955 %}
 9956 
 9957 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9958 %{
 9959   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9960   predicate(UseBMI1Instructions);
 9961   effect(KILL cr);
 9962 
 9963   ins_cost(125);
 9964   format %{ "blsmskq $dst, $src" %}
 9965 
 9966   ins_encode %{
 9967     __ blsmskq($dst$$Register, $src$$Address);
 9968   %}
 9969   ins_pipe(ialu_reg_mem);
 9970 %}
 9971 
 9972 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9973 %{
 9974   match(Set dst (XorL (AddL src minus_1) src));
 9975   predicate(UseBMI1Instructions);
 9976   effect(KILL cr);
 9977 
 9978   format %{ "blsmskq $dst, $src" %}
 9979 
 9980   ins_encode %{
 9981     __ blsmskq($dst$$Register, $src$$Register);
 9982   %}
 9983 
 9984   ins_pipe(ialu_reg);
 9985 %}
 9986 
 9987 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9988 %{
 9989   match(Set dst (AndL (AddL src minus_1) src) );
 9990   predicate(UseBMI1Instructions);
 9991   effect(KILL cr);
 9992 
 9993   format %{ "blsrq  $dst, $src" %}
 9994 
 9995   ins_encode %{
 9996     __ blsrq($dst$$Register, $src$$Register);
 9997   %}
 9998 
 9999   ins_pipe(ialu_reg);
10000 %}
10001 
10002 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10003 %{
10004   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10005   predicate(UseBMI1Instructions);
10006   effect(KILL cr);
10007 
10008   ins_cost(125);
10009   format %{ "blsrq  $dst, $src" %}
10010 
10011   ins_encode %{
10012     __ blsrq($dst$$Register, $src$$Address);
10013   %}
10014 
10015   ins_pipe(ialu_reg);
10016 %}
10017 
10018 // Or Instructions
10019 // Or Register with Register
10020 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10021 %{
10022   match(Set dst (OrL dst src));
10023   effect(KILL cr);
10024 
10025   format %{ "orq     $dst, $src\t# long" %}
10026   ins_encode %{
10027     __ orq($dst$$Register, $src$$Register);
10028   %}
10029   ins_pipe(ialu_reg_reg);
10030 %}
10031 
10032 // Use any_RegP to match R15 (TLS register) without spilling.
10033 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10034   match(Set dst (OrL dst (CastP2X src)));
10035   effect(KILL cr);
10036 
10037   format %{ "orq     $dst, $src\t# long" %}
10038   ins_encode %{
10039     __ orq($dst$$Register, $src$$Register);
10040   %}
10041   ins_pipe(ialu_reg_reg);
10042 %}
10043 
10044 
10045 // Or Register with Immediate
10046 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10047 %{
10048   match(Set dst (OrL dst src));
10049   effect(KILL cr);
10050 
10051   format %{ "orq     $dst, $src\t# long" %}
10052   ins_encode %{
10053     __ orq($dst$$Register, $src$$constant);
10054   %}
10055   ins_pipe(ialu_reg);
10056 %}
10057 
10058 // Or Register with Memory
10059 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10060 %{
10061   match(Set dst (OrL dst (LoadL src)));
10062   effect(KILL cr);
10063 
10064   ins_cost(125);
10065   format %{ "orq     $dst, $src\t# long" %}
10066   ins_encode %{
10067     __ orq($dst$$Register, $src$$Address);
10068   %}
10069   ins_pipe(ialu_reg_mem);
10070 %}
10071 
10072 // Or Memory with Register
10073 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10074 %{
10075   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10076   effect(KILL cr);
10077 
10078   ins_cost(150);
10079   format %{ "orq     $dst, $src\t# long" %}
10080   ins_encode %{
10081     __ orq($dst$$Address, $src$$Register);
10082   %}
10083   ins_pipe(ialu_mem_reg);
10084 %}
10085 
10086 // Or Memory with Immediate
10087 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10088 %{
10089   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10090   effect(KILL cr);
10091 
10092   ins_cost(125);
10093   format %{ "orq     $dst, $src\t# long" %}
10094   ins_encode %{
10095     __ orq($dst$$Address, $src$$constant);
10096   %}
10097   ins_pipe(ialu_mem_imm);
10098 %}
10099 
10100 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10101 %{
10102   // con should be a pure 64-bit power of 2 immediate
10103   // because AND/OR works well enough for 8/32-bit values.
10104   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10105 
10106   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10107   effect(KILL cr);
10108 
10109   ins_cost(125);
10110   format %{ "btsq    $dst, log2($con)\t# long" %}
10111   ins_encode %{
10112     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10113   %}
10114   ins_pipe(ialu_mem_imm);
10115 %}
10116 
10117 // Xor Instructions
10118 // Xor Register with Register
10119 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10120 %{
10121   match(Set dst (XorL dst src));
10122   effect(KILL cr);
10123 
10124   format %{ "xorq    $dst, $src\t# long" %}
10125   ins_encode %{
10126     __ xorq($dst$$Register, $src$$Register);
10127   %}
10128   ins_pipe(ialu_reg_reg);
10129 %}
10130 
10131 // Xor Register with Immediate -1
10132 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10133   match(Set dst (XorL dst imm));
10134 
10135   format %{ "notq   $dst" %}
10136   ins_encode %{
10137      __ notq($dst$$Register);
10138   %}
10139   ins_pipe(ialu_reg);
10140 %}
10141 
10142 // Xor Register with Immediate
10143 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10144 %{
10145   match(Set dst (XorL dst src));
10146   effect(KILL cr);
10147 
10148   format %{ "xorq    $dst, $src\t# long" %}
10149   ins_encode %{
10150     __ xorq($dst$$Register, $src$$constant);
10151   %}
10152   ins_pipe(ialu_reg);
10153 %}
10154 
10155 // Xor Register with Memory
10156 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10157 %{
10158   match(Set dst (XorL dst (LoadL src)));
10159   effect(KILL cr);
10160 
10161   ins_cost(125);
10162   format %{ "xorq    $dst, $src\t# long" %}
10163   ins_encode %{
10164     __ xorq($dst$$Register, $src$$Address);
10165   %}
10166   ins_pipe(ialu_reg_mem);
10167 %}
10168 
10169 // Xor Memory with Register
10170 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10171 %{
10172   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10173   effect(KILL cr);
10174 
10175   ins_cost(150);
10176   format %{ "xorq    $dst, $src\t# long" %}
10177   ins_encode %{
10178     __ xorq($dst$$Address, $src$$Register);
10179   %}
10180   ins_pipe(ialu_mem_reg);
10181 %}
10182 
10183 // Xor Memory with Immediate
10184 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10185 %{
10186   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10187   effect(KILL cr);
10188 
10189   ins_cost(125);
10190   format %{ "xorq    $dst, $src\t# long" %}
10191   ins_encode %{
10192     __ xorq($dst$$Address, $src$$constant);
10193   %}
10194   ins_pipe(ialu_mem_imm);
10195 %}
10196 
10197 // Convert Int to Boolean
10198 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10199 %{
10200   match(Set dst (Conv2B src));
10201   effect(KILL cr);
10202 
10203   format %{ "testl   $src, $src\t# ci2b\n\t"
10204             "setnz   $dst\n\t"
10205             "movzbl  $dst, $dst" %}
10206   ins_encode %{
10207     __ testl($src$$Register, $src$$Register);
10208     __ set_byte_if_not_zero($dst$$Register);
10209     __ movzbl($dst$$Register, $dst$$Register);
10210   %}
10211   ins_pipe(pipe_slow); // XXX
10212 %}
10213 
10214 // Convert Pointer to Boolean
10215 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10216 %{
10217   match(Set dst (Conv2B src));
10218   effect(KILL cr);
10219 
10220   format %{ "testq   $src, $src\t# cp2b\n\t"
10221             "setnz   $dst\n\t"
10222             "movzbl  $dst, $dst" %}
10223   ins_encode %{
10224     __ testq($src$$Register, $src$$Register);
10225     __ set_byte_if_not_zero($dst$$Register);
10226     __ movzbl($dst$$Register, $dst$$Register);
10227   %}
10228   ins_pipe(pipe_slow); // XXX
10229 %}
10230 
10231 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10232 %{
10233   match(Set dst (CmpLTMask p q));
10234   effect(KILL cr);
10235 
10236   ins_cost(400);
10237   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10238             "setlt   $dst\n\t"
10239             "movzbl  $dst, $dst\n\t"
10240             "negl    $dst" %}
10241   ins_encode %{
10242     __ cmpl($p$$Register, $q$$Register);
10243     __ setl($dst$$Register);
10244     __ movzbl($dst$$Register, $dst$$Register);
10245     __ negl($dst$$Register);
10246   %}
10247   ins_pipe(pipe_slow);
10248 %}
10249 
10250 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10251 %{
10252   match(Set dst (CmpLTMask dst zero));
10253   effect(KILL cr);
10254 
10255   ins_cost(100);
10256   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10257   ins_encode %{
10258     __ sarl($dst$$Register, 31);
10259   %}
10260   ins_pipe(ialu_reg);
10261 %}
10262 
10263 /* Better to save a register than avoid a branch */
10264 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10265 %{
10266   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10267   effect(KILL cr);
10268   ins_cost(300);
10269   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10270             "jge     done\n\t"
10271             "addl    $p,$y\n"
10272             "done:   " %}
10273   ins_encode %{
10274     Register Rp = $p$$Register;
10275     Register Rq = $q$$Register;
10276     Register Ry = $y$$Register;
10277     Label done;
10278     __ subl(Rp, Rq);
10279     __ jccb(Assembler::greaterEqual, done);
10280     __ addl(Rp, Ry);
10281     __ bind(done);
10282   %}
10283   ins_pipe(pipe_cmplt);
10284 %}
10285 
10286 /* Better to save a register than avoid a branch */
10287 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10288 %{
10289   match(Set y (AndI (CmpLTMask p q) y));
10290   effect(KILL cr);
10291 
10292   ins_cost(300);
10293 
10294   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10295             "jlt     done\n\t"
10296             "xorl    $y, $y\n"
10297             "done:   " %}
10298   ins_encode %{
10299     Register Rp = $p$$Register;
10300     Register Rq = $q$$Register;
10301     Register Ry = $y$$Register;
10302     Label done;
10303     __ cmpl(Rp, Rq);
10304     __ jccb(Assembler::less, done);
10305     __ xorl(Ry, Ry);
10306     __ bind(done);
10307   %}
10308   ins_pipe(pipe_cmplt);
10309 %}
10310 
10311 
10312 //---------- FP Instructions------------------------------------------------
10313 
10314 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10315 %{
10316   match(Set cr (CmpF src1 src2));
10317 
10318   ins_cost(145);
10319   format %{ "ucomiss $src1, $src2\n\t"
10320             "jnp,s   exit\n\t"
10321             "pushfq\t# saw NaN, set CF\n\t"
10322             "andq    [rsp], #0xffffff2b\n\t"
10323             "popfq\n"
10324     "exit:" %}
10325   ins_encode %{
10326     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10327     emit_cmpfp_fixup(_masm);
10328   %}
10329   ins_pipe(pipe_slow);
10330 %}
10331 
10332 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10333   match(Set cr (CmpF src1 src2));
10334 
10335   ins_cost(100);
10336   format %{ "ucomiss $src1, $src2" %}
10337   ins_encode %{
10338     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10339   %}
10340   ins_pipe(pipe_slow);
10341 %}
10342 
10343 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10344 %{
10345   match(Set cr (CmpF src1 (LoadF src2)));
10346 
10347   ins_cost(145);
10348   format %{ "ucomiss $src1, $src2\n\t"
10349             "jnp,s   exit\n\t"
10350             "pushfq\t# saw NaN, set CF\n\t"
10351             "andq    [rsp], #0xffffff2b\n\t"
10352             "popfq\n"
10353     "exit:" %}
10354   ins_encode %{
10355     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10356     emit_cmpfp_fixup(_masm);
10357   %}
10358   ins_pipe(pipe_slow);
10359 %}
10360 
10361 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10362   match(Set cr (CmpF src1 (LoadF src2)));
10363 
10364   ins_cost(100);
10365   format %{ "ucomiss $src1, $src2" %}
10366   ins_encode %{
10367     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10368   %}
10369   ins_pipe(pipe_slow);
10370 %}
10371 
10372 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10373   match(Set cr (CmpF src con));
10374 
10375   ins_cost(145);
10376   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10377             "jnp,s   exit\n\t"
10378             "pushfq\t# saw NaN, set CF\n\t"
10379             "andq    [rsp], #0xffffff2b\n\t"
10380             "popfq\n"
10381     "exit:" %}
10382   ins_encode %{
10383     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10384     emit_cmpfp_fixup(_masm);
10385   %}
10386   ins_pipe(pipe_slow);
10387 %}
10388 
10389 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10390   match(Set cr (CmpF src con));
10391   ins_cost(100);
10392   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10393   ins_encode %{
10394     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10395   %}
10396   ins_pipe(pipe_slow);
10397 %}
10398 
10399 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10400 %{
10401   match(Set cr (CmpD src1 src2));
10402 
10403   ins_cost(145);
10404   format %{ "ucomisd $src1, $src2\n\t"
10405             "jnp,s   exit\n\t"
10406             "pushfq\t# saw NaN, set CF\n\t"
10407             "andq    [rsp], #0xffffff2b\n\t"
10408             "popfq\n"
10409     "exit:" %}
10410   ins_encode %{
10411     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10412     emit_cmpfp_fixup(_masm);
10413   %}
10414   ins_pipe(pipe_slow);
10415 %}
10416 
10417 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10418   match(Set cr (CmpD src1 src2));
10419 
10420   ins_cost(100);
10421   format %{ "ucomisd $src1, $src2 test" %}
10422   ins_encode %{
10423     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10424   %}
10425   ins_pipe(pipe_slow);
10426 %}
10427 
10428 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10429 %{
10430   match(Set cr (CmpD src1 (LoadD src2)));
10431 
10432   ins_cost(145);
10433   format %{ "ucomisd $src1, $src2\n\t"
10434             "jnp,s   exit\n\t"
10435             "pushfq\t# saw NaN, set CF\n\t"
10436             "andq    [rsp], #0xffffff2b\n\t"
10437             "popfq\n"
10438     "exit:" %}
10439   ins_encode %{
10440     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10441     emit_cmpfp_fixup(_masm);
10442   %}
10443   ins_pipe(pipe_slow);
10444 %}
10445 
10446 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10447   match(Set cr (CmpD src1 (LoadD src2)));
10448 
10449   ins_cost(100);
10450   format %{ "ucomisd $src1, $src2" %}
10451   ins_encode %{
10452     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10453   %}
10454   ins_pipe(pipe_slow);
10455 %}
10456 
10457 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10458   match(Set cr (CmpD src con));
10459 
10460   ins_cost(145);
10461   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10462             "jnp,s   exit\n\t"
10463             "pushfq\t# saw NaN, set CF\n\t"
10464             "andq    [rsp], #0xffffff2b\n\t"
10465             "popfq\n"
10466     "exit:" %}
10467   ins_encode %{
10468     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10469     emit_cmpfp_fixup(_masm);
10470   %}
10471   ins_pipe(pipe_slow);
10472 %}
10473 
10474 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10475   match(Set cr (CmpD src con));
10476   ins_cost(100);
10477   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10478   ins_encode %{
10479     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10480   %}
10481   ins_pipe(pipe_slow);
10482 %}
10483 
10484 // Compare into -1,0,1
10485 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10486 %{
10487   match(Set dst (CmpF3 src1 src2));
10488   effect(KILL cr);
10489 
10490   ins_cost(275);
10491   format %{ "ucomiss $src1, $src2\n\t"
10492             "movl    $dst, #-1\n\t"
10493             "jp,s    done\n\t"
10494             "jb,s    done\n\t"
10495             "setne   $dst\n\t"
10496             "movzbl  $dst, $dst\n"
10497     "done:" %}
10498   ins_encode %{
10499     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10500     emit_cmpfp3(_masm, $dst$$Register);
10501   %}
10502   ins_pipe(pipe_slow);
10503 %}
10504 
10505 // Compare into -1,0,1
10506 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10507 %{
10508   match(Set dst (CmpF3 src1 (LoadF src2)));
10509   effect(KILL cr);
10510 
10511   ins_cost(275);
10512   format %{ "ucomiss $src1, $src2\n\t"
10513             "movl    $dst, #-1\n\t"
10514             "jp,s    done\n\t"
10515             "jb,s    done\n\t"
10516             "setne   $dst\n\t"
10517             "movzbl  $dst, $dst\n"
10518     "done:" %}
10519   ins_encode %{
10520     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10521     emit_cmpfp3(_masm, $dst$$Register);
10522   %}
10523   ins_pipe(pipe_slow);
10524 %}
10525 
10526 // Compare into -1,0,1
10527 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10528   match(Set dst (CmpF3 src con));
10529   effect(KILL cr);
10530 
10531   ins_cost(275);
10532   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10533             "movl    $dst, #-1\n\t"
10534             "jp,s    done\n\t"
10535             "jb,s    done\n\t"
10536             "setne   $dst\n\t"
10537             "movzbl  $dst, $dst\n"
10538     "done:" %}
10539   ins_encode %{
10540     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10541     emit_cmpfp3(_masm, $dst$$Register);
10542   %}
10543   ins_pipe(pipe_slow);
10544 %}
10545 
10546 // Compare into -1,0,1
10547 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10548 %{
10549   match(Set dst (CmpD3 src1 src2));
10550   effect(KILL cr);
10551 
10552   ins_cost(275);
10553   format %{ "ucomisd $src1, $src2\n\t"
10554             "movl    $dst, #-1\n\t"
10555             "jp,s    done\n\t"
10556             "jb,s    done\n\t"
10557             "setne   $dst\n\t"
10558             "movzbl  $dst, $dst\n"
10559     "done:" %}
10560   ins_encode %{
10561     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10562     emit_cmpfp3(_masm, $dst$$Register);
10563   %}
10564   ins_pipe(pipe_slow);
10565 %}
10566 
10567 // Compare into -1,0,1
10568 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10569 %{
10570   match(Set dst (CmpD3 src1 (LoadD src2)));
10571   effect(KILL cr);
10572 
10573   ins_cost(275);
10574   format %{ "ucomisd $src1, $src2\n\t"
10575             "movl    $dst, #-1\n\t"
10576             "jp,s    done\n\t"
10577             "jb,s    done\n\t"
10578             "setne   $dst\n\t"
10579             "movzbl  $dst, $dst\n"
10580     "done:" %}
10581   ins_encode %{
10582     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10583     emit_cmpfp3(_masm, $dst$$Register);
10584   %}
10585   ins_pipe(pipe_slow);
10586 %}
10587 
10588 // Compare into -1,0,1
10589 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10590   match(Set dst (CmpD3 src con));
10591   effect(KILL cr);
10592 
10593   ins_cost(275);
10594   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10595             "movl    $dst, #-1\n\t"
10596             "jp,s    done\n\t"
10597             "jb,s    done\n\t"
10598             "setne   $dst\n\t"
10599             "movzbl  $dst, $dst\n"
10600     "done:" %}
10601   ins_encode %{
10602     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10603     emit_cmpfp3(_masm, $dst$$Register);
10604   %}
10605   ins_pipe(pipe_slow);
10606 %}
10607 
10608 //----------Arithmetic Conversion Instructions---------------------------------
10609 
10610 instruct convF2D_reg_reg(regD dst, regF src)
10611 %{
10612   match(Set dst (ConvF2D src));
10613 
10614   format %{ "cvtss2sd $dst, $src" %}
10615   ins_encode %{
10616     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10617   %}
10618   ins_pipe(pipe_slow); // XXX
10619 %}
10620 
10621 instruct convF2D_reg_mem(regD dst, memory src)
10622 %{
10623   match(Set dst (ConvF2D (LoadF src)));
10624 
10625   format %{ "cvtss2sd $dst, $src" %}
10626   ins_encode %{
10627     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10628   %}
10629   ins_pipe(pipe_slow); // XXX
10630 %}
10631 
10632 instruct convD2F_reg_reg(regF dst, regD src)
10633 %{
10634   match(Set dst (ConvD2F src));
10635 
10636   format %{ "cvtsd2ss $dst, $src" %}
10637   ins_encode %{
10638     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10639   %}
10640   ins_pipe(pipe_slow); // XXX
10641 %}
10642 
10643 instruct convD2F_reg_mem(regF dst, memory src)
10644 %{
10645   match(Set dst (ConvD2F (LoadD src)));
10646 
10647   format %{ "cvtsd2ss $dst, $src" %}
10648   ins_encode %{
10649     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10650   %}
10651   ins_pipe(pipe_slow); // XXX
10652 %}
10653 
10654 // XXX do mem variants
10655 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10656 %{
10657   match(Set dst (ConvF2I src));
10658   effect(KILL cr);
10659   format %{ "convert_f2i $dst,$src" %}
10660   ins_encode %{
10661     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10662   %}
10663   ins_pipe(pipe_slow);
10664 %}
10665 
10666 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10667 %{
10668   match(Set dst (ConvF2L src));
10669   effect(KILL cr);
10670   format %{ "convert_f2l $dst,$src"%}
10671   ins_encode %{
10672     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10673   %}
10674   ins_pipe(pipe_slow);
10675 %}
10676 
10677 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10678 %{
10679   match(Set dst (ConvD2I src));
10680   effect(KILL cr);
10681   format %{ "convert_d2i $dst,$src"%}
10682   ins_encode %{
10683     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10684   %}
10685   ins_pipe(pipe_slow);
10686 %}
10687 
10688 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10689 %{
10690   match(Set dst (ConvD2L src));
10691   effect(KILL cr);
10692   format %{ "convert_d2l $dst,$src"%}
10693   ins_encode %{
10694     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10695   %}
10696   ins_pipe(pipe_slow);
10697 %}
10698 
10699 instruct convI2F_reg_reg(regF dst, rRegI src)
10700 %{
10701   predicate(!UseXmmI2F);
10702   match(Set dst (ConvI2F src));
10703 
10704   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10705   ins_encode %{
10706     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10707   %}
10708   ins_pipe(pipe_slow); // XXX
10709 %}
10710 
10711 instruct convI2F_reg_mem(regF dst, memory src)
10712 %{
10713   match(Set dst (ConvI2F (LoadI src)));
10714 
10715   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10716   ins_encode %{
10717     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10718   %}
10719   ins_pipe(pipe_slow); // XXX
10720 %}
10721 
10722 instruct convI2D_reg_reg(regD dst, rRegI src)
10723 %{
10724   predicate(!UseXmmI2D);
10725   match(Set dst (ConvI2D src));
10726 
10727   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10728   ins_encode %{
10729     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10730   %}
10731   ins_pipe(pipe_slow); // XXX
10732 %}
10733 
10734 instruct convI2D_reg_mem(regD dst, memory src)
10735 %{
10736   match(Set dst (ConvI2D (LoadI src)));
10737 
10738   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10739   ins_encode %{
10740     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10741   %}
10742   ins_pipe(pipe_slow); // XXX
10743 %}
10744 
10745 instruct convXI2F_reg(regF dst, rRegI src)
10746 %{
10747   predicate(UseXmmI2F);
10748   match(Set dst (ConvI2F src));
10749 
10750   format %{ "movdl $dst, $src\n\t"
10751             "cvtdq2psl $dst, $dst\t# i2f" %}
10752   ins_encode %{
10753     __ movdl($dst$$XMMRegister, $src$$Register);
10754     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10755   %}
10756   ins_pipe(pipe_slow); // XXX
10757 %}
10758 
10759 instruct convXI2D_reg(regD dst, rRegI src)
10760 %{
10761   predicate(UseXmmI2D);
10762   match(Set dst (ConvI2D src));
10763 
10764   format %{ "movdl $dst, $src\n\t"
10765             "cvtdq2pdl $dst, $dst\t# i2d" %}
10766   ins_encode %{
10767     __ movdl($dst$$XMMRegister, $src$$Register);
10768     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10769   %}
10770   ins_pipe(pipe_slow); // XXX
10771 %}
10772 
10773 instruct convL2F_reg_reg(regF dst, rRegL src)
10774 %{
10775   match(Set dst (ConvL2F src));
10776 
10777   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10778   ins_encode %{
10779     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10780   %}
10781   ins_pipe(pipe_slow); // XXX
10782 %}
10783 
10784 instruct convL2F_reg_mem(regF dst, memory src)
10785 %{
10786   match(Set dst (ConvL2F (LoadL src)));
10787 
10788   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10789   ins_encode %{
10790     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10791   %}
10792   ins_pipe(pipe_slow); // XXX
10793 %}
10794 
10795 instruct convL2D_reg_reg(regD dst, rRegL src)
10796 %{
10797   match(Set dst (ConvL2D src));
10798 
10799   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10800   ins_encode %{
10801     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10802   %}
10803   ins_pipe(pipe_slow); // XXX
10804 %}
10805 
10806 instruct convL2D_reg_mem(regD dst, memory src)
10807 %{
10808   match(Set dst (ConvL2D (LoadL src)));
10809 
10810   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10811   ins_encode %{
10812     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10813   %}
10814   ins_pipe(pipe_slow); // XXX
10815 %}
10816 
10817 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10818 %{
10819   match(Set dst (ConvI2L src));
10820 
10821   ins_cost(125);
10822   format %{ "movslq  $dst, $src\t# i2l" %}
10823   ins_encode %{
10824     __ movslq($dst$$Register, $src$$Register);
10825   %}
10826   ins_pipe(ialu_reg_reg);
10827 %}
10828 
10829 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10830 // %{
10831 //   match(Set dst (ConvI2L src));
10832 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10833 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10834 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10835 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10836 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10837 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10838 
10839 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10840 //   ins_encode(enc_copy(dst, src));
10841 // //   opcode(0x63); // needs REX.W
10842 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10843 //   ins_pipe(ialu_reg_reg);
10844 // %}
10845 
10846 // Zero-extend convert int to long
10847 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10848 %{
10849   match(Set dst (AndL (ConvI2L src) mask));
10850 
10851   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10852   ins_encode %{
10853     if ($dst$$reg != $src$$reg) {
10854       __ movl($dst$$Register, $src$$Register);
10855     }
10856   %}
10857   ins_pipe(ialu_reg_reg);
10858 %}
10859 
10860 // Zero-extend convert int to long
10861 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10862 %{
10863   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10864 
10865   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10866   ins_encode %{
10867     __ movl($dst$$Register, $src$$Address);
10868   %}
10869   ins_pipe(ialu_reg_mem);
10870 %}
10871 
10872 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10873 %{
10874   match(Set dst (AndL src mask));
10875 
10876   format %{ "movl    $dst, $src\t# zero-extend long" %}
10877   ins_encode %{
10878     __ movl($dst$$Register, $src$$Register);
10879   %}
10880   ins_pipe(ialu_reg_reg);
10881 %}
10882 
10883 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10884 %{
10885   match(Set dst (ConvL2I src));
10886 
10887   format %{ "movl    $dst, $src\t# l2i" %}
10888   ins_encode %{
10889     __ movl($dst$$Register, $src$$Register);
10890   %}
10891   ins_pipe(ialu_reg_reg);
10892 %}
10893 
10894 
10895 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10896   match(Set dst (MoveF2I src));
10897   effect(DEF dst, USE src);
10898 
10899   ins_cost(125);
10900   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10901   ins_encode %{
10902     __ movl($dst$$Register, Address(rsp, $src$$disp));
10903   %}
10904   ins_pipe(ialu_reg_mem);
10905 %}
10906 
10907 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10908   match(Set dst (MoveI2F src));
10909   effect(DEF dst, USE src);
10910 
10911   ins_cost(125);
10912   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10913   ins_encode %{
10914     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10915   %}
10916   ins_pipe(pipe_slow);
10917 %}
10918 
10919 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10920   match(Set dst (MoveD2L src));
10921   effect(DEF dst, USE src);
10922 
10923   ins_cost(125);
10924   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10925   ins_encode %{
10926     __ movq($dst$$Register, Address(rsp, $src$$disp));
10927   %}
10928   ins_pipe(ialu_reg_mem);
10929 %}
10930 
10931 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10932   predicate(!UseXmmLoadAndClearUpper);
10933   match(Set dst (MoveL2D src));
10934   effect(DEF dst, USE src);
10935 
10936   ins_cost(125);
10937   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10938   ins_encode %{
10939     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10940   %}
10941   ins_pipe(pipe_slow);
10942 %}
10943 
10944 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10945   predicate(UseXmmLoadAndClearUpper);
10946   match(Set dst (MoveL2D src));
10947   effect(DEF dst, USE src);
10948 
10949   ins_cost(125);
10950   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10951   ins_encode %{
10952     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10953   %}
10954   ins_pipe(pipe_slow);
10955 %}
10956 
10957 
10958 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10959   match(Set dst (MoveF2I src));
10960   effect(DEF dst, USE src);
10961 
10962   ins_cost(95); // XXX
10963   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10964   ins_encode %{
10965     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10966   %}
10967   ins_pipe(pipe_slow);
10968 %}
10969 
10970 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10971   match(Set dst (MoveI2F src));
10972   effect(DEF dst, USE src);
10973 
10974   ins_cost(100);
10975   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10976   ins_encode %{
10977     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10978   %}
10979   ins_pipe( ialu_mem_reg );
10980 %}
10981 
10982 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10983   match(Set dst (MoveD2L src));
10984   effect(DEF dst, USE src);
10985 
10986   ins_cost(95); // XXX
10987   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10988   ins_encode %{
10989     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10990   %}
10991   ins_pipe(pipe_slow);
10992 %}
10993 
10994 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10995   match(Set dst (MoveL2D src));
10996   effect(DEF dst, USE src);
10997 
10998   ins_cost(100);
10999   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11000   ins_encode %{
11001     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11002   %}
11003   ins_pipe(ialu_mem_reg);
11004 %}
11005 
11006 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11007   match(Set dst (MoveF2I src));
11008   effect(DEF dst, USE src);
11009   ins_cost(85);
11010   format %{ "movd    $dst,$src\t# MoveF2I" %}
11011   ins_encode %{
11012     __ movdl($dst$$Register, $src$$XMMRegister);
11013   %}
11014   ins_pipe( pipe_slow );
11015 %}
11016 
11017 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11018   match(Set dst (MoveD2L src));
11019   effect(DEF dst, USE src);
11020   ins_cost(85);
11021   format %{ "movd    $dst,$src\t# MoveD2L" %}
11022   ins_encode %{
11023     __ movdq($dst$$Register, $src$$XMMRegister);
11024   %}
11025   ins_pipe( pipe_slow );
11026 %}
11027 
11028 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11029   match(Set dst (MoveI2F src));
11030   effect(DEF dst, USE src);
11031   ins_cost(100);
11032   format %{ "movd    $dst,$src\t# MoveI2F" %}
11033   ins_encode %{
11034     __ movdl($dst$$XMMRegister, $src$$Register);
11035   %}
11036   ins_pipe( pipe_slow );
11037 %}
11038 
11039 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11040   match(Set dst (MoveL2D src));
11041   effect(DEF dst, USE src);
11042   ins_cost(100);
11043   format %{ "movd    $dst,$src\t# MoveL2D" %}
11044   ins_encode %{
11045      __ movdq($dst$$XMMRegister, $src$$Register);
11046   %}
11047   ins_pipe( pipe_slow );
11048 %}
11049 
11050 // Fast clearing of an array
11051 // Small ClearArray non-AVX512.
11052 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11053                   Universe dummy, rFlagsReg cr)
11054 %{
11055   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11056   match(Set dummy (ClearArray cnt base));
11057   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11058 
11059   format %{ $$template
11060     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11061     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11062     $$emit$$"jg      LARGE\n\t"
11063     $$emit$$"dec     rcx\n\t"
11064     $$emit$$"js      DONE\t# Zero length\n\t"
11065     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11066     $$emit$$"dec     rcx\n\t"
11067     $$emit$$"jge     LOOP\n\t"
11068     $$emit$$"jmp     DONE\n\t"
11069     $$emit$$"# LARGE:\n\t"
11070     if (UseFastStosb) {
11071        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11072        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11073     } else if (UseXMMForObjInit) {
11074        $$emit$$"mov     rdi,rax\n\t"
11075        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11076        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11077        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11078        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11079        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11080        $$emit$$"add     0x40,rax\n\t"
11081        $$emit$$"# L_zero_64_bytes:\n\t"
11082        $$emit$$"sub     0x8,rcx\n\t"
11083        $$emit$$"jge     L_loop\n\t"
11084        $$emit$$"add     0x4,rcx\n\t"
11085        $$emit$$"jl      L_tail\n\t"
11086        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11087        $$emit$$"add     0x20,rax\n\t"
11088        $$emit$$"sub     0x4,rcx\n\t"
11089        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11090        $$emit$$"add     0x4,rcx\n\t"
11091        $$emit$$"jle     L_end\n\t"
11092        $$emit$$"dec     rcx\n\t"
11093        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11094        $$emit$$"vmovq   xmm0,(rax)\n\t"
11095        $$emit$$"add     0x8,rax\n\t"
11096        $$emit$$"dec     rcx\n\t"
11097        $$emit$$"jge     L_sloop\n\t"
11098        $$emit$$"# L_end:\n\t"
11099     } else {
11100        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11101     }
11102     $$emit$$"# DONE"
11103   %}
11104   ins_encode %{
11105     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11106                  $tmp$$XMMRegister, false, knoreg);
11107   %}
11108   ins_pipe(pipe_slow);
11109 %}
11110 
11111 // Small ClearArray AVX512 non-constant length.
11112 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11113                        Universe dummy, rFlagsReg cr)
11114 %{
11115   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11116   match(Set dummy (ClearArray cnt base));
11117   ins_cost(125);
11118   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11119 
11120   format %{ $$template
11121     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11122     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11123     $$emit$$"jg      LARGE\n\t"
11124     $$emit$$"dec     rcx\n\t"
11125     $$emit$$"js      DONE\t# Zero length\n\t"
11126     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11127     $$emit$$"dec     rcx\n\t"
11128     $$emit$$"jge     LOOP\n\t"
11129     $$emit$$"jmp     DONE\n\t"
11130     $$emit$$"# LARGE:\n\t"
11131     if (UseFastStosb) {
11132        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11133        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11134     } else if (UseXMMForObjInit) {
11135        $$emit$$"mov     rdi,rax\n\t"
11136        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11137        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11138        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11139        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11140        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11141        $$emit$$"add     0x40,rax\n\t"
11142        $$emit$$"# L_zero_64_bytes:\n\t"
11143        $$emit$$"sub     0x8,rcx\n\t"
11144        $$emit$$"jge     L_loop\n\t"
11145        $$emit$$"add     0x4,rcx\n\t"
11146        $$emit$$"jl      L_tail\n\t"
11147        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11148        $$emit$$"add     0x20,rax\n\t"
11149        $$emit$$"sub     0x4,rcx\n\t"
11150        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11151        $$emit$$"add     0x4,rcx\n\t"
11152        $$emit$$"jle     L_end\n\t"
11153        $$emit$$"dec     rcx\n\t"
11154        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11155        $$emit$$"vmovq   xmm0,(rax)\n\t"
11156        $$emit$$"add     0x8,rax\n\t"
11157        $$emit$$"dec     rcx\n\t"
11158        $$emit$$"jge     L_sloop\n\t"
11159        $$emit$$"# L_end:\n\t"
11160     } else {
11161        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11162     }
11163     $$emit$$"# DONE"
11164   %}
11165   ins_encode %{
11166     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11167                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11168   %}
11169   ins_pipe(pipe_slow);
11170 %}
11171 
11172 // Large ClearArray non-AVX512.
11173 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11174                         Universe dummy, rFlagsReg cr)
11175 %{
11176   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11177   match(Set dummy (ClearArray cnt base));
11178   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11179 
11180   format %{ $$template
11181     if (UseFastStosb) {
11182        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11183        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11184        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11185     } else if (UseXMMForObjInit) {
11186        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11187        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11188        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11189        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11190        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11191        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11192        $$emit$$"add     0x40,rax\n\t"
11193        $$emit$$"# L_zero_64_bytes:\n\t"
11194        $$emit$$"sub     0x8,rcx\n\t"
11195        $$emit$$"jge     L_loop\n\t"
11196        $$emit$$"add     0x4,rcx\n\t"
11197        $$emit$$"jl      L_tail\n\t"
11198        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11199        $$emit$$"add     0x20,rax\n\t"
11200        $$emit$$"sub     0x4,rcx\n\t"
11201        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11202        $$emit$$"add     0x4,rcx\n\t"
11203        $$emit$$"jle     L_end\n\t"
11204        $$emit$$"dec     rcx\n\t"
11205        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11206        $$emit$$"vmovq   xmm0,(rax)\n\t"
11207        $$emit$$"add     0x8,rax\n\t"
11208        $$emit$$"dec     rcx\n\t"
11209        $$emit$$"jge     L_sloop\n\t"
11210        $$emit$$"# L_end:\n\t"
11211     } else {
11212        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11213        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11214     }
11215   %}
11216   ins_encode %{
11217     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11218                  $tmp$$XMMRegister, true, knoreg);
11219   %}
11220   ins_pipe(pipe_slow);
11221 %}
11222 
11223 // Large ClearArray AVX512.
11224 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11225                              Universe dummy, rFlagsReg cr)
11226 %{
11227   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11228   match(Set dummy (ClearArray cnt base));
11229   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11230 
11231   format %{ $$template
11232     if (UseFastStosb) {
11233        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11234        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11235        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11236     } else if (UseXMMForObjInit) {
11237        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11238        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11239        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11240        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11241        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11242        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11243        $$emit$$"add     0x40,rax\n\t"
11244        $$emit$$"# L_zero_64_bytes:\n\t"
11245        $$emit$$"sub     0x8,rcx\n\t"
11246        $$emit$$"jge     L_loop\n\t"
11247        $$emit$$"add     0x4,rcx\n\t"
11248        $$emit$$"jl      L_tail\n\t"
11249        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11250        $$emit$$"add     0x20,rax\n\t"
11251        $$emit$$"sub     0x4,rcx\n\t"
11252        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11253        $$emit$$"add     0x4,rcx\n\t"
11254        $$emit$$"jle     L_end\n\t"
11255        $$emit$$"dec     rcx\n\t"
11256        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11257        $$emit$$"vmovq   xmm0,(rax)\n\t"
11258        $$emit$$"add     0x8,rax\n\t"
11259        $$emit$$"dec     rcx\n\t"
11260        $$emit$$"jge     L_sloop\n\t"
11261        $$emit$$"# L_end:\n\t"
11262     } else {
11263        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11264        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11265     }
11266   %}
11267   ins_encode %{
11268     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11269                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11270   %}
11271   ins_pipe(pipe_slow);
11272 %}
11273 
11274 // Small ClearArray AVX512 constant length.
11275 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11276 %{
11277   predicate(!((ClearArrayNode*)n)->is_large() &&
11278               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11279   match(Set dummy (ClearArray cnt base));
11280   ins_cost(100);
11281   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11282   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11283   ins_encode %{
11284    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11285   %}
11286   ins_pipe(pipe_slow);
11287 %}
11288 
11289 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11290                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11291 %{
11292   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11293   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11294   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11295 
11296   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11297   ins_encode %{
11298     __ string_compare($str1$$Register, $str2$$Register,
11299                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11300                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11301   %}
11302   ins_pipe( pipe_slow );
11303 %}
11304 
11305 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11306                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11307 %{
11308   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11309   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11310   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11311 
11312   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11313   ins_encode %{
11314     __ string_compare($str1$$Register, $str2$$Register,
11315                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11316                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11317   %}
11318   ins_pipe( pipe_slow );
11319 %}
11320 
11321 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11322                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11323 %{
11324   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11325   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11326   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11327 
11328   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11329   ins_encode %{
11330     __ string_compare($str1$$Register, $str2$$Register,
11331                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11332                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11333   %}
11334   ins_pipe( pipe_slow );
11335 %}
11336 
11337 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11338                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11339 %{
11340   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11341   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11342   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11343 
11344   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11345   ins_encode %{
11346     __ string_compare($str1$$Register, $str2$$Register,
11347                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11348                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11349   %}
11350   ins_pipe( pipe_slow );
11351 %}
11352 
11353 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11354                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11355 %{
11356   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11357   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11358   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11359 
11360   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11361   ins_encode %{
11362     __ string_compare($str1$$Register, $str2$$Register,
11363                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11364                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11365   %}
11366   ins_pipe( pipe_slow );
11367 %}
11368 
11369 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11370                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11371 %{
11372   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11373   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11374   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11375 
11376   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11377   ins_encode %{
11378     __ string_compare($str1$$Register, $str2$$Register,
11379                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11380                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11381   %}
11382   ins_pipe( pipe_slow );
11383 %}
11384 
11385 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11386                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11387 %{
11388   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11389   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11390   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11391 
11392   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11393   ins_encode %{
11394     __ string_compare($str2$$Register, $str1$$Register,
11395                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11396                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11397   %}
11398   ins_pipe( pipe_slow );
11399 %}
11400 
11401 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11402                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11403 %{
11404   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11405   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11406   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11407 
11408   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11409   ins_encode %{
11410     __ string_compare($str2$$Register, $str1$$Register,
11411                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11412                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11413   %}
11414   ins_pipe( pipe_slow );
11415 %}
11416 
11417 // fast search of substring with known size.
11418 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11419                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11420 %{
11421   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11422   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11423   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11424 
11425   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11426   ins_encode %{
11427     int icnt2 = (int)$int_cnt2$$constant;
11428     if (icnt2 >= 16) {
11429       // IndexOf for constant substrings with size >= 16 elements
11430       // which don't need to be loaded through stack.
11431       __ string_indexofC8($str1$$Register, $str2$$Register,
11432                           $cnt1$$Register, $cnt2$$Register,
11433                           icnt2, $result$$Register,
11434                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11435     } else {
11436       // Small strings are loaded through stack if they cross page boundary.
11437       __ string_indexof($str1$$Register, $str2$$Register,
11438                         $cnt1$$Register, $cnt2$$Register,
11439                         icnt2, $result$$Register,
11440                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11441     }
11442   %}
11443   ins_pipe( pipe_slow );
11444 %}
11445 
11446 // fast search of substring with known size.
11447 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11448                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11449 %{
11450   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11451   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11452   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11453 
11454   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11455   ins_encode %{
11456     int icnt2 = (int)$int_cnt2$$constant;
11457     if (icnt2 >= 8) {
11458       // IndexOf for constant substrings with size >= 8 elements
11459       // which don't need to be loaded through stack.
11460       __ string_indexofC8($str1$$Register, $str2$$Register,
11461                           $cnt1$$Register, $cnt2$$Register,
11462                           icnt2, $result$$Register,
11463                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11464     } else {
11465       // Small strings are loaded through stack if they cross page boundary.
11466       __ string_indexof($str1$$Register, $str2$$Register,
11467                         $cnt1$$Register, $cnt2$$Register,
11468                         icnt2, $result$$Register,
11469                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11470     }
11471   %}
11472   ins_pipe( pipe_slow );
11473 %}
11474 
11475 // fast search of substring with known size.
11476 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11477                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11478 %{
11479   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11480   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11481   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11482 
11483   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11484   ins_encode %{
11485     int icnt2 = (int)$int_cnt2$$constant;
11486     if (icnt2 >= 8) {
11487       // IndexOf for constant substrings with size >= 8 elements
11488       // which don't need to be loaded through stack.
11489       __ string_indexofC8($str1$$Register, $str2$$Register,
11490                           $cnt1$$Register, $cnt2$$Register,
11491                           icnt2, $result$$Register,
11492                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11493     } else {
11494       // Small strings are loaded through stack if they cross page boundary.
11495       __ string_indexof($str1$$Register, $str2$$Register,
11496                         $cnt1$$Register, $cnt2$$Register,
11497                         icnt2, $result$$Register,
11498                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11499     }
11500   %}
11501   ins_pipe( pipe_slow );
11502 %}
11503 
11504 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11505                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11506 %{
11507   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11508   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11509   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11510 
11511   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11512   ins_encode %{
11513     __ string_indexof($str1$$Register, $str2$$Register,
11514                       $cnt1$$Register, $cnt2$$Register,
11515                       (-1), $result$$Register,
11516                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11517   %}
11518   ins_pipe( pipe_slow );
11519 %}
11520 
11521 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11522                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11523 %{
11524   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11525   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11526   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11527 
11528   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11529   ins_encode %{
11530     __ string_indexof($str1$$Register, $str2$$Register,
11531                       $cnt1$$Register, $cnt2$$Register,
11532                       (-1), $result$$Register,
11533                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11534   %}
11535   ins_pipe( pipe_slow );
11536 %}
11537 
11538 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11539                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11540 %{
11541   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11542   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11543   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11544 
11545   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11546   ins_encode %{
11547     __ string_indexof($str1$$Register, $str2$$Register,
11548                       $cnt1$$Register, $cnt2$$Register,
11549                       (-1), $result$$Register,
11550                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11551   %}
11552   ins_pipe( pipe_slow );
11553 %}
11554 
11555 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11556                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11557 %{
11558   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11559   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11560   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11561   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11562   ins_encode %{
11563     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11564                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11565   %}
11566   ins_pipe( pipe_slow );
11567 %}
11568 
11569 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11570                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11571 %{
11572   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11573   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11574   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11575   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11576   ins_encode %{
11577     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11578                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11579   %}
11580   ins_pipe( pipe_slow );
11581 %}
11582 
11583 // fast string equals
11584 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11585                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11586 %{
11587   predicate(!VM_Version::supports_avx512vlbw());
11588   match(Set result (StrEquals (Binary str1 str2) cnt));
11589   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11590 
11591   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11592   ins_encode %{
11593     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11594                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11595                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11596   %}
11597   ins_pipe( pipe_slow );
11598 %}
11599 
11600 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11601                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11602 %{
11603   predicate(VM_Version::supports_avx512vlbw());
11604   match(Set result (StrEquals (Binary str1 str2) cnt));
11605   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11606 
11607   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11608   ins_encode %{
11609     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11610                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11611                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11612   %}
11613   ins_pipe( pipe_slow );
11614 %}
11615 
11616 // fast array equals
11617 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11618                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11619 %{
11620   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11621   match(Set result (AryEq ary1 ary2));
11622   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11623 
11624   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11625   ins_encode %{
11626     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11627                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11628                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11629   %}
11630   ins_pipe( pipe_slow );
11631 %}
11632 
11633 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11634                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11635 %{
11636   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11637   match(Set result (AryEq ary1 ary2));
11638   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11639 
11640   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11641   ins_encode %{
11642     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11643                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11644                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11645   %}
11646   ins_pipe( pipe_slow );
11647 %}
11648 
11649 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11650                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11651 %{
11652   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11653   match(Set result (AryEq ary1 ary2));
11654   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11655 
11656   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11657   ins_encode %{
11658     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11659                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11660                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11661   %}
11662   ins_pipe( pipe_slow );
11663 %}
11664 
11665 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11666                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11667 %{
11668   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11669   match(Set result (AryEq ary1 ary2));
11670   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11671 
11672   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11673   ins_encode %{
11674     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11675                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11676                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11677   %}
11678   ins_pipe( pipe_slow );
11679 %}
11680 
11681 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11682                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11683 %{
11684   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11685   match(Set result (HasNegatives ary1 len));
11686   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11687 
11688   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11689   ins_encode %{
11690     __ has_negatives($ary1$$Register, $len$$Register,
11691                      $result$$Register, $tmp3$$Register,
11692                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11693   %}
11694   ins_pipe( pipe_slow );
11695 %}
11696 
11697 instruct has_negatives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11698                             legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11699 %{
11700   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11701   match(Set result (HasNegatives ary1 len));
11702   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11703 
11704   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11705   ins_encode %{
11706     __ has_negatives($ary1$$Register, $len$$Register,
11707                      $result$$Register, $tmp3$$Register,
11708                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11709   %}
11710   ins_pipe( pipe_slow );
11711 %}
11712 
11713 // fast char[] to byte[] compression
11714 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11715                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11716   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11717   match(Set result (StrCompressedCopy src (Binary dst len)));
11718   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11719          USE_KILL len, KILL tmp5, KILL cr);
11720 
11721   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11722   ins_encode %{
11723     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11724                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11725                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11726                            knoreg, knoreg);
11727   %}
11728   ins_pipe( pipe_slow );
11729 %}
11730 
11731 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11732                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11733   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11734   match(Set result (StrCompressedCopy src (Binary dst len)));
11735   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11736          USE_KILL len, KILL tmp5, KILL cr);
11737 
11738   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11739   ins_encode %{
11740     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11741                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11742                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11743                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11744   %}
11745   ins_pipe( pipe_slow );
11746 %}
11747 // fast byte[] to char[] inflation
11748 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11749                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11750   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11751   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11752   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11753 
11754   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11755   ins_encode %{
11756     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11757                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11758   %}
11759   ins_pipe( pipe_slow );
11760 %}
11761 
11762 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11763                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11764   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11765   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11766   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11767 
11768   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11769   ins_encode %{
11770     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11771                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11772   %}
11773   ins_pipe( pipe_slow );
11774 %}
11775 
11776 // encode char[] to byte[] in ISO_8859_1
11777 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11778                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11779                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11780   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11781   match(Set result (EncodeISOArray src (Binary dst len)));
11782   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11783 
11784   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11785   ins_encode %{
11786     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11787                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11788                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11789   %}
11790   ins_pipe( pipe_slow );
11791 %}
11792 
11793 // encode char[] to byte[] in ASCII
11794 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11795                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11796                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11797   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11798   match(Set result (EncodeISOArray src (Binary dst len)));
11799   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11800 
11801   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11802   ins_encode %{
11803     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11804                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11805                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11806   %}
11807   ins_pipe( pipe_slow );
11808 %}
11809 
11810 //----------Overflow Math Instructions-----------------------------------------
11811 
11812 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11813 %{
11814   match(Set cr (OverflowAddI op1 op2));
11815   effect(DEF cr, USE_KILL op1, USE op2);
11816 
11817   format %{ "addl    $op1, $op2\t# overflow check int" %}
11818 
11819   ins_encode %{
11820     __ addl($op1$$Register, $op2$$Register);
11821   %}
11822   ins_pipe(ialu_reg_reg);
11823 %}
11824 
11825 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11826 %{
11827   match(Set cr (OverflowAddI op1 op2));
11828   effect(DEF cr, USE_KILL op1, USE op2);
11829 
11830   format %{ "addl    $op1, $op2\t# overflow check int" %}
11831 
11832   ins_encode %{
11833     __ addl($op1$$Register, $op2$$constant);
11834   %}
11835   ins_pipe(ialu_reg_reg);
11836 %}
11837 
11838 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11839 %{
11840   match(Set cr (OverflowAddL op1 op2));
11841   effect(DEF cr, USE_KILL op1, USE op2);
11842 
11843   format %{ "addq    $op1, $op2\t# overflow check long" %}
11844   ins_encode %{
11845     __ addq($op1$$Register, $op2$$Register);
11846   %}
11847   ins_pipe(ialu_reg_reg);
11848 %}
11849 
11850 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11851 %{
11852   match(Set cr (OverflowAddL op1 op2));
11853   effect(DEF cr, USE_KILL op1, USE op2);
11854 
11855   format %{ "addq    $op1, $op2\t# overflow check long" %}
11856   ins_encode %{
11857     __ addq($op1$$Register, $op2$$constant);
11858   %}
11859   ins_pipe(ialu_reg_reg);
11860 %}
11861 
11862 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11863 %{
11864   match(Set cr (OverflowSubI op1 op2));
11865 
11866   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11867   ins_encode %{
11868     __ cmpl($op1$$Register, $op2$$Register);
11869   %}
11870   ins_pipe(ialu_reg_reg);
11871 %}
11872 
11873 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11874 %{
11875   match(Set cr (OverflowSubI op1 op2));
11876 
11877   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11878   ins_encode %{
11879     __ cmpl($op1$$Register, $op2$$constant);
11880   %}
11881   ins_pipe(ialu_reg_reg);
11882 %}
11883 
11884 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11885 %{
11886   match(Set cr (OverflowSubL op1 op2));
11887 
11888   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11889   ins_encode %{
11890     __ cmpq($op1$$Register, $op2$$Register);
11891   %}
11892   ins_pipe(ialu_reg_reg);
11893 %}
11894 
11895 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11896 %{
11897   match(Set cr (OverflowSubL op1 op2));
11898 
11899   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11900   ins_encode %{
11901     __ cmpq($op1$$Register, $op2$$constant);
11902   %}
11903   ins_pipe(ialu_reg_reg);
11904 %}
11905 
11906 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11907 %{
11908   match(Set cr (OverflowSubI zero op2));
11909   effect(DEF cr, USE_KILL op2);
11910 
11911   format %{ "negl    $op2\t# overflow check int" %}
11912   ins_encode %{
11913     __ negl($op2$$Register);
11914   %}
11915   ins_pipe(ialu_reg_reg);
11916 %}
11917 
11918 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11919 %{
11920   match(Set cr (OverflowSubL zero op2));
11921   effect(DEF cr, USE_KILL op2);
11922 
11923   format %{ "negq    $op2\t# overflow check long" %}
11924   ins_encode %{
11925     __ negq($op2$$Register);
11926   %}
11927   ins_pipe(ialu_reg_reg);
11928 %}
11929 
11930 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11931 %{
11932   match(Set cr (OverflowMulI op1 op2));
11933   effect(DEF cr, USE_KILL op1, USE op2);
11934 
11935   format %{ "imull    $op1, $op2\t# overflow check int" %}
11936   ins_encode %{
11937     __ imull($op1$$Register, $op2$$Register);
11938   %}
11939   ins_pipe(ialu_reg_reg_alu0);
11940 %}
11941 
11942 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11943 %{
11944   match(Set cr (OverflowMulI op1 op2));
11945   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11946 
11947   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11948   ins_encode %{
11949     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11950   %}
11951   ins_pipe(ialu_reg_reg_alu0);
11952 %}
11953 
11954 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11955 %{
11956   match(Set cr (OverflowMulL op1 op2));
11957   effect(DEF cr, USE_KILL op1, USE op2);
11958 
11959   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11960   ins_encode %{
11961     __ imulq($op1$$Register, $op2$$Register);
11962   %}
11963   ins_pipe(ialu_reg_reg_alu0);
11964 %}
11965 
11966 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11967 %{
11968   match(Set cr (OverflowMulL op1 op2));
11969   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11970 
11971   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11972   ins_encode %{
11973     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11974   %}
11975   ins_pipe(ialu_reg_reg_alu0);
11976 %}
11977 
11978 
11979 //----------Control Flow Instructions------------------------------------------
11980 // Signed compare Instructions
11981 
11982 // XXX more variants!!
11983 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11984 %{
11985   match(Set cr (CmpI op1 op2));
11986   effect(DEF cr, USE op1, USE op2);
11987 
11988   format %{ "cmpl    $op1, $op2" %}
11989   ins_encode %{
11990     __ cmpl($op1$$Register, $op2$$Register);
11991   %}
11992   ins_pipe(ialu_cr_reg_reg);
11993 %}
11994 
11995 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11996 %{
11997   match(Set cr (CmpI op1 op2));
11998 
11999   format %{ "cmpl    $op1, $op2" %}
12000   ins_encode %{
12001     __ cmpl($op1$$Register, $op2$$constant);
12002   %}
12003   ins_pipe(ialu_cr_reg_imm);
12004 %}
12005 
12006 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12007 %{
12008   match(Set cr (CmpI op1 (LoadI op2)));
12009 
12010   ins_cost(500); // XXX
12011   format %{ "cmpl    $op1, $op2" %}
12012   ins_encode %{
12013     __ cmpl($op1$$Register, $op2$$Address);
12014   %}
12015   ins_pipe(ialu_cr_reg_mem);
12016 %}
12017 
12018 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12019 %{
12020   match(Set cr (CmpI src zero));
12021 
12022   format %{ "testl   $src, $src" %}
12023   ins_encode %{
12024     __ testl($src$$Register, $src$$Register);
12025   %}
12026   ins_pipe(ialu_cr_reg_imm);
12027 %}
12028 
12029 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12030 %{
12031   match(Set cr (CmpI (AndI src con) zero));
12032 
12033   format %{ "testl   $src, $con" %}
12034   ins_encode %{
12035     __ testl($src$$Register, $con$$constant);
12036   %}
12037   ins_pipe(ialu_cr_reg_imm);
12038 %}
12039 
12040 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12041 %{
12042   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12043 
12044   format %{ "testl   $src, $mem" %}
12045   ins_encode %{
12046     __ testl($src$$Register, $mem$$Address);
12047   %}
12048   ins_pipe(ialu_cr_reg_mem);
12049 %}
12050 
12051 // Unsigned compare Instructions; really, same as signed except they
12052 // produce an rFlagsRegU instead of rFlagsReg.
12053 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12054 %{
12055   match(Set cr (CmpU op1 op2));
12056 
12057   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12058   ins_encode %{
12059     __ cmpl($op1$$Register, $op2$$Register);
12060   %}
12061   ins_pipe(ialu_cr_reg_reg);
12062 %}
12063 
12064 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12065 %{
12066   match(Set cr (CmpU op1 op2));
12067 
12068   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12069   ins_encode %{
12070     __ cmpl($op1$$Register, $op2$$constant);
12071   %}
12072   ins_pipe(ialu_cr_reg_imm);
12073 %}
12074 
12075 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12076 %{
12077   match(Set cr (CmpU op1 (LoadI op2)));
12078 
12079   ins_cost(500); // XXX
12080   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12081   ins_encode %{
12082     __ cmpl($op1$$Register, $op2$$Address);
12083   %}
12084   ins_pipe(ialu_cr_reg_mem);
12085 %}
12086 
12087 // // // Cisc-spilled version of cmpU_rReg
12088 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12089 // //%{
12090 // //  match(Set cr (CmpU (LoadI op1) op2));
12091 // //
12092 // //  format %{ "CMPu   $op1,$op2" %}
12093 // //  ins_cost(500);
12094 // //  opcode(0x39);  /* Opcode 39 /r */
12095 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12096 // //%}
12097 
12098 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12099 %{
12100   match(Set cr (CmpU src zero));
12101 
12102   format %{ "testl   $src, $src\t# unsigned" %}
12103   ins_encode %{
12104     __ testl($src$$Register, $src$$Register);
12105   %}
12106   ins_pipe(ialu_cr_reg_imm);
12107 %}
12108 
12109 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12110 %{
12111   match(Set cr (CmpP op1 op2));
12112 
12113   format %{ "cmpq    $op1, $op2\t# ptr" %}
12114   ins_encode %{
12115     __ cmpq($op1$$Register, $op2$$Register);
12116   %}
12117   ins_pipe(ialu_cr_reg_reg);
12118 %}
12119 
12120 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12121 %{
12122   match(Set cr (CmpP op1 (LoadP op2)));
12123   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12124 
12125   ins_cost(500); // XXX
12126   format %{ "cmpq    $op1, $op2\t# ptr" %}
12127   ins_encode %{
12128     __ cmpq($op1$$Register, $op2$$Address);
12129   %}
12130   ins_pipe(ialu_cr_reg_mem);
12131 %}
12132 
12133 // // // Cisc-spilled version of cmpP_rReg
12134 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12135 // //%{
12136 // //  match(Set cr (CmpP (LoadP op1) op2));
12137 // //
12138 // //  format %{ "CMPu   $op1,$op2" %}
12139 // //  ins_cost(500);
12140 // //  opcode(0x39);  /* Opcode 39 /r */
12141 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12142 // //%}
12143 
12144 // XXX this is generalized by compP_rReg_mem???
12145 // Compare raw pointer (used in out-of-heap check).
12146 // Only works because non-oop pointers must be raw pointers
12147 // and raw pointers have no anti-dependencies.
12148 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12149 %{
12150   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12151             n->in(2)->as_Load()->barrier_data() == 0);
12152   match(Set cr (CmpP op1 (LoadP op2)));
12153 
12154   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12155   ins_encode %{
12156     __ cmpq($op1$$Register, $op2$$Address);
12157   %}
12158   ins_pipe(ialu_cr_reg_mem);
12159 %}
12160 
12161 // This will generate a signed flags result. This should be OK since
12162 // any compare to a zero should be eq/neq.
12163 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12164 %{
12165   match(Set cr (CmpP src zero));
12166 
12167   format %{ "testq   $src, $src\t# ptr" %}
12168   ins_encode %{
12169     __ testq($src$$Register, $src$$Register);
12170   %}
12171   ins_pipe(ialu_cr_reg_imm);
12172 %}
12173 
12174 // This will generate a signed flags result. This should be OK since
12175 // any compare to a zero should be eq/neq.
12176 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12177 %{
12178   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12179             n->in(1)->as_Load()->barrier_data() == 0);
12180   match(Set cr (CmpP (LoadP op) zero));
12181 
12182   ins_cost(500); // XXX
12183   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12184   ins_encode %{
12185     __ testq($op$$Address, 0xFFFFFFFF);
12186   %}
12187   ins_pipe(ialu_cr_reg_imm);
12188 %}
12189 
12190 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12191 %{
12192   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12193             n->in(1)->as_Load()->barrier_data() == 0);
12194   match(Set cr (CmpP (LoadP mem) zero));
12195 
12196   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12197   ins_encode %{
12198     __ cmpq(r12, $mem$$Address);
12199   %}
12200   ins_pipe(ialu_cr_reg_mem);
12201 %}
12202 
12203 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12204 %{
12205   match(Set cr (CmpN op1 op2));
12206 
12207   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12208   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12209   ins_pipe(ialu_cr_reg_reg);
12210 %}
12211 
12212 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12213 %{
12214   match(Set cr (CmpN src (LoadN mem)));
12215 
12216   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12217   ins_encode %{
12218     __ cmpl($src$$Register, $mem$$Address);
12219   %}
12220   ins_pipe(ialu_cr_reg_mem);
12221 %}
12222 
12223 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12224   match(Set cr (CmpN op1 op2));
12225 
12226   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12227   ins_encode %{
12228     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12229   %}
12230   ins_pipe(ialu_cr_reg_imm);
12231 %}
12232 
12233 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12234 %{
12235   match(Set cr (CmpN src (LoadN mem)));
12236 
12237   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12238   ins_encode %{
12239     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12240   %}
12241   ins_pipe(ialu_cr_reg_mem);
12242 %}
12243 
12244 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12245   match(Set cr (CmpN op1 op2));
12246 
12247   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12248   ins_encode %{
12249     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12250   %}
12251   ins_pipe(ialu_cr_reg_imm);
12252 %}
12253 
12254 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12255 %{
12256   match(Set cr (CmpN src (LoadNKlass mem)));
12257 
12258   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12259   ins_encode %{
12260     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12261   %}
12262   ins_pipe(ialu_cr_reg_mem);
12263 %}
12264 
12265 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12266   match(Set cr (CmpN src zero));
12267 
12268   format %{ "testl   $src, $src\t# compressed ptr" %}
12269   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12270   ins_pipe(ialu_cr_reg_imm);
12271 %}
12272 
12273 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12274 %{
12275   predicate(CompressedOops::base() != NULL);
12276   match(Set cr (CmpN (LoadN mem) zero));
12277 
12278   ins_cost(500); // XXX
12279   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12280   ins_encode %{
12281     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12282   %}
12283   ins_pipe(ialu_cr_reg_mem);
12284 %}
12285 
12286 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12287 %{
12288   predicate(CompressedOops::base() == NULL);
12289   match(Set cr (CmpN (LoadN mem) zero));
12290 
12291   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12292   ins_encode %{
12293     __ cmpl(r12, $mem$$Address);
12294   %}
12295   ins_pipe(ialu_cr_reg_mem);
12296 %}
12297 
12298 // Yanked all unsigned pointer compare operations.
12299 // Pointer compares are done with CmpP which is already unsigned.
12300 
12301 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12302 %{
12303   match(Set cr (CmpL op1 op2));
12304 
12305   format %{ "cmpq    $op1, $op2" %}
12306   ins_encode %{
12307     __ cmpq($op1$$Register, $op2$$Register);
12308   %}
12309   ins_pipe(ialu_cr_reg_reg);
12310 %}
12311 
12312 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12313 %{
12314   match(Set cr (CmpL op1 op2));
12315 
12316   format %{ "cmpq    $op1, $op2" %}
12317   ins_encode %{
12318     __ cmpq($op1$$Register, $op2$$constant);
12319   %}
12320   ins_pipe(ialu_cr_reg_imm);
12321 %}
12322 
12323 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12324 %{
12325   match(Set cr (CmpL op1 (LoadL op2)));
12326 
12327   format %{ "cmpq    $op1, $op2" %}
12328   ins_encode %{
12329     __ cmpq($op1$$Register, $op2$$Address);
12330   %}
12331   ins_pipe(ialu_cr_reg_mem);
12332 %}
12333 
12334 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12335 %{
12336   match(Set cr (CmpL src zero));
12337 
12338   format %{ "testq   $src, $src" %}
12339   ins_encode %{
12340     __ testq($src$$Register, $src$$Register);
12341   %}
12342   ins_pipe(ialu_cr_reg_imm);
12343 %}
12344 
12345 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12346 %{
12347   match(Set cr (CmpL (AndL src con) zero));
12348 
12349   format %{ "testq   $src, $con\t# long" %}
12350   ins_encode %{
12351     __ testq($src$$Register, $con$$constant);
12352   %}
12353   ins_pipe(ialu_cr_reg_imm);
12354 %}
12355 
12356 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12357 %{
12358   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12359 
12360   format %{ "testq   $src, $mem" %}
12361   ins_encode %{
12362     __ testq($src$$Register, $mem$$Address);
12363   %}
12364   ins_pipe(ialu_cr_reg_mem);
12365 %}
12366 
12367 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12368 %{
12369   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12370 
12371   format %{ "testq   $src, $mem" %}
12372   ins_encode %{
12373     __ testq($src$$Register, $mem$$Address);
12374   %}
12375   ins_pipe(ialu_cr_reg_mem);
12376 %}
12377 
12378 // Manifest a CmpL result in an integer register.  Very painful.
12379 // This is the test to avoid.
12380 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12381 %{
12382   match(Set dst (CmpL3 src1 src2));
12383   effect(KILL flags);
12384 
12385   ins_cost(275); // XXX
12386   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12387             "movl    $dst, -1\n\t"
12388             "jl,s    done\n\t"
12389             "setne   $dst\n\t"
12390             "movzbl  $dst, $dst\n\t"
12391     "done:" %}
12392   ins_encode %{
12393     Label done;
12394     __ cmpq($src1$$Register, $src2$$Register);
12395     __ movl($dst$$Register, -1);
12396     __ jccb(Assembler::less, done);
12397     __ setne($dst$$Register);
12398     __ movzbl($dst$$Register, $dst$$Register);
12399     __ bind(done);
12400   %}
12401   ins_pipe(pipe_slow);
12402 %}
12403 
12404 // Unsigned long compare Instructions; really, same as signed long except they
12405 // produce an rFlagsRegU instead of rFlagsReg.
12406 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12407 %{
12408   match(Set cr (CmpUL op1 op2));
12409 
12410   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12411   ins_encode %{
12412     __ cmpq($op1$$Register, $op2$$Register);
12413   %}
12414   ins_pipe(ialu_cr_reg_reg);
12415 %}
12416 
12417 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12418 %{
12419   match(Set cr (CmpUL op1 op2));
12420 
12421   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12422   ins_encode %{
12423     __ cmpq($op1$$Register, $op2$$constant);
12424   %}
12425   ins_pipe(ialu_cr_reg_imm);
12426 %}
12427 
12428 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12429 %{
12430   match(Set cr (CmpUL op1 (LoadL op2)));
12431 
12432   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12433   ins_encode %{
12434     __ cmpq($op1$$Register, $op2$$Address);
12435   %}
12436   ins_pipe(ialu_cr_reg_mem);
12437 %}
12438 
12439 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12440 %{
12441   match(Set cr (CmpUL src zero));
12442 
12443   format %{ "testq   $src, $src\t# unsigned" %}
12444   ins_encode %{
12445     __ testq($src$$Register, $src$$Register);
12446   %}
12447   ins_pipe(ialu_cr_reg_imm);
12448 %}
12449 
12450 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12451 %{
12452   match(Set cr (CmpI (LoadB mem) imm));
12453 
12454   ins_cost(125);
12455   format %{ "cmpb    $mem, $imm" %}
12456   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12457   ins_pipe(ialu_cr_reg_mem);
12458 %}
12459 
12460 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12461 %{
12462   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12463 
12464   ins_cost(125);
12465   format %{ "testb   $mem, $imm\t# ubyte" %}
12466   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12467   ins_pipe(ialu_cr_reg_mem);
12468 %}
12469 
12470 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12471 %{
12472   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12473 
12474   ins_cost(125);
12475   format %{ "testb   $mem, $imm\t# byte" %}
12476   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12477   ins_pipe(ialu_cr_reg_mem);
12478 %}
12479 
12480 //----------Max and Min--------------------------------------------------------
12481 // Min Instructions
12482 
12483 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12484 %{
12485   effect(USE_DEF dst, USE src, USE cr);
12486 
12487   format %{ "cmovlgt $dst, $src\t# min" %}
12488   ins_encode %{
12489     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12490   %}
12491   ins_pipe(pipe_cmov_reg);
12492 %}
12493 
12494 
12495 instruct minI_rReg(rRegI dst, rRegI src)
12496 %{
12497   match(Set dst (MinI dst src));
12498 
12499   ins_cost(200);
12500   expand %{
12501     rFlagsReg cr;
12502     compI_rReg(cr, dst, src);
12503     cmovI_reg_g(dst, src, cr);
12504   %}
12505 %}
12506 
12507 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12508 %{
12509   effect(USE_DEF dst, USE src, USE cr);
12510 
12511   format %{ "cmovllt $dst, $src\t# max" %}
12512   ins_encode %{
12513     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12514   %}
12515   ins_pipe(pipe_cmov_reg);
12516 %}
12517 
12518 
12519 instruct maxI_rReg(rRegI dst, rRegI src)
12520 %{
12521   match(Set dst (MaxI dst src));
12522 
12523   ins_cost(200);
12524   expand %{
12525     rFlagsReg cr;
12526     compI_rReg(cr, dst, src);
12527     cmovI_reg_l(dst, src, cr);
12528   %}
12529 %}
12530 
12531 // ============================================================================
12532 // Branch Instructions
12533 
12534 // Jump Direct - Label defines a relative address from JMP+1
12535 instruct jmpDir(label labl)
12536 %{
12537   match(Goto);
12538   effect(USE labl);
12539 
12540   ins_cost(300);
12541   format %{ "jmp     $labl" %}
12542   size(5);
12543   ins_encode %{
12544     Label* L = $labl$$label;
12545     __ jmp(*L, false); // Always long jump
12546   %}
12547   ins_pipe(pipe_jmp);
12548 %}
12549 
12550 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12551 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12552 %{
12553   match(If cop cr);
12554   effect(USE labl);
12555 
12556   ins_cost(300);
12557   format %{ "j$cop     $labl" %}
12558   size(6);
12559   ins_encode %{
12560     Label* L = $labl$$label;
12561     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12562   %}
12563   ins_pipe(pipe_jcc);
12564 %}
12565 
12566 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12567 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12568 %{
12569   predicate(!n->has_vector_mask_set());
12570   match(CountedLoopEnd cop cr);
12571   effect(USE labl);
12572 
12573   ins_cost(300);
12574   format %{ "j$cop     $labl\t# loop end" %}
12575   size(6);
12576   ins_encode %{
12577     Label* L = $labl$$label;
12578     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12579   %}
12580   ins_pipe(pipe_jcc);
12581 %}
12582 
12583 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12584 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12585   predicate(!n->has_vector_mask_set());
12586   match(CountedLoopEnd cop cmp);
12587   effect(USE labl);
12588 
12589   ins_cost(300);
12590   format %{ "j$cop,u   $labl\t# loop end" %}
12591   size(6);
12592   ins_encode %{
12593     Label* L = $labl$$label;
12594     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12595   %}
12596   ins_pipe(pipe_jcc);
12597 %}
12598 
12599 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12600   predicate(!n->has_vector_mask_set());
12601   match(CountedLoopEnd cop cmp);
12602   effect(USE labl);
12603 
12604   ins_cost(200);
12605   format %{ "j$cop,u   $labl\t# loop end" %}
12606   size(6);
12607   ins_encode %{
12608     Label* L = $labl$$label;
12609     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12610   %}
12611   ins_pipe(pipe_jcc);
12612 %}
12613 
12614 // mask version
12615 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12616 // Bounded mask operand used in following patten is needed for
12617 // post-loop multiversioning.
12618 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, kReg_K1 ktmp, rFlagsReg cr, label labl)
12619 %{
12620   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12621   match(CountedLoopEnd cop cr);
12622   effect(USE labl, TEMP ktmp);
12623 
12624   ins_cost(400);
12625   format %{ "j$cop     $labl\t# loop end\n\t"
12626             "restorevectmask \t# vector mask restore for loops" %}
12627   size(10);
12628   ins_encode %{
12629     Label* L = $labl$$label;
12630     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12631     __ restorevectmask($ktmp$$KRegister);
12632   %}
12633   ins_pipe(pipe_jcc);
12634 %}
12635 
12636 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12637 // Bounded mask operand used in following patten is needed for
12638 // post-loop multiversioning.
12639 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, kReg_K1 ktmp, rFlagsRegU cmp, label labl) %{
12640   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12641   match(CountedLoopEnd cop cmp);
12642   effect(USE labl, TEMP ktmp);
12643 
12644   ins_cost(400);
12645   format %{ "j$cop,u   $labl\t# loop end\n\t"
12646             "restorevectmask \t# vector mask restore for loops" %}
12647   size(10);
12648   ins_encode %{
12649     Label* L = $labl$$label;
12650     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12651     __ restorevectmask($ktmp$$KRegister);
12652   %}
12653   ins_pipe(pipe_jcc);
12654 %}
12655 
12656 // Bounded mask operand used in following patten is needed for
12657 // post-loop multiversioning.
12658 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, kReg_K1 ktmp, rFlagsRegUCF cmp, label labl) %{
12659   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12660   match(CountedLoopEnd cop cmp);
12661   effect(USE labl, TEMP ktmp);
12662 
12663   ins_cost(300);
12664   format %{ "j$cop,u   $labl\t# loop end\n\t"
12665             "restorevectmask \t# vector mask restore for loops" %}
12666   size(10);
12667   ins_encode %{
12668     Label* L = $labl$$label;
12669     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12670     __ restorevectmask($ktmp$$KRegister);
12671   %}
12672   ins_pipe(pipe_jcc);
12673 %}
12674 
12675 // Jump Direct Conditional - using unsigned comparison
12676 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12677   match(If cop cmp);
12678   effect(USE labl);
12679 
12680   ins_cost(300);
12681   format %{ "j$cop,u   $labl" %}
12682   size(6);
12683   ins_encode %{
12684     Label* L = $labl$$label;
12685     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12686   %}
12687   ins_pipe(pipe_jcc);
12688 %}
12689 
12690 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12691   match(If cop cmp);
12692   effect(USE labl);
12693 
12694   ins_cost(200);
12695   format %{ "j$cop,u   $labl" %}
12696   size(6);
12697   ins_encode %{
12698     Label* L = $labl$$label;
12699     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12700   %}
12701   ins_pipe(pipe_jcc);
12702 %}
12703 
12704 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12705   match(If cop cmp);
12706   effect(USE labl);
12707 
12708   ins_cost(200);
12709   format %{ $$template
12710     if ($cop$$cmpcode == Assembler::notEqual) {
12711       $$emit$$"jp,u    $labl\n\t"
12712       $$emit$$"j$cop,u   $labl"
12713     } else {
12714       $$emit$$"jp,u    done\n\t"
12715       $$emit$$"j$cop,u   $labl\n\t"
12716       $$emit$$"done:"
12717     }
12718   %}
12719   ins_encode %{
12720     Label* l = $labl$$label;
12721     if ($cop$$cmpcode == Assembler::notEqual) {
12722       __ jcc(Assembler::parity, *l, false);
12723       __ jcc(Assembler::notEqual, *l, false);
12724     } else if ($cop$$cmpcode == Assembler::equal) {
12725       Label done;
12726       __ jccb(Assembler::parity, done);
12727       __ jcc(Assembler::equal, *l, false);
12728       __ bind(done);
12729     } else {
12730        ShouldNotReachHere();
12731     }
12732   %}
12733   ins_pipe(pipe_jcc);
12734 %}
12735 
12736 // ============================================================================
12737 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12738 // superklass array for an instance of the superklass.  Set a hidden
12739 // internal cache on a hit (cache is checked with exposed code in
12740 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12741 // encoding ALSO sets flags.
12742 
12743 instruct partialSubtypeCheck(rdi_RegP result,
12744                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12745                              rFlagsReg cr)
12746 %{
12747   match(Set result (PartialSubtypeCheck sub super));
12748   effect(KILL rcx, KILL cr);
12749 
12750   ins_cost(1100);  // slightly larger than the next version
12751   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12752             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12753             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12754             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12755             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12756             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12757             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12758     "miss:\t" %}
12759 
12760   opcode(0x1); // Force a XOR of RDI
12761   ins_encode(enc_PartialSubtypeCheck());
12762   ins_pipe(pipe_slow);
12763 %}
12764 
12765 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12766                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12767                                      immP0 zero,
12768                                      rdi_RegP result)
12769 %{
12770   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12771   effect(KILL rcx, KILL result);
12772 
12773   ins_cost(1000);
12774   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12775             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12776             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12777             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12778             "jne,s   miss\t\t# Missed: flags nz\n\t"
12779             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12780     "miss:\t" %}
12781 
12782   opcode(0x0); // No need to XOR RDI
12783   ins_encode(enc_PartialSubtypeCheck());
12784   ins_pipe(pipe_slow);
12785 %}
12786 
12787 // ============================================================================
12788 // Branch Instructions -- short offset versions
12789 //
12790 // These instructions are used to replace jumps of a long offset (the default
12791 // match) with jumps of a shorter offset.  These instructions are all tagged
12792 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12793 // match rules in general matching.  Instead, the ADLC generates a conversion
12794 // method in the MachNode which can be used to do in-place replacement of the
12795 // long variant with the shorter variant.  The compiler will determine if a
12796 // branch can be taken by the is_short_branch_offset() predicate in the machine
12797 // specific code section of the file.
12798 
12799 // Jump Direct - Label defines a relative address from JMP+1
12800 instruct jmpDir_short(label labl) %{
12801   match(Goto);
12802   effect(USE labl);
12803 
12804   ins_cost(300);
12805   format %{ "jmp,s   $labl" %}
12806   size(2);
12807   ins_encode %{
12808     Label* L = $labl$$label;
12809     __ jmpb(*L);
12810   %}
12811   ins_pipe(pipe_jmp);
12812   ins_short_branch(1);
12813 %}
12814 
12815 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12816 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12817   match(If cop cr);
12818   effect(USE labl);
12819 
12820   ins_cost(300);
12821   format %{ "j$cop,s   $labl" %}
12822   size(2);
12823   ins_encode %{
12824     Label* L = $labl$$label;
12825     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12826   %}
12827   ins_pipe(pipe_jcc);
12828   ins_short_branch(1);
12829 %}
12830 
12831 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12832 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12833   match(CountedLoopEnd cop cr);
12834   effect(USE labl);
12835 
12836   ins_cost(300);
12837   format %{ "j$cop,s   $labl\t# loop end" %}
12838   size(2);
12839   ins_encode %{
12840     Label* L = $labl$$label;
12841     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12842   %}
12843   ins_pipe(pipe_jcc);
12844   ins_short_branch(1);
12845 %}
12846 
12847 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12848 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12849   match(CountedLoopEnd cop cmp);
12850   effect(USE labl);
12851 
12852   ins_cost(300);
12853   format %{ "j$cop,us  $labl\t# loop end" %}
12854   size(2);
12855   ins_encode %{
12856     Label* L = $labl$$label;
12857     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12858   %}
12859   ins_pipe(pipe_jcc);
12860   ins_short_branch(1);
12861 %}
12862 
12863 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12864   match(CountedLoopEnd cop cmp);
12865   effect(USE labl);
12866 
12867   ins_cost(300);
12868   format %{ "j$cop,us  $labl\t# loop end" %}
12869   size(2);
12870   ins_encode %{
12871     Label* L = $labl$$label;
12872     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12873   %}
12874   ins_pipe(pipe_jcc);
12875   ins_short_branch(1);
12876 %}
12877 
12878 // Jump Direct Conditional - using unsigned comparison
12879 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12880   match(If cop cmp);
12881   effect(USE labl);
12882 
12883   ins_cost(300);
12884   format %{ "j$cop,us  $labl" %}
12885   size(2);
12886   ins_encode %{
12887     Label* L = $labl$$label;
12888     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12889   %}
12890   ins_pipe(pipe_jcc);
12891   ins_short_branch(1);
12892 %}
12893 
12894 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12895   match(If cop cmp);
12896   effect(USE labl);
12897 
12898   ins_cost(300);
12899   format %{ "j$cop,us  $labl" %}
12900   size(2);
12901   ins_encode %{
12902     Label* L = $labl$$label;
12903     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12904   %}
12905   ins_pipe(pipe_jcc);
12906   ins_short_branch(1);
12907 %}
12908 
12909 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12910   match(If cop cmp);
12911   effect(USE labl);
12912 
12913   ins_cost(300);
12914   format %{ $$template
12915     if ($cop$$cmpcode == Assembler::notEqual) {
12916       $$emit$$"jp,u,s  $labl\n\t"
12917       $$emit$$"j$cop,u,s  $labl"
12918     } else {
12919       $$emit$$"jp,u,s  done\n\t"
12920       $$emit$$"j$cop,u,s  $labl\n\t"
12921       $$emit$$"done:"
12922     }
12923   %}
12924   size(4);
12925   ins_encode %{
12926     Label* l = $labl$$label;
12927     if ($cop$$cmpcode == Assembler::notEqual) {
12928       __ jccb(Assembler::parity, *l);
12929       __ jccb(Assembler::notEqual, *l);
12930     } else if ($cop$$cmpcode == Assembler::equal) {
12931       Label done;
12932       __ jccb(Assembler::parity, done);
12933       __ jccb(Assembler::equal, *l);
12934       __ bind(done);
12935     } else {
12936        ShouldNotReachHere();
12937     }
12938   %}
12939   ins_pipe(pipe_jcc);
12940   ins_short_branch(1);
12941 %}
12942 
12943 // ============================================================================
12944 // inlined locking and unlocking
12945 
12946 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12947   predicate(Compile::current()->use_rtm());
12948   match(Set cr (FastLock object box));
12949   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12950   ins_cost(300);
12951   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12952   ins_encode %{
12953     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12954                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12955                  _rtm_counters, _stack_rtm_counters,
12956                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12957                  true, ra_->C->profile_rtm());
12958   %}
12959   ins_pipe(pipe_slow);
12960 %}
12961 
12962 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
12963   predicate(!Compile::current()->use_rtm());
12964   match(Set cr (FastLock object box));
12965   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
12966   ins_cost(300);
12967   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12968   ins_encode %{
12969     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12970                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
12971   %}
12972   ins_pipe(pipe_slow);
12973 %}
12974 
12975 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12976   match(Set cr (FastUnlock object box));
12977   effect(TEMP tmp, USE_KILL box);
12978   ins_cost(300);
12979   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12980   ins_encode %{
12981     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12982   %}
12983   ins_pipe(pipe_slow);
12984 %}
12985 
12986 
12987 // ============================================================================
12988 // Safepoint Instructions
12989 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12990 %{
12991   match(SafePoint poll);
12992   effect(KILL cr, USE poll);
12993 
12994   format %{ "testl   rax, [$poll]\t"
12995             "# Safepoint: poll for GC" %}
12996   ins_cost(125);
12997   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12998   ins_encode %{
12999     __ relocate(relocInfo::poll_type);
13000     address pre_pc = __ pc();
13001     __ testl(rax, Address($poll$$Register, 0));
13002     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13003   %}
13004   ins_pipe(ialu_reg_mem);
13005 %}
13006 
13007 // ============================================================================
13008 // Procedure Call/Return Instructions
13009 // Call Java Static Instruction
13010 // Note: If this code changes, the corresponding ret_addr_offset() and
13011 //       compute_padding() functions will have to be adjusted.
13012 instruct CallStaticJavaDirect(method meth) %{
13013   match(CallStaticJava);
13014   effect(USE meth);
13015 
13016   ins_cost(300);
13017   format %{ "call,static " %}
13018   opcode(0xE8); /* E8 cd */
13019   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13020   ins_pipe(pipe_slow);
13021   ins_alignment(4);
13022 %}
13023 
13024 // Call Java Dynamic Instruction
13025 // Note: If this code changes, the corresponding ret_addr_offset() and
13026 //       compute_padding() functions will have to be adjusted.
13027 instruct CallDynamicJavaDirect(method meth)
13028 %{
13029   match(CallDynamicJava);
13030   effect(USE meth);
13031 
13032   ins_cost(300);
13033   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13034             "call,dynamic " %}
13035   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13036   ins_pipe(pipe_slow);
13037   ins_alignment(4);
13038 %}
13039 
13040 // Call Runtime Instruction
13041 instruct CallRuntimeDirect(method meth)
13042 %{
13043   match(CallRuntime);
13044   effect(USE meth);
13045 
13046   ins_cost(300);
13047   format %{ "call,runtime " %}
13048   ins_encode(clear_avx, Java_To_Runtime(meth));
13049   ins_pipe(pipe_slow);
13050 %}
13051 
13052 // Call runtime without safepoint
13053 instruct CallLeafDirect(method meth)
13054 %{
13055   match(CallLeaf);
13056   effect(USE meth);
13057 
13058   ins_cost(300);
13059   format %{ "call_leaf,runtime " %}
13060   ins_encode(clear_avx, Java_To_Runtime(meth));
13061   ins_pipe(pipe_slow);
13062 %}
13063 
13064 // Call runtime without safepoint and with vector arguments
13065 instruct CallLeafDirectVector(method meth)
13066 %{
13067   match(CallLeafVector);
13068   effect(USE meth);
13069 
13070   ins_cost(300);
13071   format %{ "call_leaf,vector " %}
13072   ins_encode(Java_To_Runtime(meth));
13073   ins_pipe(pipe_slow);
13074 %}
13075 
13076 //
13077 instruct CallNativeDirect(method meth)
13078 %{
13079   match(CallNative);
13080   effect(USE meth);
13081 
13082   ins_cost(300);
13083   format %{ "call_native " %}
13084   ins_encode(clear_avx, Java_To_Runtime(meth));
13085   ins_pipe(pipe_slow);
13086 %}
13087 
13088 // Call runtime without safepoint
13089 instruct CallLeafNoFPDirect(method meth)
13090 %{
13091   match(CallLeafNoFP);
13092   effect(USE meth);
13093 
13094   ins_cost(300);
13095   format %{ "call_leaf_nofp,runtime " %}
13096   ins_encode(clear_avx, Java_To_Runtime(meth));
13097   ins_pipe(pipe_slow);
13098 %}
13099 
13100 // Return Instruction
13101 // Remove the return address & jump to it.
13102 // Notice: We always emit a nop after a ret to make sure there is room
13103 // for safepoint patching
13104 instruct Ret()
13105 %{
13106   match(Return);
13107 
13108   format %{ "ret" %}
13109   ins_encode %{
13110     __ ret(0);
13111   %}
13112   ins_pipe(pipe_jmp);
13113 %}
13114 
13115 // Tail Call; Jump from runtime stub to Java code.
13116 // Also known as an 'interprocedural jump'.
13117 // Target of jump will eventually return to caller.
13118 // TailJump below removes the return address.
13119 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13120 %{
13121   match(TailCall jump_target method_ptr);
13122 
13123   ins_cost(300);
13124   format %{ "jmp     $jump_target\t# rbx holds method" %}
13125   ins_encode %{
13126     __ jmp($jump_target$$Register);
13127   %}
13128   ins_pipe(pipe_jmp);
13129 %}
13130 
13131 // Tail Jump; remove the return address; jump to target.
13132 // TailCall above leaves the return address around.
13133 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13134 %{
13135   match(TailJump jump_target ex_oop);
13136 
13137   ins_cost(300);
13138   format %{ "popq    rdx\t# pop return address\n\t"
13139             "jmp     $jump_target" %}
13140   ins_encode %{
13141     __ popq(as_Register(RDX_enc));
13142     __ jmp($jump_target$$Register);
13143   %}
13144   ins_pipe(pipe_jmp);
13145 %}
13146 
13147 // Create exception oop: created by stack-crawling runtime code.
13148 // Created exception is now available to this handler, and is setup
13149 // just prior to jumping to this handler.  No code emitted.
13150 instruct CreateException(rax_RegP ex_oop)
13151 %{
13152   match(Set ex_oop (CreateEx));
13153 
13154   size(0);
13155   // use the following format syntax
13156   format %{ "# exception oop is in rax; no code emitted" %}
13157   ins_encode();
13158   ins_pipe(empty);
13159 %}
13160 
13161 // Rethrow exception:
13162 // The exception oop will come in the first argument position.
13163 // Then JUMP (not call) to the rethrow stub code.
13164 instruct RethrowException()
13165 %{
13166   match(Rethrow);
13167 
13168   // use the following format syntax
13169   format %{ "jmp     rethrow_stub" %}
13170   ins_encode(enc_rethrow);
13171   ins_pipe(pipe_jmp);
13172 %}
13173 
13174 // ============================================================================
13175 // This name is KNOWN by the ADLC and cannot be changed.
13176 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13177 // for this guy.
13178 instruct tlsLoadP(r15_RegP dst) %{
13179   match(Set dst (ThreadLocal));
13180   effect(DEF dst);
13181 
13182   size(0);
13183   format %{ "# TLS is in R15" %}
13184   ins_encode( /*empty encoding*/ );
13185   ins_pipe(ialu_reg_reg);
13186 %}
13187 
13188 
13189 //----------PEEPHOLE RULES-----------------------------------------------------
13190 // These must follow all instruction definitions as they use the names
13191 // defined in the instructions definitions.
13192 //
13193 // peepmatch ( root_instr_name [preceding_instruction]* );
13194 //
13195 // peepconstraint %{
13196 // (instruction_number.operand_name relational_op instruction_number.operand_name
13197 //  [, ...] );
13198 // // instruction numbers are zero-based using left to right order in peepmatch
13199 //
13200 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13201 // // provide an instruction_number.operand_name for each operand that appears
13202 // // in the replacement instruction's match rule
13203 //
13204 // ---------VM FLAGS---------------------------------------------------------
13205 //
13206 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13207 //
13208 // Each peephole rule is given an identifying number starting with zero and
13209 // increasing by one in the order seen by the parser.  An individual peephole
13210 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13211 // on the command-line.
13212 //
13213 // ---------CURRENT LIMITATIONS----------------------------------------------
13214 //
13215 // Only match adjacent instructions in same basic block
13216 // Only equality constraints
13217 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13218 // Only one replacement instruction
13219 //
13220 // ---------EXAMPLE----------------------------------------------------------
13221 //
13222 // // pertinent parts of existing instructions in architecture description
13223 // instruct movI(rRegI dst, rRegI src)
13224 // %{
13225 //   match(Set dst (CopyI src));
13226 // %}
13227 //
13228 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13229 // %{
13230 //   match(Set dst (AddI dst src));
13231 //   effect(KILL cr);
13232 // %}
13233 //
13234 // // Change (inc mov) to lea
13235 // peephole %{
13236 //   // increment preceeded by register-register move
13237 //   peepmatch ( incI_rReg movI );
13238 //   // require that the destination register of the increment
13239 //   // match the destination register of the move
13240 //   peepconstraint ( 0.dst == 1.dst );
13241 //   // construct a replacement instruction that sets
13242 //   // the destination to ( move's source register + one )
13243 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13244 // %}
13245 //
13246 
13247 // Implementation no longer uses movX instructions since
13248 // machine-independent system no longer uses CopyX nodes.
13249 //
13250 // peephole
13251 // %{
13252 //   peepmatch (incI_rReg movI);
13253 //   peepconstraint (0.dst == 1.dst);
13254 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13255 // %}
13256 
13257 // peephole
13258 // %{
13259 //   peepmatch (decI_rReg movI);
13260 //   peepconstraint (0.dst == 1.dst);
13261 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13262 // %}
13263 
13264 // peephole
13265 // %{
13266 //   peepmatch (addI_rReg_imm movI);
13267 //   peepconstraint (0.dst == 1.dst);
13268 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13269 // %}
13270 
13271 // peephole
13272 // %{
13273 //   peepmatch (incL_rReg movL);
13274 //   peepconstraint (0.dst == 1.dst);
13275 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13276 // %}
13277 
13278 // peephole
13279 // %{
13280 //   peepmatch (decL_rReg movL);
13281 //   peepconstraint (0.dst == 1.dst);
13282 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13283 // %}
13284 
13285 // peephole
13286 // %{
13287 //   peepmatch (addL_rReg_imm movL);
13288 //   peepconstraint (0.dst == 1.dst);
13289 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13290 // %}
13291 
13292 // peephole
13293 // %{
13294 //   peepmatch (addP_rReg_imm movP);
13295 //   peepconstraint (0.dst == 1.dst);
13296 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13297 // %}
13298 
13299 // // Change load of spilled value to only a spill
13300 // instruct storeI(memory mem, rRegI src)
13301 // %{
13302 //   match(Set mem (StoreI mem src));
13303 // %}
13304 //
13305 // instruct loadI(rRegI dst, memory mem)
13306 // %{
13307 //   match(Set dst (LoadI mem));
13308 // %}
13309 //
13310 
13311 peephole
13312 %{
13313   peepmatch (loadI storeI);
13314   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13315   peepreplace (storeI(1.mem 1.mem 1.src));
13316 %}
13317 
13318 peephole
13319 %{
13320   peepmatch (loadL storeL);
13321   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13322   peepreplace (storeL(1.mem 1.mem 1.src));
13323 %}
13324 
13325 //----------SMARTSPILL RULES---------------------------------------------------
13326 // These must follow all instruction definitions as they use the names
13327 // defined in the instructions definitions.