1 //
    2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // archtecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all int registers (excluding RSP)
  245 reg_class int_reg %{
  246   return _INT_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RAX, RDX, and RSP)
  250 reg_class int_no_rax_rdx_reg %{
  251   return _INT_NO_RAX_RDX_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RCX and RSP)
  255 reg_class int_no_rcx_reg %{
  256   return _INT_NO_RCX_REG_mask;
  257 %}
  258 
  259 // Singleton class for RAX pointer register
  260 reg_class ptr_rax_reg(RAX, RAX_H);
  261 
  262 // Singleton class for RBX pointer register
  263 reg_class ptr_rbx_reg(RBX, RBX_H);
  264 
  265 // Singleton class for RSI pointer register
  266 reg_class ptr_rsi_reg(RSI, RSI_H);
  267 
  268 // Singleton class for RBP pointer register
  269 reg_class ptr_rbp_reg(RBP, RBP_H);
  270 
  271 // Singleton class for RDI pointer register
  272 reg_class ptr_rdi_reg(RDI, RDI_H);
  273 
  274 // Singleton class for stack pointer
  275 reg_class ptr_rsp_reg(RSP, RSP_H);
  276 
  277 // Singleton class for TLS pointer
  278 reg_class ptr_r15_reg(R15, R15_H);
  279 
  280 // Singleton class for RAX long register
  281 reg_class long_rax_reg(RAX, RAX_H);
  282 
  283 // Singleton class for RCX long register
  284 reg_class long_rcx_reg(RCX, RCX_H);
  285 
  286 // Singleton class for RDX long register
  287 reg_class long_rdx_reg(RDX, RDX_H);
  288 
  289 // Singleton class for RAX int register
  290 reg_class int_rax_reg(RAX);
  291 
  292 // Singleton class for RBX int register
  293 reg_class int_rbx_reg(RBX);
  294 
  295 // Singleton class for RCX int register
  296 reg_class int_rcx_reg(RCX);
  297 
  298 // Singleton class for RCX int register
  299 reg_class int_rdx_reg(RDX);
  300 
  301 // Singleton class for RCX int register
  302 reg_class int_rdi_reg(RDI);
  303 
  304 // Singleton class for instruction pointer
  305 // reg_class ip_reg(RIP);
  306 
  307 %}
  308 
  309 //----------SOURCE BLOCK-------------------------------------------------------
  310 // This is a block of C++ code which provides values, functions, and
  311 // definitions necessary in the rest of the architecture description
  312 source_hpp %{
  313 
  314 extern RegMask _ANY_REG_mask;
  315 extern RegMask _PTR_REG_mask;
  316 extern RegMask _PTR_REG_NO_RBP_mask;
  317 extern RegMask _PTR_NO_RAX_REG_mask;
  318 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  319 extern RegMask _LONG_REG_mask;
  320 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  321 extern RegMask _LONG_NO_RCX_REG_mask;
  322 extern RegMask _INT_REG_mask;
  323 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  324 extern RegMask _INT_NO_RCX_REG_mask;
  325 extern RegMask _FLOAT_REG_mask;
  326 
  327 extern RegMask _STACK_OR_PTR_REG_mask;
  328 extern RegMask _STACK_OR_LONG_REG_mask;
  329 extern RegMask _STACK_OR_INT_REG_mask;
  330 
  331 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  332 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  333 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  334 
  335 %}
  336 
  337 source %{
  338 #define   RELOC_IMM64    Assembler::imm_operand
  339 #define   RELOC_DISP32   Assembler::disp32_operand
  340 
  341 #define __ _masm.
  342 
  343 RegMask _ANY_REG_mask;
  344 RegMask _PTR_REG_mask;
  345 RegMask _PTR_REG_NO_RBP_mask;
  346 RegMask _PTR_NO_RAX_REG_mask;
  347 RegMask _PTR_NO_RAX_RBX_REG_mask;
  348 RegMask _LONG_REG_mask;
  349 RegMask _LONG_NO_RAX_RDX_REG_mask;
  350 RegMask _LONG_NO_RCX_REG_mask;
  351 RegMask _INT_REG_mask;
  352 RegMask _INT_NO_RAX_RDX_REG_mask;
  353 RegMask _INT_NO_RCX_REG_mask;
  354 RegMask _FLOAT_REG_mask;
  355 RegMask _STACK_OR_PTR_REG_mask;
  356 RegMask _STACK_OR_LONG_REG_mask;
  357 RegMask _STACK_OR_INT_REG_mask;
  358 
  359 static bool need_r12_heapbase() {
  360   return UseCompressedOops;
  361 }
  362 
  363 void reg_mask_init() {
  364   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  365   // We derive a number of subsets from it.
  366   _ANY_REG_mask = _ALL_REG_mask;
  367 
  368   if (PreserveFramePointer) {
  369     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  371   }
  372   if (need_r12_heapbase()) {
  373     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  374     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  375   }
  376 
  377   _PTR_REG_mask = _ANY_REG_mask;
  378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  379   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  380   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  381   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  382 
  383   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  384   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  385 
  386   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  387   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  388   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  389 
  390   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  391   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  392   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  393 
  394   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  395   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  396   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  397 
  398   _LONG_REG_mask = _PTR_REG_mask;
  399   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  400   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  401 
  402   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  404   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  405   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  406   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  407 
  408   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  409   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  410   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  411 
  412   _INT_REG_mask = _ALL_INT_REG_mask;
  413   if (PreserveFramePointer) {
  414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  415   }
  416   if (need_r12_heapbase()) {
  417     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  418   }
  419 
  420   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  421   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  422 
  423   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  424   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  425   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  426 
  427   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  428   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  429 
  430   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  431   // from the float_reg_legacy/float_reg_evex register class.
  432   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  433 
  434   if (Matcher::has_predicated_vectors()) {
  435     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
  436     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
  437     // curruption of value held in K1 register.
  438     if (PostLoopMultiversioning) {
  439       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()));
  440       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()->next()));
  441     }
  442   }
  443 }
  444 
  445 static bool generate_vzeroupper(Compile* C) {
  446   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  447 }
  448 
  449 static int clear_avx_size() {
  450   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  451 }
  452 
  453 // !!!!! Special hack to get all types of calls to specify the byte offset
  454 //       from the start of the call to the point where the return address
  455 //       will point.
  456 int MachCallStaticJavaNode::ret_addr_offset()
  457 {
  458   int offset = 5; // 5 bytes from start of call to where return address points
  459   offset += clear_avx_size();
  460   return offset;
  461 }
  462 
  463 int MachCallDynamicJavaNode::ret_addr_offset()
  464 {
  465   int offset = 15; // 15 bytes from start of call to where return address points
  466   offset += clear_avx_size();
  467   return offset;
  468 }
  469 
  470 int MachCallRuntimeNode::ret_addr_offset() {
  471   int offset = 13; // movq r10,#addr; callq (r10)
  472   if (this->ideal_Opcode() != Op_CallLeafVector) {
  473     offset += clear_avx_size();
  474   }
  475   return offset;
  476 }
  477 
  478 int MachCallNativeNode::ret_addr_offset() {
  479   int offset = 13; // movq r10,#addr; callq (r10)
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 //
  484 // Compute padding required for nodes which need alignment
  485 //
  486 
  487 // The address of the call instruction needs to be 4-byte aligned to
  488 // ensure that it does not span a cache line so that it can be patched.
  489 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  490 {
  491   current_offset += clear_avx_size(); // skip vzeroupper
  492   current_offset += 1; // skip call opcode byte
  493   return align_up(current_offset, alignment_required()) - current_offset;
  494 }
  495 
  496 // The address of the call instruction needs to be 4-byte aligned to
  497 // ensure that it does not span a cache line so that it can be patched.
  498 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  499 {
  500   current_offset += clear_avx_size(); // skip vzeroupper
  501   current_offset += 11; // skip movq instruction + call opcode byte
  502   return align_up(current_offset, alignment_required()) - current_offset;
  503 }
  504 
  505 // EMIT_RM()
  506 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  507   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  508   cbuf.insts()->emit_int8(c);
  509 }
  510 
  511 // EMIT_CC()
  512 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  513   unsigned char c = (unsigned char) (f1 | f2);
  514   cbuf.insts()->emit_int8(c);
  515 }
  516 
  517 // EMIT_OPCODE()
  518 void emit_opcode(CodeBuffer &cbuf, int code) {
  519   cbuf.insts()->emit_int8((unsigned char) code);
  520 }
  521 
  522 // EMIT_OPCODE() w/ relocation information
  523 void emit_opcode(CodeBuffer &cbuf,
  524                  int code, relocInfo::relocType reloc, int offset, int format)
  525 {
  526   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  527   emit_opcode(cbuf, code);
  528 }
  529 
  530 // EMIT_D8()
  531 void emit_d8(CodeBuffer &cbuf, int d8) {
  532   cbuf.insts()->emit_int8((unsigned char) d8);
  533 }
  534 
  535 // EMIT_D16()
  536 void emit_d16(CodeBuffer &cbuf, int d16) {
  537   cbuf.insts()->emit_int16(d16);
  538 }
  539 
  540 // EMIT_D32()
  541 void emit_d32(CodeBuffer &cbuf, int d32) {
  542   cbuf.insts()->emit_int32(d32);
  543 }
  544 
  545 // EMIT_D64()
  546 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  547   cbuf.insts()->emit_int64(d64);
  548 }
  549 
  550 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  551 void emit_d32_reloc(CodeBuffer& cbuf,
  552                     int d32,
  553                     relocInfo::relocType reloc,
  554                     int format)
  555 {
  556   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  557   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  558   cbuf.insts()->emit_int32(d32);
  559 }
  560 
  561 // emit 32 bit value and construct relocation entry from RelocationHolder
  562 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  563 #ifdef ASSERT
  564   if (rspec.reloc()->type() == relocInfo::oop_type &&
  565       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  566     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  567     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  568   }
  569 #endif
  570   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  571   cbuf.insts()->emit_int32(d32);
  572 }
  573 
  574 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  575   address next_ip = cbuf.insts_end() + 4;
  576   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  577                  external_word_Relocation::spec(addr),
  578                  RELOC_DISP32);
  579 }
  580 
  581 
  582 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  583 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  584   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  585   cbuf.insts()->emit_int64(d64);
  586 }
  587 
  588 // emit 64 bit value and construct relocation entry from RelocationHolder
  589 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  590 #ifdef ASSERT
  591   if (rspec.reloc()->type() == relocInfo::oop_type &&
  592       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  593     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  594     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  595   }
  596 #endif
  597   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  598   cbuf.insts()->emit_int64(d64);
  599 }
  600 
  601 // Access stack slot for load or store
  602 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  603 {
  604   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  605   if (-0x80 <= disp && disp < 0x80) {
  606     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  607     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  608     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  609   } else {
  610     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  611     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  612     emit_d32(cbuf, disp);     // Displacement // R/M byte
  613   }
  614 }
  615 
  616    // rRegI ereg, memory mem) %{    // emit_reg_mem
  617 void encode_RegMem(CodeBuffer &cbuf,
  618                    int reg,
  619                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  620 {
  621   assert(disp_reloc == relocInfo::none, "cannot have disp");
  622   int regenc = reg & 7;
  623   int baseenc = base & 7;
  624   int indexenc = index & 7;
  625 
  626   // There is no index & no scale, use form without SIB byte
  627   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  628     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  629     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  630       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  631     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  632       // If 8-bit displacement, mode 0x1
  633       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  634       emit_d8(cbuf, disp);
  635     } else {
  636       // If 32-bit displacement
  637       if (base == -1) { // Special flag for absolute address
  638         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  639         if (disp_reloc != relocInfo::none) {
  640           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  641         } else {
  642           emit_d32(cbuf, disp);
  643         }
  644       } else {
  645         // Normal base + offset
  646         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       }
  653     }
  654   } else {
  655     // Else, encode with the SIB byte
  656     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  657     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  658       // If no displacement
  659       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  660       emit_rm(cbuf, scale, indexenc, baseenc);
  661     } else {
  662       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  663         // If 8-bit displacement, mode 0x1
  664         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  665         emit_rm(cbuf, scale, indexenc, baseenc);
  666         emit_d8(cbuf, disp);
  667       } else {
  668         // If 32-bit displacement
  669         if (base == 0x04 ) {
  670           emit_rm(cbuf, 0x2, regenc, 0x4);
  671           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  672         } else {
  673           emit_rm(cbuf, 0x2, regenc, 0x4);
  674           emit_rm(cbuf, scale, indexenc, baseenc); // *
  675         }
  676         if (disp_reloc != relocInfo::none) {
  677           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  678         } else {
  679           emit_d32(cbuf, disp);
  680         }
  681       }
  682     }
  683   }
  684 }
  685 
  686 // This could be in MacroAssembler but it's fairly C2 specific
  687 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  688   Label exit;
  689   __ jccb(Assembler::noParity, exit);
  690   __ pushf();
  691   //
  692   // comiss/ucomiss instructions set ZF,PF,CF flags and
  693   // zero OF,AF,SF for NaN values.
  694   // Fixup flags by zeroing ZF,PF so that compare of NaN
  695   // values returns 'less than' result (CF is set).
  696   // Leave the rest of flags unchanged.
  697   //
  698   //    7 6 5 4 3 2 1 0
  699   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  700   //    0 0 1 0 1 0 1 1   (0x2B)
  701   //
  702   __ andq(Address(rsp, 0), 0xffffff2b);
  703   __ popf();
  704   __ bind(exit);
  705 }
  706 
  707 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  708   Label done;
  709   __ movl(dst, -1);
  710   __ jcc(Assembler::parity, done);
  711   __ jcc(Assembler::below, done);
  712   __ setb(Assembler::notEqual, dst);
  713   __ movzbl(dst, dst);
  714   __ bind(done);
  715 }
  716 
  717 // Math.min()    # Math.max()
  718 // --------------------------
  719 // ucomis[s/d]   #
  720 // ja   -> b     # a
  721 // jp   -> NaN   # NaN
  722 // jb   -> a     # b
  723 // je            #
  724 // |-jz -> a | b # a & b
  725 // |    -> a     #
  726 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  727                      XMMRegister a, XMMRegister b,
  728                      XMMRegister xmmt, Register rt,
  729                      bool min, bool single) {
  730 
  731   Label nan, zero, below, above, done;
  732 
  733   if (single)
  734     __ ucomiss(a, b);
  735   else
  736     __ ucomisd(a, b);
  737 
  738   if (dst->encoding() != (min ? b : a)->encoding())
  739     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  740   else
  741     __ jccb(Assembler::above, done);
  742 
  743   __ jccb(Assembler::parity, nan);  // PF=1
  744   __ jccb(Assembler::below, below); // CF=1
  745 
  746   // equal
  747   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  748   if (single) {
  749     __ ucomiss(a, xmmt);
  750     __ jccb(Assembler::equal, zero);
  751 
  752     __ movflt(dst, a);
  753     __ jmp(done);
  754   }
  755   else {
  756     __ ucomisd(a, xmmt);
  757     __ jccb(Assembler::equal, zero);
  758 
  759     __ movdbl(dst, a);
  760     __ jmp(done);
  761   }
  762 
  763   __ bind(zero);
  764   if (min)
  765     __ vpor(dst, a, b, Assembler::AVX_128bit);
  766   else
  767     __ vpand(dst, a, b, Assembler::AVX_128bit);
  768 
  769   __ jmp(done);
  770 
  771   __ bind(above);
  772   if (single)
  773     __ movflt(dst, min ? b : a);
  774   else
  775     __ movdbl(dst, min ? b : a);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(nan);
  780   if (single) {
  781     __ movl(rt, 0x7fc00000); // Float.NaN
  782     __ movdl(dst, rt);
  783   }
  784   else {
  785     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  786     __ movdq(dst, rt);
  787   }
  788   __ jmp(done);
  789 
  790   __ bind(below);
  791   if (single)
  792     __ movflt(dst, min ? a : b);
  793   else
  794     __ movdbl(dst, min ? a : b);
  795 
  796   __ bind(done);
  797 }
  798 
  799 //=============================================================================
  800 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  801 
  802 int ConstantTable::calculate_table_base_offset() const {
  803   return 0;  // absolute addressing, no offset
  804 }
  805 
  806 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  807 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  808   ShouldNotReachHere();
  809 }
  810 
  811 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  812   // Empty encoding
  813 }
  814 
  815 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  816   return 0;
  817 }
  818 
  819 #ifndef PRODUCT
  820 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  821   st->print("# MachConstantBaseNode (empty encoding)");
  822 }
  823 #endif
  824 
  825 
  826 //=============================================================================
  827 #ifndef PRODUCT
  828 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   Compile* C = ra_->C;
  830 
  831   int framesize = C->output()->frame_size_in_bytes();
  832   int bangsize = C->output()->bang_size_in_bytes();
  833   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  834   // Remove wordSize for return addr which is already pushed.
  835   framesize -= wordSize;
  836 
  837   if (C->output()->need_stack_bang(bangsize)) {
  838     framesize -= wordSize;
  839     st->print("# stack bang (%d bytes)", bangsize);
  840     st->print("\n\t");
  841     st->print("pushq   rbp\t# Save rbp");
  842     if (PreserveFramePointer) {
  843         st->print("\n\t");
  844         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  845     }
  846     if (framesize) {
  847       st->print("\n\t");
  848       st->print("subq    rsp, #%d\t# Create frame",framesize);
  849     }
  850   } else {
  851     st->print("subq    rsp, #%d\t# Create frame",framesize);
  852     st->print("\n\t");
  853     framesize -= wordSize;
  854     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  855     if (PreserveFramePointer) {
  856       st->print("\n\t");
  857       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  858       if (framesize > 0) {
  859         st->print("\n\t");
  860         st->print("addq    rbp, #%d", framesize);
  861       }
  862     }
  863   }
  864 
  865   if (VerifyStackAtCalls) {
  866     st->print("\n\t");
  867     framesize -= wordSize;
  868     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  869 #ifdef ASSERT
  870     st->print("\n\t");
  871     st->print("# stack alignment check");
  872 #endif
  873   }
  874   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  875     st->print("\n\t");
  876     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  877     st->print("\n\t");
  878     st->print("je      fast_entry\t");
  879     st->print("\n\t");
  880     st->print("call    #nmethod_entry_barrier_stub\t");
  881     st->print("\n\tfast_entry:");
  882   }
  883   st->cr();
  884 }
  885 #endif
  886 
  887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  888   Compile* C = ra_->C;
  889   MacroAssembler _masm(&cbuf);
  890 
  891   int framesize = C->output()->frame_size_in_bytes();
  892   int bangsize = C->output()->bang_size_in_bytes();
  893 
  894   if (C->clinit_barrier_on_entry()) {
  895     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  896     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  897 
  898     Label L_skip_barrier;
  899     Register klass = rscratch1;
  900 
  901     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  902     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  903 
  904     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  905 
  906     __ bind(L_skip_barrier);
  907   }
  908 
  909   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  910 
  911   C->output()->set_frame_complete(cbuf.insts_size());
  912 
  913   if (C->has_mach_constant_base_node()) {
  914     // NOTE: We set the table base offset here because users might be
  915     // emitted before MachConstantBaseNode.
  916     ConstantTable& constant_table = C->output()->constant_table();
  917     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  918   }
  919 }
  920 
  921 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  922 {
  923   return MachNode::size(ra_); // too many variables; just compute it
  924                               // the hard way
  925 }
  926 
  927 int MachPrologNode::reloc() const
  928 {
  929   return 0; // a large enough number
  930 }
  931 
  932 //=============================================================================
  933 #ifndef PRODUCT
  934 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  935 {
  936   Compile* C = ra_->C;
  937   if (generate_vzeroupper(C)) {
  938     st->print("vzeroupper");
  939     st->cr(); st->print("\t");
  940   }
  941 
  942   int framesize = C->output()->frame_size_in_bytes();
  943   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  944   // Remove word for return adr already pushed
  945   // and RBP
  946   framesize -= 2*wordSize;
  947 
  948   if (framesize) {
  949     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  950     st->print("\t");
  951   }
  952 
  953   st->print_cr("popq    rbp");
  954   if (do_polling() && C->is_method_compilation()) {
  955     st->print("\t");
  956     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  957                  "ja       #safepoint_stub\t"
  958                  "# Safepoint: poll for GC");
  959   }
  960 }
  961 #endif
  962 
  963 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  964 {
  965   Compile* C = ra_->C;
  966   MacroAssembler _masm(&cbuf);
  967 
  968   if (generate_vzeroupper(C)) {
  969     // Clear upper bits of YMM registers when current compiled code uses
  970     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  971     __ vzeroupper();
  972   }
  973 
  974   int framesize = C->output()->frame_size_in_bytes();
  975   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  976   // Remove word for return adr already pushed
  977   // and RBP
  978   framesize -= 2*wordSize;
  979 
  980   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  981 
  982   if (framesize) {
  983     emit_opcode(cbuf, Assembler::REX_W);
  984     if (framesize < 0x80) {
  985       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  986       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  987       emit_d8(cbuf, framesize);
  988     } else {
  989       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  990       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  991       emit_d32(cbuf, framesize);
  992     }
  993   }
  994 
  995   // popq rbp
  996   emit_opcode(cbuf, 0x58 | RBP_enc);
  997 
  998   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  999     __ reserved_stack_check();
 1000   }
 1001 
 1002   if (do_polling() && C->is_method_compilation()) {
 1003     MacroAssembler _masm(&cbuf);
 1004     Label dummy_label;
 1005     Label* code_stub = &dummy_label;
 1006     if (!C->output()->in_scratch_emit_size()) {
 1007       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1008     }
 1009     __ relocate(relocInfo::poll_return_type);
 1010     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1011   }
 1012 }
 1013 
 1014 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1015 {
 1016   return MachNode::size(ra_); // too many variables; just compute it
 1017                               // the hard way
 1018 }
 1019 
 1020 int MachEpilogNode::reloc() const
 1021 {
 1022   return 2; // a large enough number
 1023 }
 1024 
 1025 const Pipeline* MachEpilogNode::pipeline() const
 1026 {
 1027   return MachNode::pipeline_class();
 1028 }
 1029 
 1030 //=============================================================================
 1031 
 1032 enum RC {
 1033   rc_bad,
 1034   rc_int,
 1035   rc_kreg,
 1036   rc_float,
 1037   rc_stack
 1038 };
 1039 
 1040 static enum RC rc_class(OptoReg::Name reg)
 1041 {
 1042   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1043 
 1044   if (OptoReg::is_stack(reg)) return rc_stack;
 1045 
 1046   VMReg r = OptoReg::as_VMReg(reg);
 1047 
 1048   if (r->is_Register()) return rc_int;
 1049 
 1050   if (r->is_KRegister()) return rc_kreg;
 1051 
 1052   assert(r->is_XMMRegister(), "must be");
 1053   return rc_float;
 1054 }
 1055 
 1056 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1057 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1058                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1059 
 1060 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1061                      int stack_offset, int reg, uint ireg, outputStream* st);
 1062 
 1063 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1064                                       int dst_offset, uint ireg, outputStream* st) {
 1065   if (cbuf) {
 1066     MacroAssembler _masm(cbuf);
 1067     switch (ireg) {
 1068     case Op_VecS:
 1069       __ movq(Address(rsp, -8), rax);
 1070       __ movl(rax, Address(rsp, src_offset));
 1071       __ movl(Address(rsp, dst_offset), rax);
 1072       __ movq(rax, Address(rsp, -8));
 1073       break;
 1074     case Op_VecD:
 1075       __ pushq(Address(rsp, src_offset));
 1076       __ popq (Address(rsp, dst_offset));
 1077       break;
 1078     case Op_VecX:
 1079       __ pushq(Address(rsp, src_offset));
 1080       __ popq (Address(rsp, dst_offset));
 1081       __ pushq(Address(rsp, src_offset+8));
 1082       __ popq (Address(rsp, dst_offset+8));
 1083       break;
 1084     case Op_VecY:
 1085       __ vmovdqu(Address(rsp, -32), xmm0);
 1086       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1087       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1088       __ vmovdqu(xmm0, Address(rsp, -32));
 1089       break;
 1090     case Op_VecZ:
 1091       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1092       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1093       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1094       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1095       break;
 1096     default:
 1097       ShouldNotReachHere();
 1098     }
 1099 #ifndef PRODUCT
 1100   } else {
 1101     switch (ireg) {
 1102     case Op_VecS:
 1103       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1104                 "movl    rax, [rsp + #%d]\n\t"
 1105                 "movl    [rsp + #%d], rax\n\t"
 1106                 "movq    rax, [rsp - #8]",
 1107                 src_offset, dst_offset);
 1108       break;
 1109     case Op_VecD:
 1110       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1111                 "popq    [rsp + #%d]",
 1112                 src_offset, dst_offset);
 1113       break;
 1114      case Op_VecX:
 1115       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1116                 "popq    [rsp + #%d]\n\t"
 1117                 "pushq   [rsp + #%d]\n\t"
 1118                 "popq    [rsp + #%d]",
 1119                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1120       break;
 1121     case Op_VecY:
 1122       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1123                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1124                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1125                 "vmovdqu xmm0, [rsp - #32]",
 1126                 src_offset, dst_offset);
 1127       break;
 1128     case Op_VecZ:
 1129       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1130                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1131                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1132                 "vmovdqu xmm0, [rsp - #64]",
 1133                 src_offset, dst_offset);
 1134       break;
 1135     default:
 1136       ShouldNotReachHere();
 1137     }
 1138 #endif
 1139   }
 1140 }
 1141 
 1142 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1143                                        PhaseRegAlloc* ra_,
 1144                                        bool do_size,
 1145                                        outputStream* st) const {
 1146   assert(cbuf != NULL || st  != NULL, "sanity");
 1147   // Get registers to move
 1148   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1149   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1150   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1151   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1152 
 1153   enum RC src_second_rc = rc_class(src_second);
 1154   enum RC src_first_rc = rc_class(src_first);
 1155   enum RC dst_second_rc = rc_class(dst_second);
 1156   enum RC dst_first_rc = rc_class(dst_first);
 1157 
 1158   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1159          "must move at least 1 register" );
 1160 
 1161   if (src_first == dst_first && src_second == dst_second) {
 1162     // Self copy, no move
 1163     return 0;
 1164   }
 1165   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1166     uint ireg = ideal_reg();
 1167     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1168     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1169     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1170       // mem -> mem
 1171       int src_offset = ra_->reg2offset(src_first);
 1172       int dst_offset = ra_->reg2offset(dst_first);
 1173       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1174     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1175       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1176     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1177       int stack_offset = ra_->reg2offset(dst_first);
 1178       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1179     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1180       int stack_offset = ra_->reg2offset(src_first);
 1181       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1182     } else {
 1183       ShouldNotReachHere();
 1184     }
 1185     return 0;
 1186   }
 1187   if (src_first_rc == rc_stack) {
 1188     // mem ->
 1189     if (dst_first_rc == rc_stack) {
 1190       // mem -> mem
 1191       assert(src_second != dst_first, "overlap");
 1192       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1193           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1194         // 64-bit
 1195         int src_offset = ra_->reg2offset(src_first);
 1196         int dst_offset = ra_->reg2offset(dst_first);
 1197         if (cbuf) {
 1198           MacroAssembler _masm(cbuf);
 1199           __ pushq(Address(rsp, src_offset));
 1200           __ popq (Address(rsp, dst_offset));
 1201 #ifndef PRODUCT
 1202         } else {
 1203           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1204                     "popq    [rsp + #%d]",
 1205                      src_offset, dst_offset);
 1206 #endif
 1207         }
 1208       } else {
 1209         // 32-bit
 1210         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1211         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1212         // No pushl/popl, so:
 1213         int src_offset = ra_->reg2offset(src_first);
 1214         int dst_offset = ra_->reg2offset(dst_first);
 1215         if (cbuf) {
 1216           MacroAssembler _masm(cbuf);
 1217           __ movq(Address(rsp, -8), rax);
 1218           __ movl(rax, Address(rsp, src_offset));
 1219           __ movl(Address(rsp, dst_offset), rax);
 1220           __ movq(rax, Address(rsp, -8));
 1221 #ifndef PRODUCT
 1222         } else {
 1223           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1224                     "movl    rax, [rsp + #%d]\n\t"
 1225                     "movl    [rsp + #%d], rax\n\t"
 1226                     "movq    rax, [rsp - #8]",
 1227                      src_offset, dst_offset);
 1228 #endif
 1229         }
 1230       }
 1231       return 0;
 1232     } else if (dst_first_rc == rc_int) {
 1233       // mem -> gpr
 1234       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1235           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1236         // 64-bit
 1237         int offset = ra_->reg2offset(src_first);
 1238         if (cbuf) {
 1239           MacroAssembler _masm(cbuf);
 1240           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1241 #ifndef PRODUCT
 1242         } else {
 1243           st->print("movq    %s, [rsp + #%d]\t# spill",
 1244                      Matcher::regName[dst_first],
 1245                      offset);
 1246 #endif
 1247         }
 1248       } else {
 1249         // 32-bit
 1250         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1251         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1252         int offset = ra_->reg2offset(src_first);
 1253         if (cbuf) {
 1254           MacroAssembler _masm(cbuf);
 1255           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1256 #ifndef PRODUCT
 1257         } else {
 1258           st->print("movl    %s, [rsp + #%d]\t# spill",
 1259                      Matcher::regName[dst_first],
 1260                      offset);
 1261 #endif
 1262         }
 1263       }
 1264       return 0;
 1265     } else if (dst_first_rc == rc_float) {
 1266       // mem-> xmm
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(src_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("%s  %s, [rsp + #%d]\t# spill",
 1277                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1278                      Matcher::regName[dst_first],
 1279                      offset);
 1280 #endif
 1281         }
 1282       } else {
 1283         // 32-bit
 1284         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1285         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1286         int offset = ra_->reg2offset(src_first);
 1287         if (cbuf) {
 1288           MacroAssembler _masm(cbuf);
 1289           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1290 #ifndef PRODUCT
 1291         } else {
 1292           st->print("movss   %s, [rsp + #%d]\t# spill",
 1293                      Matcher::regName[dst_first],
 1294                      offset);
 1295 #endif
 1296         }
 1297       }
 1298       return 0;
 1299     } else if (dst_first_rc == rc_kreg) {
 1300       // mem -> kreg
 1301       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1302           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1303         // 64-bit
 1304         int offset = ra_->reg2offset(src_first);
 1305         if (cbuf) {
 1306           MacroAssembler _masm(cbuf);
 1307           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1308 #ifndef PRODUCT
 1309         } else {
 1310           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1311                      Matcher::regName[dst_first],
 1312                      offset);
 1313 #endif
 1314         }
 1315       }
 1316       return 0;
 1317     }
 1318   } else if (src_first_rc == rc_int) {
 1319     // gpr ->
 1320     if (dst_first_rc == rc_stack) {
 1321       // gpr -> mem
 1322       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1323           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1324         // 64-bit
 1325         int offset = ra_->reg2offset(dst_first);
 1326         if (cbuf) {
 1327           MacroAssembler _masm(cbuf);
 1328           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1329 #ifndef PRODUCT
 1330         } else {
 1331           st->print("movq    [rsp + #%d], %s\t# spill",
 1332                      offset,
 1333                      Matcher::regName[src_first]);
 1334 #endif
 1335         }
 1336       } else {
 1337         // 32-bit
 1338         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1339         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1340         int offset = ra_->reg2offset(dst_first);
 1341         if (cbuf) {
 1342           MacroAssembler _masm(cbuf);
 1343           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1344 #ifndef PRODUCT
 1345         } else {
 1346           st->print("movl    [rsp + #%d], %s\t# spill",
 1347                      offset,
 1348                      Matcher::regName[src_first]);
 1349 #endif
 1350         }
 1351       }
 1352       return 0;
 1353     } else if (dst_first_rc == rc_int) {
 1354       // gpr -> gpr
 1355       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1356           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1357         // 64-bit
 1358         if (cbuf) {
 1359           MacroAssembler _masm(cbuf);
 1360           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1361                   as_Register(Matcher::_regEncode[src_first]));
 1362 #ifndef PRODUCT
 1363         } else {
 1364           st->print("movq    %s, %s\t# spill",
 1365                      Matcher::regName[dst_first],
 1366                      Matcher::regName[src_first]);
 1367 #endif
 1368         }
 1369         return 0;
 1370       } else {
 1371         // 32-bit
 1372         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1373         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1374         if (cbuf) {
 1375           MacroAssembler _masm(cbuf);
 1376           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1377                   as_Register(Matcher::_regEncode[src_first]));
 1378 #ifndef PRODUCT
 1379         } else {
 1380           st->print("movl    %s, %s\t# spill",
 1381                      Matcher::regName[dst_first],
 1382                      Matcher::regName[src_first]);
 1383 #endif
 1384         }
 1385         return 0;
 1386       }
 1387     } else if (dst_first_rc == rc_float) {
 1388       // gpr -> xmm
 1389       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1390           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1391         // 64-bit
 1392         if (cbuf) {
 1393           MacroAssembler _masm(cbuf);
 1394           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1395 #ifndef PRODUCT
 1396         } else {
 1397           st->print("movdq   %s, %s\t# spill",
 1398                      Matcher::regName[dst_first],
 1399                      Matcher::regName[src_first]);
 1400 #endif
 1401         }
 1402       } else {
 1403         // 32-bit
 1404         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1405         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdl   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_kreg) {
 1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1421         // 64-bit
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425   #ifndef PRODUCT
 1426         } else {
 1427            st->print("kmovq   %s, %s\t# spill",
 1428                        Matcher::regName[dst_first],
 1429                        Matcher::regName[src_first]);
 1430   #endif
 1431         }
 1432       }
 1433       Unimplemented();
 1434       return 0;
 1435     }
 1436   } else if (src_first_rc == rc_float) {
 1437     // xmm ->
 1438     if (dst_first_rc == rc_stack) {
 1439       // xmm -> mem
 1440       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1441           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1442         // 64-bit
 1443         int offset = ra_->reg2offset(dst_first);
 1444         if (cbuf) {
 1445           MacroAssembler _masm(cbuf);
 1446           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1447 #ifndef PRODUCT
 1448         } else {
 1449           st->print("movsd   [rsp + #%d], %s\t# spill",
 1450                      offset,
 1451                      Matcher::regName[src_first]);
 1452 #endif
 1453         }
 1454       } else {
 1455         // 32-bit
 1456         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1457         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1458         int offset = ra_->reg2offset(dst_first);
 1459         if (cbuf) {
 1460           MacroAssembler _masm(cbuf);
 1461           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1462 #ifndef PRODUCT
 1463         } else {
 1464           st->print("movss   [rsp + #%d], %s\t# spill",
 1465                      offset,
 1466                      Matcher::regName[src_first]);
 1467 #endif
 1468         }
 1469       }
 1470       return 0;
 1471     } else if (dst_first_rc == rc_int) {
 1472       // xmm -> gpr
 1473       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1474           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1475         // 64-bit
 1476         if (cbuf) {
 1477           MacroAssembler _masm(cbuf);
 1478           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1479 #ifndef PRODUCT
 1480         } else {
 1481           st->print("movdq   %s, %s\t# spill",
 1482                      Matcher::regName[dst_first],
 1483                      Matcher::regName[src_first]);
 1484 #endif
 1485         }
 1486       } else {
 1487         // 32-bit
 1488         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1489         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1490         if (cbuf) {
 1491           MacroAssembler _masm(cbuf);
 1492           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1493 #ifndef PRODUCT
 1494         } else {
 1495           st->print("movdl   %s, %s\t# spill",
 1496                      Matcher::regName[dst_first],
 1497                      Matcher::regName[src_first]);
 1498 #endif
 1499         }
 1500       }
 1501       return 0;
 1502     } else if (dst_first_rc == rc_float) {
 1503       // xmm -> xmm
 1504       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1505           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1506         // 64-bit
 1507         if (cbuf) {
 1508           MacroAssembler _masm(cbuf);
 1509           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1510 #ifndef PRODUCT
 1511         } else {
 1512           st->print("%s  %s, %s\t# spill",
 1513                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1514                      Matcher::regName[dst_first],
 1515                      Matcher::regName[src_first]);
 1516 #endif
 1517         }
 1518       } else {
 1519         // 32-bit
 1520         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1521         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1522         if (cbuf) {
 1523           MacroAssembler _masm(cbuf);
 1524           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1525 #ifndef PRODUCT
 1526         } else {
 1527           st->print("%s  %s, %s\t# spill",
 1528                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1529                      Matcher::regName[dst_first],
 1530                      Matcher::regName[src_first]);
 1531 #endif
 1532         }
 1533       }
 1534       return 0;
 1535     } else if (dst_first_rc == rc_kreg) {
 1536       assert(false, "Illegal spilling");
 1537       return 0;
 1538     }
 1539   } else if (src_first_rc == rc_kreg) {
 1540     if (dst_first_rc == rc_stack) {
 1541       // mem -> kreg
 1542       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1543           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1544         // 64-bit
 1545         int offset = ra_->reg2offset(dst_first);
 1546         if (cbuf) {
 1547           MacroAssembler _masm(cbuf);
 1548           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1549 #ifndef PRODUCT
 1550         } else {
 1551           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1552                      offset,
 1553                      Matcher::regName[src_first]);
 1554 #endif
 1555         }
 1556       }
 1557       return 0;
 1558     } else if (dst_first_rc == rc_int) {
 1559       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1560           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1561         // 64-bit
 1562         if (cbuf) {
 1563           MacroAssembler _masm(cbuf);
 1564           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1565 #ifndef PRODUCT
 1566         } else {
 1567          st->print("kmovq   %s, %s\t# spill",
 1568                      Matcher::regName[dst_first],
 1569                      Matcher::regName[src_first]);
 1570 #endif
 1571         }
 1572       }
 1573       Unimplemented();
 1574       return 0;
 1575     } else if (dst_first_rc == rc_kreg) {
 1576       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1577           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1578         // 64-bit
 1579         if (cbuf) {
 1580           MacroAssembler _masm(cbuf);
 1581           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1582 #ifndef PRODUCT
 1583         } else {
 1584          st->print("kmovq   %s, %s\t# spill",
 1585                      Matcher::regName[dst_first],
 1586                      Matcher::regName[src_first]);
 1587 #endif
 1588         }
 1589       }
 1590       return 0;
 1591     } else if (dst_first_rc == rc_float) {
 1592       assert(false, "Illegal spill");
 1593       return 0;
 1594     }
 1595   }
 1596 
 1597   assert(0," foo ");
 1598   Unimplemented();
 1599   return 0;
 1600 }
 1601 
 1602 #ifndef PRODUCT
 1603 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1604   implementation(NULL, ra_, false, st);
 1605 }
 1606 #endif
 1607 
 1608 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1609   implementation(&cbuf, ra_, false, NULL);
 1610 }
 1611 
 1612 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1613   return MachNode::size(ra_);
 1614 }
 1615 
 1616 //=============================================================================
 1617 #ifndef PRODUCT
 1618 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1619 {
 1620   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1621   int reg = ra_->get_reg_first(this);
 1622   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1623             Matcher::regName[reg], offset);
 1624 }
 1625 #endif
 1626 
 1627 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1628 {
 1629   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1630   int reg = ra_->get_encode(this);
 1631   if (offset >= 0x80) {
 1632     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1633     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1634     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1635     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1636     emit_d32(cbuf, offset);
 1637   } else {
 1638     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1639     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1640     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1641     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1642     emit_d8(cbuf, offset);
 1643   }
 1644 }
 1645 
 1646 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1647 {
 1648   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1649   return (offset < 0x80) ? 5 : 8; // REX
 1650 }
 1651 
 1652 //=============================================================================
 1653 #ifndef PRODUCT
 1654 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1655 {
 1656   if (UseCompressedClassPointers) {
 1657     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1658     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1659     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1660   } else {
 1661     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1662                  "# Inline cache check");
 1663   }
 1664   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1665   st->print_cr("\tnop\t# nops to align entry point");
 1666 }
 1667 #endif
 1668 
 1669 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1670 {
 1671   MacroAssembler masm(&cbuf);
 1672   uint insts_size = cbuf.insts_size();
 1673   if (UseCompressedClassPointers) {
 1674     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1675     masm.cmpptr(rax, rscratch1);
 1676   } else {
 1677     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1678   }
 1679 
 1680   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1681 
 1682   /* WARNING these NOPs are critical so that verified entry point is properly
 1683      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1684   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1685   if (OptoBreakpoint) {
 1686     // Leave space for int3
 1687     nops_cnt -= 1;
 1688   }
 1689   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1690   if (nops_cnt > 0)
 1691     masm.nop(nops_cnt);
 1692 }
 1693 
 1694 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1695 {
 1696   return MachNode::size(ra_); // too many variables; just compute it
 1697                               // the hard way
 1698 }
 1699 
 1700 
 1701 //=============================================================================
 1702 
 1703 const bool Matcher::supports_vector_calling_convention(void) {
 1704   if (EnableVectorSupport && UseVectorStubs) {
 1705     return true;
 1706   }
 1707   return false;
 1708 }
 1709 
 1710 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1711   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1712   int lo = XMM0_num;
 1713   int hi = XMM0b_num;
 1714   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1715   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1716   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1717   return OptoRegPair(hi, lo);
 1718 }
 1719 
 1720 // Is this branch offset short enough that a short branch can be used?
 1721 //
 1722 // NOTE: If the platform does not provide any short branch variants, then
 1723 //       this method should return false for offset 0.
 1724 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1725   // The passed offset is relative to address of the branch.
 1726   // On 86 a branch displacement is calculated relative to address
 1727   // of a next instruction.
 1728   offset -= br_size;
 1729 
 1730   // the short version of jmpConUCF2 contains multiple branches,
 1731   // making the reach slightly less
 1732   if (rule == jmpConUCF2_rule)
 1733     return (-126 <= offset && offset <= 125);
 1734   return (-128 <= offset && offset <= 127);
 1735 }
 1736 
 1737 // Return whether or not this register is ever used as an argument.
 1738 // This function is used on startup to build the trampoline stubs in
 1739 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1740 // call in the trampoline, and arguments in those registers not be
 1741 // available to the callee.
 1742 bool Matcher::can_be_java_arg(int reg)
 1743 {
 1744   return
 1745     reg ==  RDI_num || reg == RDI_H_num ||
 1746     reg ==  RSI_num || reg == RSI_H_num ||
 1747     reg ==  RDX_num || reg == RDX_H_num ||
 1748     reg ==  RCX_num || reg == RCX_H_num ||
 1749     reg ==   R8_num || reg ==  R8_H_num ||
 1750     reg ==   R9_num || reg ==  R9_H_num ||
 1751     reg ==  R12_num || reg == R12_H_num ||
 1752     reg == XMM0_num || reg == XMM0b_num ||
 1753     reg == XMM1_num || reg == XMM1b_num ||
 1754     reg == XMM2_num || reg == XMM2b_num ||
 1755     reg == XMM3_num || reg == XMM3b_num ||
 1756     reg == XMM4_num || reg == XMM4b_num ||
 1757     reg == XMM5_num || reg == XMM5b_num ||
 1758     reg == XMM6_num || reg == XMM6b_num ||
 1759     reg == XMM7_num || reg == XMM7b_num;
 1760 }
 1761 
 1762 bool Matcher::is_spillable_arg(int reg)
 1763 {
 1764   return can_be_java_arg(reg);
 1765 }
 1766 
 1767 uint Matcher::int_pressure_limit()
 1768 {
 1769   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1770 }
 1771 
 1772 uint Matcher::float_pressure_limit()
 1773 {
 1774   // After experiment around with different values, the following default threshold
 1775   // works best for LCM's register pressure scheduling on x64.
 1776   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1777   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1778   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1779 }
 1780 
 1781 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1782   // In 64 bit mode a code which use multiply when
 1783   // devisor is constant is faster than hardware
 1784   // DIV instruction (it uses MulHiL).
 1785   return false;
 1786 }
 1787 
 1788 // Register for DIVI projection of divmodI
 1789 RegMask Matcher::divI_proj_mask() {
 1790   return INT_RAX_REG_mask();
 1791 }
 1792 
 1793 // Register for MODI projection of divmodI
 1794 RegMask Matcher::modI_proj_mask() {
 1795   return INT_RDX_REG_mask();
 1796 }
 1797 
 1798 // Register for DIVL projection of divmodL
 1799 RegMask Matcher::divL_proj_mask() {
 1800   return LONG_RAX_REG_mask();
 1801 }
 1802 
 1803 // Register for MODL projection of divmodL
 1804 RegMask Matcher::modL_proj_mask() {
 1805   return LONG_RDX_REG_mask();
 1806 }
 1807 
 1808 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1809 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1810     return NO_REG_mask();
 1811 }
 1812 
 1813 %}
 1814 
 1815 //----------ENCODING BLOCK-----------------------------------------------------
 1816 // This block specifies the encoding classes used by the compiler to
 1817 // output byte streams.  Encoding classes are parameterized macros
 1818 // used by Machine Instruction Nodes in order to generate the bit
 1819 // encoding of the instruction.  Operands specify their base encoding
 1820 // interface with the interface keyword.  There are currently
 1821 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1822 // COND_INTER.  REG_INTER causes an operand to generate a function
 1823 // which returns its register number when queried.  CONST_INTER causes
 1824 // an operand to generate a function which returns the value of the
 1825 // constant when queried.  MEMORY_INTER causes an operand to generate
 1826 // four functions which return the Base Register, the Index Register,
 1827 // the Scale Value, and the Offset Value of the operand when queried.
 1828 // COND_INTER causes an operand to generate six functions which return
 1829 // the encoding code (ie - encoding bits for the instruction)
 1830 // associated with each basic boolean condition for a conditional
 1831 // instruction.
 1832 //
 1833 // Instructions specify two basic values for encoding.  Again, a
 1834 // function is available to check if the constant displacement is an
 1835 // oop. They use the ins_encode keyword to specify their encoding
 1836 // classes (which must be a sequence of enc_class names, and their
 1837 // parameters, specified in the encoding block), and they use the
 1838 // opcode keyword to specify, in order, their primary, secondary, and
 1839 // tertiary opcode.  Only the opcode sections which a particular
 1840 // instruction needs for encoding need to be specified.
 1841 encode %{
 1842   // Build emit functions for each basic byte or larger field in the
 1843   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1844   // from C++ code in the enc_class source block.  Emit functions will
 1845   // live in the main source block for now.  In future, we can
 1846   // generalize this by adding a syntax that specifies the sizes of
 1847   // fields in an order, so that the adlc can build the emit functions
 1848   // automagically
 1849 
 1850   // Emit primary opcode
 1851   enc_class OpcP
 1852   %{
 1853     emit_opcode(cbuf, $primary);
 1854   %}
 1855 
 1856   // Emit secondary opcode
 1857   enc_class OpcS
 1858   %{
 1859     emit_opcode(cbuf, $secondary);
 1860   %}
 1861 
 1862   // Emit tertiary opcode
 1863   enc_class OpcT
 1864   %{
 1865     emit_opcode(cbuf, $tertiary);
 1866   %}
 1867 
 1868   // Emit opcode directly
 1869   enc_class Opcode(immI d8)
 1870   %{
 1871     emit_opcode(cbuf, $d8$$constant);
 1872   %}
 1873 
 1874   // Emit size prefix
 1875   enc_class SizePrefix
 1876   %{
 1877     emit_opcode(cbuf, 0x66);
 1878   %}
 1879 
 1880   enc_class reg(rRegI reg)
 1881   %{
 1882     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1883   %}
 1884 
 1885   enc_class reg_reg(rRegI dst, rRegI src)
 1886   %{
 1887     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1888   %}
 1889 
 1890   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1891   %{
 1892     emit_opcode(cbuf, $opcode$$constant);
 1893     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1894   %}
 1895 
 1896   enc_class cdql_enc(no_rax_rdx_RegI div)
 1897   %{
 1898     // Full implementation of Java idiv and irem; checks for
 1899     // special case as described in JVM spec., p.243 & p.271.
 1900     //
 1901     //         normal case                           special case
 1902     //
 1903     // input : rax: dividend                         min_int
 1904     //         reg: divisor                          -1
 1905     //
 1906     // output: rax: quotient  (= rax idiv reg)       min_int
 1907     //         rdx: remainder (= rax irem reg)       0
 1908     //
 1909     //  Code sequnce:
 1910     //
 1911     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1912     //    5:   75 07/08                jne    e <normal>
 1913     //    7:   33 d2                   xor    %edx,%edx
 1914     //  [div >= 8 -> offset + 1]
 1915     //  [REX_B]
 1916     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1917     //    c:   74 03/04                je     11 <done>
 1918     // 000000000000000e <normal>:
 1919     //    e:   99                      cltd
 1920     //  [div >= 8 -> offset + 1]
 1921     //  [REX_B]
 1922     //    f:   f7 f9                   idiv   $div
 1923     // 0000000000000011 <done>:
 1924     MacroAssembler _masm(&cbuf);
 1925     Label normal;
 1926     Label done;
 1927 
 1928     // cmp    $0x80000000,%eax
 1929     __ cmp(as_Register(RAX_enc), 0x80000000);
 1930 
 1931     // jne    e <normal>
 1932     __ jccb(Assembler::notEqual, normal);
 1933 
 1934     // xor    %edx,%edx
 1935     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1936 
 1937     // cmp    $0xffffffffffffffff,%ecx
 1938     __ cmpl($div$$Register, -1);
 1939 
 1940     // je     11 <done>
 1941     __ jccb(Assembler::equal, done);
 1942 
 1943     // <normal>
 1944     // cltd
 1945     __ bind(normal);
 1946     __ cdql();
 1947 
 1948     // idivl
 1949     // <done>
 1950     __ idivl($div$$Register);
 1951     __ bind(done);
 1952   %}
 1953 
 1954   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1955   %{
 1956     // Full implementation of Java ldiv and lrem; checks for
 1957     // special case as described in JVM spec., p.243 & p.271.
 1958     //
 1959     //         normal case                           special case
 1960     //
 1961     // input : rax: dividend                         min_long
 1962     //         reg: divisor                          -1
 1963     //
 1964     // output: rax: quotient  (= rax idiv reg)       min_long
 1965     //         rdx: remainder (= rax irem reg)       0
 1966     //
 1967     //  Code sequnce:
 1968     //
 1969     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1970     //    7:   00 00 80
 1971     //    a:   48 39 d0                cmp    %rdx,%rax
 1972     //    d:   75 08                   jne    17 <normal>
 1973     //    f:   33 d2                   xor    %edx,%edx
 1974     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1975     //   15:   74 05                   je     1c <done>
 1976     // 0000000000000017 <normal>:
 1977     //   17:   48 99                   cqto
 1978     //   19:   48 f7 f9                idiv   $div
 1979     // 000000000000001c <done>:
 1980     MacroAssembler _masm(&cbuf);
 1981     Label normal;
 1982     Label done;
 1983 
 1984     // mov    $0x8000000000000000,%rdx
 1985     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1986 
 1987     // cmp    %rdx,%rax
 1988     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1989 
 1990     // jne    17 <normal>
 1991     __ jccb(Assembler::notEqual, normal);
 1992 
 1993     // xor    %edx,%edx
 1994     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1995 
 1996     // cmp    $0xffffffffffffffff,$div
 1997     __ cmpq($div$$Register, -1);
 1998 
 1999     // je     1e <done>
 2000     __ jccb(Assembler::equal, done);
 2001 
 2002     // <normal>
 2003     // cqto
 2004     __ bind(normal);
 2005     __ cdqq();
 2006 
 2007     // idivq (note: must be emitted by the user of this rule)
 2008     // <done>
 2009     __ idivq($div$$Register);
 2010     __ bind(done);
 2011   %}
 2012 
 2013   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2014   enc_class OpcSE(immI imm)
 2015   %{
 2016     // Emit primary opcode and set sign-extend bit
 2017     // Check for 8-bit immediate, and set sign extend bit in opcode
 2018     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2019       emit_opcode(cbuf, $primary | 0x02);
 2020     } else {
 2021       // 32-bit immediate
 2022       emit_opcode(cbuf, $primary);
 2023     }
 2024   %}
 2025 
 2026   enc_class OpcSErm(rRegI dst, immI imm)
 2027   %{
 2028     // OpcSEr/m
 2029     int dstenc = $dst$$reg;
 2030     if (dstenc >= 8) {
 2031       emit_opcode(cbuf, Assembler::REX_B);
 2032       dstenc -= 8;
 2033     }
 2034     // Emit primary opcode and set sign-extend bit
 2035     // Check for 8-bit immediate, and set sign extend bit in opcode
 2036     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2037       emit_opcode(cbuf, $primary | 0x02);
 2038     } else {
 2039       // 32-bit immediate
 2040       emit_opcode(cbuf, $primary);
 2041     }
 2042     // Emit r/m byte with secondary opcode, after primary opcode.
 2043     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2044   %}
 2045 
 2046   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2047   %{
 2048     // OpcSEr/m
 2049     int dstenc = $dst$$reg;
 2050     if (dstenc < 8) {
 2051       emit_opcode(cbuf, Assembler::REX_W);
 2052     } else {
 2053       emit_opcode(cbuf, Assembler::REX_WB);
 2054       dstenc -= 8;
 2055     }
 2056     // Emit primary opcode and set sign-extend bit
 2057     // Check for 8-bit immediate, and set sign extend bit in opcode
 2058     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2059       emit_opcode(cbuf, $primary | 0x02);
 2060     } else {
 2061       // 32-bit immediate
 2062       emit_opcode(cbuf, $primary);
 2063     }
 2064     // Emit r/m byte with secondary opcode, after primary opcode.
 2065     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2066   %}
 2067 
 2068   enc_class Con8or32(immI imm)
 2069   %{
 2070     // Check for 8-bit immediate, and set sign extend bit in opcode
 2071     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2072       $$$emit8$imm$$constant;
 2073     } else {
 2074       // 32-bit immediate
 2075       $$$emit32$imm$$constant;
 2076     }
 2077   %}
 2078 
 2079   enc_class opc2_reg(rRegI dst)
 2080   %{
 2081     // BSWAP
 2082     emit_cc(cbuf, $secondary, $dst$$reg);
 2083   %}
 2084 
 2085   enc_class opc3_reg(rRegI dst)
 2086   %{
 2087     // BSWAP
 2088     emit_cc(cbuf, $tertiary, $dst$$reg);
 2089   %}
 2090 
 2091   enc_class reg_opc(rRegI div)
 2092   %{
 2093     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2094     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2095   %}
 2096 
 2097   enc_class enc_cmov(cmpOp cop)
 2098   %{
 2099     // CMOV
 2100     $$$emit8$primary;
 2101     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2102   %}
 2103 
 2104   enc_class enc_PartialSubtypeCheck()
 2105   %{
 2106     Register Rrdi = as_Register(RDI_enc); // result register
 2107     Register Rrax = as_Register(RAX_enc); // super class
 2108     Register Rrcx = as_Register(RCX_enc); // killed
 2109     Register Rrsi = as_Register(RSI_enc); // sub class
 2110     Label miss;
 2111     const bool set_cond_codes = true;
 2112 
 2113     MacroAssembler _masm(&cbuf);
 2114     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2115                                      NULL, &miss,
 2116                                      /*set_cond_codes:*/ true);
 2117     if ($primary) {
 2118       __ xorptr(Rrdi, Rrdi);
 2119     }
 2120     __ bind(miss);
 2121   %}
 2122 
 2123   enc_class clear_avx %{
 2124     debug_only(int off0 = cbuf.insts_size());
 2125     if (generate_vzeroupper(Compile::current())) {
 2126       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2127       // Clear upper bits of YMM registers when current compiled code uses
 2128       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2129       MacroAssembler _masm(&cbuf);
 2130       __ vzeroupper();
 2131     }
 2132     debug_only(int off1 = cbuf.insts_size());
 2133     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2134   %}
 2135 
 2136   enc_class Java_To_Runtime(method meth) %{
 2137     // No relocation needed
 2138     MacroAssembler _masm(&cbuf);
 2139     __ mov64(r10, (int64_t) $meth$$method);
 2140     __ call(r10);
 2141     __ post_call_nop();
 2142   %}
 2143 
 2144   enc_class Java_To_Interpreter(method meth)
 2145   %{
 2146     // CALL Java_To_Interpreter
 2147     // This is the instruction starting address for relocation info.
 2148     cbuf.set_insts_mark();
 2149     $$$emit8$primary;
 2150     // CALL directly to the runtime
 2151     emit_d32_reloc(cbuf,
 2152                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2153                    runtime_call_Relocation::spec(),
 2154                    RELOC_DISP32);
 2155     __ post_call_nop();
 2156   %}
 2157 
 2158   enc_class Java_Static_Call(method meth)
 2159   %{
 2160     // JAVA STATIC CALL
 2161     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2162     // determine who we intended to call.
 2163     MacroAssembler _masm(&cbuf);
 2164     cbuf.set_insts_mark();
 2165     $$$emit8$primary;
 2166 
 2167     if (!_method) {
 2168       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2169                      runtime_call_Relocation::spec(),
 2170                      RELOC_DISP32);
 2171     } else {
 2172       int method_index = resolved_method_index(cbuf);
 2173       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2174                                                   : static_call_Relocation::spec(method_index);
 2175       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2176                      rspec, RELOC_DISP32);
 2177       // Emit stubs for static call.
 2178       address mark = cbuf.insts_mark();
 2179       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2180       if (stub == NULL) {
 2181         ciEnv::current()->record_failure("CodeCache is full");
 2182         return;
 2183       }
 2184     }
 2185     __ post_call_nop();
 2186   %}
 2187 
 2188   enc_class Java_Dynamic_Call(method meth) %{
 2189     MacroAssembler _masm(&cbuf);
 2190     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2191     __ post_call_nop();
 2192   %}
 2193 
 2194   enc_class Java_Compiled_Call(method meth)
 2195   %{
 2196     // JAVA COMPILED CALL
 2197     int disp = in_bytes(Method:: from_compiled_offset());
 2198 
 2199     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2200     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2201 
 2202     // callq *disp(%rax)
 2203     MacroAssembler _masm(&cbuf);
 2204     cbuf.set_insts_mark();
 2205     $$$emit8$primary;
 2206     if (disp < 0x80) {
 2207       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2208       emit_d8(cbuf, disp); // Displacement
 2209     } else {
 2210       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2211       emit_d32(cbuf, disp); // Displacement
 2212     }
 2213     __ post_call_nop();
 2214   %}
 2215 
 2216   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2217   %{
 2218     // SAL, SAR, SHR
 2219     int dstenc = $dst$$reg;
 2220     if (dstenc >= 8) {
 2221       emit_opcode(cbuf, Assembler::REX_B);
 2222       dstenc -= 8;
 2223     }
 2224     $$$emit8$primary;
 2225     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2226     $$$emit8$shift$$constant;
 2227   %}
 2228 
 2229   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2230   %{
 2231     // SAL, SAR, SHR
 2232     int dstenc = $dst$$reg;
 2233     if (dstenc < 8) {
 2234       emit_opcode(cbuf, Assembler::REX_W);
 2235     } else {
 2236       emit_opcode(cbuf, Assembler::REX_WB);
 2237       dstenc -= 8;
 2238     }
 2239     $$$emit8$primary;
 2240     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2241     $$$emit8$shift$$constant;
 2242   %}
 2243 
 2244   enc_class load_immI(rRegI dst, immI src)
 2245   %{
 2246     int dstenc = $dst$$reg;
 2247     if (dstenc >= 8) {
 2248       emit_opcode(cbuf, Assembler::REX_B);
 2249       dstenc -= 8;
 2250     }
 2251     emit_opcode(cbuf, 0xB8 | dstenc);
 2252     $$$emit32$src$$constant;
 2253   %}
 2254 
 2255   enc_class load_immL(rRegL dst, immL src)
 2256   %{
 2257     int dstenc = $dst$$reg;
 2258     if (dstenc < 8) {
 2259       emit_opcode(cbuf, Assembler::REX_W);
 2260     } else {
 2261       emit_opcode(cbuf, Assembler::REX_WB);
 2262       dstenc -= 8;
 2263     }
 2264     emit_opcode(cbuf, 0xB8 | dstenc);
 2265     emit_d64(cbuf, $src$$constant);
 2266   %}
 2267 
 2268   enc_class load_immUL32(rRegL dst, immUL32 src)
 2269   %{
 2270     // same as load_immI, but this time we care about zeroes in the high word
 2271     int dstenc = $dst$$reg;
 2272     if (dstenc >= 8) {
 2273       emit_opcode(cbuf, Assembler::REX_B);
 2274       dstenc -= 8;
 2275     }
 2276     emit_opcode(cbuf, 0xB8 | dstenc);
 2277     $$$emit32$src$$constant;
 2278   %}
 2279 
 2280   enc_class load_immL32(rRegL dst, immL32 src)
 2281   %{
 2282     int dstenc = $dst$$reg;
 2283     if (dstenc < 8) {
 2284       emit_opcode(cbuf, Assembler::REX_W);
 2285     } else {
 2286       emit_opcode(cbuf, Assembler::REX_WB);
 2287       dstenc -= 8;
 2288     }
 2289     emit_opcode(cbuf, 0xC7);
 2290     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2291     $$$emit32$src$$constant;
 2292   %}
 2293 
 2294   enc_class load_immP31(rRegP dst, immP32 src)
 2295   %{
 2296     // same as load_immI, but this time we care about zeroes in the high word
 2297     int dstenc = $dst$$reg;
 2298     if (dstenc >= 8) {
 2299       emit_opcode(cbuf, Assembler::REX_B);
 2300       dstenc -= 8;
 2301     }
 2302     emit_opcode(cbuf, 0xB8 | dstenc);
 2303     $$$emit32$src$$constant;
 2304   %}
 2305 
 2306   enc_class load_immP(rRegP dst, immP src)
 2307   %{
 2308     int dstenc = $dst$$reg;
 2309     if (dstenc < 8) {
 2310       emit_opcode(cbuf, Assembler::REX_W);
 2311     } else {
 2312       emit_opcode(cbuf, Assembler::REX_WB);
 2313       dstenc -= 8;
 2314     }
 2315     emit_opcode(cbuf, 0xB8 | dstenc);
 2316     // This next line should be generated from ADLC
 2317     if ($src->constant_reloc() != relocInfo::none) {
 2318       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2319     } else {
 2320       emit_d64(cbuf, $src$$constant);
 2321     }
 2322   %}
 2323 
 2324   enc_class Con32(immI src)
 2325   %{
 2326     // Output immediate
 2327     $$$emit32$src$$constant;
 2328   %}
 2329 
 2330   enc_class Con32F_as_bits(immF src)
 2331   %{
 2332     // Output Float immediate bits
 2333     jfloat jf = $src$$constant;
 2334     jint jf_as_bits = jint_cast(jf);
 2335     emit_d32(cbuf, jf_as_bits);
 2336   %}
 2337 
 2338   enc_class Con16(immI src)
 2339   %{
 2340     // Output immediate
 2341     $$$emit16$src$$constant;
 2342   %}
 2343 
 2344   // How is this different from Con32??? XXX
 2345   enc_class Con_d32(immI src)
 2346   %{
 2347     emit_d32(cbuf,$src$$constant);
 2348   %}
 2349 
 2350   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2351     // Output immediate memory reference
 2352     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2353     emit_d32(cbuf, 0x00);
 2354   %}
 2355 
 2356   enc_class lock_prefix()
 2357   %{
 2358     emit_opcode(cbuf, 0xF0); // lock
 2359   %}
 2360 
 2361   enc_class REX_mem(memory mem)
 2362   %{
 2363     if ($mem$$base >= 8) {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_B);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_XB);
 2368       }
 2369     } else {
 2370       if ($mem$$index >= 8) {
 2371         emit_opcode(cbuf, Assembler::REX_X);
 2372       }
 2373     }
 2374   %}
 2375 
 2376   enc_class REX_mem_wide(memory mem)
 2377   %{
 2378     if ($mem$$base >= 8) {
 2379       if ($mem$$index < 8) {
 2380         emit_opcode(cbuf, Assembler::REX_WB);
 2381       } else {
 2382         emit_opcode(cbuf, Assembler::REX_WXB);
 2383       }
 2384     } else {
 2385       if ($mem$$index < 8) {
 2386         emit_opcode(cbuf, Assembler::REX_W);
 2387       } else {
 2388         emit_opcode(cbuf, Assembler::REX_WX);
 2389       }
 2390     }
 2391   %}
 2392 
 2393   // for byte regs
 2394   enc_class REX_breg(rRegI reg)
 2395   %{
 2396     if ($reg$$reg >= 4) {
 2397       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2398     }
 2399   %}
 2400 
 2401   // for byte regs
 2402   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2403   %{
 2404     if ($dst$$reg < 8) {
 2405       if ($src$$reg >= 4) {
 2406         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2407       }
 2408     } else {
 2409       if ($src$$reg < 8) {
 2410         emit_opcode(cbuf, Assembler::REX_R);
 2411       } else {
 2412         emit_opcode(cbuf, Assembler::REX_RB);
 2413       }
 2414     }
 2415   %}
 2416 
 2417   // for byte regs
 2418   enc_class REX_breg_mem(rRegI reg, memory mem)
 2419   %{
 2420     if ($reg$$reg < 8) {
 2421       if ($mem$$base < 8) {
 2422         if ($mem$$index >= 8) {
 2423           emit_opcode(cbuf, Assembler::REX_X);
 2424         } else if ($reg$$reg >= 4) {
 2425           emit_opcode(cbuf, Assembler::REX);
 2426         }
 2427       } else {
 2428         if ($mem$$index < 8) {
 2429           emit_opcode(cbuf, Assembler::REX_B);
 2430         } else {
 2431           emit_opcode(cbuf, Assembler::REX_XB);
 2432         }
 2433       }
 2434     } else {
 2435       if ($mem$$base < 8) {
 2436         if ($mem$$index < 8) {
 2437           emit_opcode(cbuf, Assembler::REX_R);
 2438         } else {
 2439           emit_opcode(cbuf, Assembler::REX_RX);
 2440         }
 2441       } else {
 2442         if ($mem$$index < 8) {
 2443           emit_opcode(cbuf, Assembler::REX_RB);
 2444         } else {
 2445           emit_opcode(cbuf, Assembler::REX_RXB);
 2446         }
 2447       }
 2448     }
 2449   %}
 2450 
 2451   enc_class REX_reg(rRegI reg)
 2452   %{
 2453     if ($reg$$reg >= 8) {
 2454       emit_opcode(cbuf, Assembler::REX_B);
 2455     }
 2456   %}
 2457 
 2458   enc_class REX_reg_wide(rRegI reg)
 2459   %{
 2460     if ($reg$$reg < 8) {
 2461       emit_opcode(cbuf, Assembler::REX_W);
 2462     } else {
 2463       emit_opcode(cbuf, Assembler::REX_WB);
 2464     }
 2465   %}
 2466 
 2467   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2468   %{
 2469     if ($dst$$reg < 8) {
 2470       if ($src$$reg >= 8) {
 2471         emit_opcode(cbuf, Assembler::REX_B);
 2472       }
 2473     } else {
 2474       if ($src$$reg < 8) {
 2475         emit_opcode(cbuf, Assembler::REX_R);
 2476       } else {
 2477         emit_opcode(cbuf, Assembler::REX_RB);
 2478       }
 2479     }
 2480   %}
 2481 
 2482   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2483   %{
 2484     if ($dst$$reg < 8) {
 2485       if ($src$$reg < 8) {
 2486         emit_opcode(cbuf, Assembler::REX_W);
 2487       } else {
 2488         emit_opcode(cbuf, Assembler::REX_WB);
 2489       }
 2490     } else {
 2491       if ($src$$reg < 8) {
 2492         emit_opcode(cbuf, Assembler::REX_WR);
 2493       } else {
 2494         emit_opcode(cbuf, Assembler::REX_WRB);
 2495       }
 2496     }
 2497   %}
 2498 
 2499   enc_class REX_reg_mem(rRegI reg, memory mem)
 2500   %{
 2501     if ($reg$$reg < 8) {
 2502       if ($mem$$base < 8) {
 2503         if ($mem$$index >= 8) {
 2504           emit_opcode(cbuf, Assembler::REX_X);
 2505         }
 2506       } else {
 2507         if ($mem$$index < 8) {
 2508           emit_opcode(cbuf, Assembler::REX_B);
 2509         } else {
 2510           emit_opcode(cbuf, Assembler::REX_XB);
 2511         }
 2512       }
 2513     } else {
 2514       if ($mem$$base < 8) {
 2515         if ($mem$$index < 8) {
 2516           emit_opcode(cbuf, Assembler::REX_R);
 2517         } else {
 2518           emit_opcode(cbuf, Assembler::REX_RX);
 2519         }
 2520       } else {
 2521         if ($mem$$index < 8) {
 2522           emit_opcode(cbuf, Assembler::REX_RB);
 2523         } else {
 2524           emit_opcode(cbuf, Assembler::REX_RXB);
 2525         }
 2526       }
 2527     }
 2528   %}
 2529 
 2530   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2531   %{
 2532     if ($reg$$reg < 8) {
 2533       if ($mem$$base < 8) {
 2534         if ($mem$$index < 8) {
 2535           emit_opcode(cbuf, Assembler::REX_W);
 2536         } else {
 2537           emit_opcode(cbuf, Assembler::REX_WX);
 2538         }
 2539       } else {
 2540         if ($mem$$index < 8) {
 2541           emit_opcode(cbuf, Assembler::REX_WB);
 2542         } else {
 2543           emit_opcode(cbuf, Assembler::REX_WXB);
 2544         }
 2545       }
 2546     } else {
 2547       if ($mem$$base < 8) {
 2548         if ($mem$$index < 8) {
 2549           emit_opcode(cbuf, Assembler::REX_WR);
 2550         } else {
 2551           emit_opcode(cbuf, Assembler::REX_WRX);
 2552         }
 2553       } else {
 2554         if ($mem$$index < 8) {
 2555           emit_opcode(cbuf, Assembler::REX_WRB);
 2556         } else {
 2557           emit_opcode(cbuf, Assembler::REX_WRXB);
 2558         }
 2559       }
 2560     }
 2561   %}
 2562 
 2563   enc_class reg_mem(rRegI ereg, memory mem)
 2564   %{
 2565     // High registers handle in encode_RegMem
 2566     int reg = $ereg$$reg;
 2567     int base = $mem$$base;
 2568     int index = $mem$$index;
 2569     int scale = $mem$$scale;
 2570     int disp = $mem$$disp;
 2571     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2572 
 2573     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2574   %}
 2575 
 2576   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2577   %{
 2578     int rm_byte_opcode = $rm_opcode$$constant;
 2579 
 2580     // High registers handle in encode_RegMem
 2581     int base = $mem$$base;
 2582     int index = $mem$$index;
 2583     int scale = $mem$$scale;
 2584     int displace = $mem$$disp;
 2585 
 2586     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2587                                             // working with static
 2588                                             // globals
 2589     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2590                   disp_reloc);
 2591   %}
 2592 
 2593   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2594   %{
 2595     int reg_encoding = $dst$$reg;
 2596     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2597     int index        = 0x04;            // 0x04 indicates no index
 2598     int scale        = 0x00;            // 0x00 indicates no scale
 2599     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2600     relocInfo::relocType disp_reloc = relocInfo::none;
 2601     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2602                   disp_reloc);
 2603   %}
 2604 
 2605   enc_class neg_reg(rRegI dst)
 2606   %{
 2607     int dstenc = $dst$$reg;
 2608     if (dstenc >= 8) {
 2609       emit_opcode(cbuf, Assembler::REX_B);
 2610       dstenc -= 8;
 2611     }
 2612     // NEG $dst
 2613     emit_opcode(cbuf, 0xF7);
 2614     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2615   %}
 2616 
 2617   enc_class neg_reg_wide(rRegI dst)
 2618   %{
 2619     int dstenc = $dst$$reg;
 2620     if (dstenc < 8) {
 2621       emit_opcode(cbuf, Assembler::REX_W);
 2622     } else {
 2623       emit_opcode(cbuf, Assembler::REX_WB);
 2624       dstenc -= 8;
 2625     }
 2626     // NEG $dst
 2627     emit_opcode(cbuf, 0xF7);
 2628     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2629   %}
 2630 
 2631   enc_class setLT_reg(rRegI dst)
 2632   %{
 2633     int dstenc = $dst$$reg;
 2634     if (dstenc >= 8) {
 2635       emit_opcode(cbuf, Assembler::REX_B);
 2636       dstenc -= 8;
 2637     } else if (dstenc >= 4) {
 2638       emit_opcode(cbuf, Assembler::REX);
 2639     }
 2640     // SETLT $dst
 2641     emit_opcode(cbuf, 0x0F);
 2642     emit_opcode(cbuf, 0x9C);
 2643     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2644   %}
 2645 
 2646   enc_class setNZ_reg(rRegI dst)
 2647   %{
 2648     int dstenc = $dst$$reg;
 2649     if (dstenc >= 8) {
 2650       emit_opcode(cbuf, Assembler::REX_B);
 2651       dstenc -= 8;
 2652     } else if (dstenc >= 4) {
 2653       emit_opcode(cbuf, Assembler::REX);
 2654     }
 2655     // SETNZ $dst
 2656     emit_opcode(cbuf, 0x0F);
 2657     emit_opcode(cbuf, 0x95);
 2658     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2659   %}
 2660 
 2661 
 2662   // Compare the lonogs and set -1, 0, or 1 into dst
 2663   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2664   %{
 2665     int src1enc = $src1$$reg;
 2666     int src2enc = $src2$$reg;
 2667     int dstenc = $dst$$reg;
 2668 
 2669     // cmpq $src1, $src2
 2670     if (src1enc < 8) {
 2671       if (src2enc < 8) {
 2672         emit_opcode(cbuf, Assembler::REX_W);
 2673       } else {
 2674         emit_opcode(cbuf, Assembler::REX_WB);
 2675       }
 2676     } else {
 2677       if (src2enc < 8) {
 2678         emit_opcode(cbuf, Assembler::REX_WR);
 2679       } else {
 2680         emit_opcode(cbuf, Assembler::REX_WRB);
 2681       }
 2682     }
 2683     emit_opcode(cbuf, 0x3B);
 2684     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2685 
 2686     // movl $dst, -1
 2687     if (dstenc >= 8) {
 2688       emit_opcode(cbuf, Assembler::REX_B);
 2689     }
 2690     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2691     emit_d32(cbuf, -1);
 2692 
 2693     // jl,s done
 2694     emit_opcode(cbuf, 0x7C);
 2695     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2696 
 2697     // setne $dst
 2698     if (dstenc >= 4) {
 2699       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2700     }
 2701     emit_opcode(cbuf, 0x0F);
 2702     emit_opcode(cbuf, 0x95);
 2703     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2704 
 2705     // movzbl $dst, $dst
 2706     if (dstenc >= 4) {
 2707       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2708     }
 2709     emit_opcode(cbuf, 0x0F);
 2710     emit_opcode(cbuf, 0xB6);
 2711     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2712   %}
 2713 
 2714   enc_class Push_ResultXD(regD dst) %{
 2715     MacroAssembler _masm(&cbuf);
 2716     __ fstp_d(Address(rsp, 0));
 2717     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2718     __ addptr(rsp, 8);
 2719   %}
 2720 
 2721   enc_class Push_SrcXD(regD src) %{
 2722     MacroAssembler _masm(&cbuf);
 2723     __ subptr(rsp, 8);
 2724     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2725     __ fld_d(Address(rsp, 0));
 2726   %}
 2727 
 2728 
 2729   enc_class enc_rethrow()
 2730   %{
 2731     cbuf.set_insts_mark();
 2732     emit_opcode(cbuf, 0xE9); // jmp entry
 2733     emit_d32_reloc(cbuf,
 2734                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2735                    runtime_call_Relocation::spec(),
 2736                    RELOC_DISP32);
 2737   %}
 2738 
 2739 %}
 2740 
 2741 
 2742 
 2743 //----------FRAME--------------------------------------------------------------
 2744 // Definition of frame structure and management information.
 2745 //
 2746 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2747 //                             |   (to get allocators register number
 2748 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2749 //  r   CALLER     |        |
 2750 //  o     |        +--------+      pad to even-align allocators stack-slot
 2751 //  w     V        |  pad0  |        numbers; owned by CALLER
 2752 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2753 //  h     ^        |   in   |  5
 2754 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2755 //  |     |        |        |  3
 2756 //  |     |        +--------+
 2757 //  V     |        | old out|      Empty on Intel, window on Sparc
 2758 //        |    old |preserve|      Must be even aligned.
 2759 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2760 //        |        |   in   |  3   area for Intel ret address
 2761 //     Owned by    |preserve|      Empty on Sparc.
 2762 //       SELF      +--------+
 2763 //        |        |  pad2  |  2   pad to align old SP
 2764 //        |        +--------+  1
 2765 //        |        | locks  |  0
 2766 //        |        +--------+----> OptoReg::stack0(), even aligned
 2767 //        |        |  pad1  | 11   pad to align new SP
 2768 //        |        +--------+
 2769 //        |        |        | 10
 2770 //        |        | spills |  9   spills
 2771 //        V        |        |  8   (pad0 slot for callee)
 2772 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2773 //        ^        |  out   |  7
 2774 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2775 //     Owned by    +--------+
 2776 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2777 //        |    new |preserve|      Must be even-aligned.
 2778 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2779 //        |        |        |
 2780 //
 2781 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2782 //         known from SELF's arguments and the Java calling convention.
 2783 //         Region 6-7 is determined per call site.
 2784 // Note 2: If the calling convention leaves holes in the incoming argument
 2785 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2786 //         are owned by the CALLEE.  Holes should not be nessecary in the
 2787 //         incoming area, as the Java calling convention is completely under
 2788 //         the control of the AD file.  Doubles can be sorted and packed to
 2789 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 2790 //         varargs C calling conventions.
 2791 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2792 //         even aligned with pad0 as needed.
 2793 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2794 //         region 6-11 is even aligned; it may be padded out more so that
 2795 //         the region from SP to FP meets the minimum stack alignment.
 2796 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2797 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2798 //         SP meets the minimum alignment.
 2799 
 2800 frame
 2801 %{
 2802   // These three registers define part of the calling convention
 2803   // between compiled code and the interpreter.
 2804   inline_cache_reg(RAX);                // Inline Cache Register
 2805 
 2806   // Optional: name the operand used by cisc-spilling to access
 2807   // [stack_pointer + offset]
 2808   cisc_spilling_operand_name(indOffset32);
 2809 
 2810   // Number of stack slots consumed by locking an object
 2811   sync_stack_slots(2);
 2812 
 2813   // Compiled code's Frame Pointer
 2814   frame_pointer(RSP);
 2815 
 2816   // Interpreter stores its frame pointer in a register which is
 2817   // stored to the stack by I2CAdaptors.
 2818   // I2CAdaptors convert from interpreted java to compiled java.
 2819   interpreter_frame_pointer(RBP);
 2820 
 2821   // Stack alignment requirement
 2822   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2823 
 2824   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2825   // for calls to C.  Supports the var-args backing area for register parms.
 2826   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2827 
 2828   // The after-PROLOG location of the return address.  Location of
 2829   // return address specifies a type (REG or STACK) and a number
 2830   // representing the register number (i.e. - use a register name) or
 2831   // stack slot.
 2832   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2833   // Otherwise, it is above the locks and verification slot and alignment word
 2834   return_addr(STACK - 2 +
 2835               align_up((Compile::current()->in_preserve_stack_slots() +
 2836                         Compile::current()->fixed_slots()),
 2837                        stack_alignment_in_slots()));
 2838 
 2839   // Location of compiled Java return values.  Same as C for now.
 2840   return_value
 2841   %{
 2842     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2843            "only return normal values");
 2844 
 2845     static const int lo[Op_RegL + 1] = {
 2846       0,
 2847       0,
 2848       RAX_num,  // Op_RegN
 2849       RAX_num,  // Op_RegI
 2850       RAX_num,  // Op_RegP
 2851       XMM0_num, // Op_RegF
 2852       XMM0_num, // Op_RegD
 2853       RAX_num   // Op_RegL
 2854     };
 2855     static const int hi[Op_RegL + 1] = {
 2856       0,
 2857       0,
 2858       OptoReg::Bad, // Op_RegN
 2859       OptoReg::Bad, // Op_RegI
 2860       RAX_H_num,    // Op_RegP
 2861       OptoReg::Bad, // Op_RegF
 2862       XMM0b_num,    // Op_RegD
 2863       RAX_H_num     // Op_RegL
 2864     };
 2865     // Excluded flags and vector registers.
 2866     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2867     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2868   %}
 2869 %}
 2870 
 2871 //----------ATTRIBUTES---------------------------------------------------------
 2872 //----------Operand Attributes-------------------------------------------------
 2873 op_attrib op_cost(0);        // Required cost attribute
 2874 
 2875 //----------Instruction Attributes---------------------------------------------
 2876 ins_attrib ins_cost(100);       // Required cost attribute
 2877 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2878 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2879                                 // a non-matching short branch variant
 2880                                 // of some long branch?
 2881 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2882                                 // be a power of 2) specifies the
 2883                                 // alignment that some part of the
 2884                                 // instruction (not necessarily the
 2885                                 // start) requires.  If > 1, a
 2886                                 // compute_padding() function must be
 2887                                 // provided for the instruction
 2888 
 2889 //----------OPERANDS-----------------------------------------------------------
 2890 // Operand definitions must precede instruction definitions for correct parsing
 2891 // in the ADLC because operands constitute user defined types which are used in
 2892 // instruction definitions.
 2893 
 2894 //----------Simple Operands----------------------------------------------------
 2895 // Immediate Operands
 2896 // Integer Immediate
 2897 operand immI()
 2898 %{
 2899   match(ConI);
 2900 
 2901   op_cost(10);
 2902   format %{ %}
 2903   interface(CONST_INTER);
 2904 %}
 2905 
 2906 // Constant for test vs zero
 2907 operand immI_0()
 2908 %{
 2909   predicate(n->get_int() == 0);
 2910   match(ConI);
 2911 
 2912   op_cost(0);
 2913   format %{ %}
 2914   interface(CONST_INTER);
 2915 %}
 2916 
 2917 // Constant for increment
 2918 operand immI_1()
 2919 %{
 2920   predicate(n->get_int() == 1);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 // Constant for decrement
 2929 operand immI_M1()
 2930 %{
 2931   predicate(n->get_int() == -1);
 2932   match(ConI);
 2933 
 2934   op_cost(0);
 2935   format %{ %}
 2936   interface(CONST_INTER);
 2937 %}
 2938 
 2939 operand immI_2()
 2940 %{
 2941   predicate(n->get_int() == 2);
 2942   match(ConI);
 2943 
 2944   op_cost(0);
 2945   format %{ %}
 2946   interface(CONST_INTER);
 2947 %}
 2948 
 2949 operand immI_4()
 2950 %{
 2951   predicate(n->get_int() == 4);
 2952   match(ConI);
 2953 
 2954   op_cost(0);
 2955   format %{ %}
 2956   interface(CONST_INTER);
 2957 %}
 2958 
 2959 operand immI_8()
 2960 %{
 2961   predicate(n->get_int() == 8);
 2962   match(ConI);
 2963 
 2964   op_cost(0);
 2965   format %{ %}
 2966   interface(CONST_INTER);
 2967 %}
 2968 
 2969 // Valid scale values for addressing modes
 2970 operand immI2()
 2971 %{
 2972   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2973   match(ConI);
 2974 
 2975   format %{ %}
 2976   interface(CONST_INTER);
 2977 %}
 2978 
 2979 operand immU7()
 2980 %{
 2981   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2982   match(ConI);
 2983 
 2984   op_cost(5);
 2985   format %{ %}
 2986   interface(CONST_INTER);
 2987 %}
 2988 
 2989 operand immI8()
 2990 %{
 2991   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2992   match(ConI);
 2993 
 2994   op_cost(5);
 2995   format %{ %}
 2996   interface(CONST_INTER);
 2997 %}
 2998 
 2999 operand immU8()
 3000 %{
 3001   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 3002   match(ConI);
 3003 
 3004   op_cost(5);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 operand immI16()
 3010 %{
 3011   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 3012   match(ConI);
 3013 
 3014   op_cost(10);
 3015   format %{ %}
 3016   interface(CONST_INTER);
 3017 %}
 3018 
 3019 // Int Immediate non-negative
 3020 operand immU31()
 3021 %{
 3022   predicate(n->get_int() >= 0);
 3023   match(ConI);
 3024 
 3025   op_cost(0);
 3026   format %{ %}
 3027   interface(CONST_INTER);
 3028 %}
 3029 
 3030 // Constant for long shifts
 3031 operand immI_32()
 3032 %{
 3033   predicate( n->get_int() == 32 );
 3034   match(ConI);
 3035 
 3036   op_cost(0);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // Constant for long shifts
 3042 operand immI_64()
 3043 %{
 3044   predicate( n->get_int() == 64 );
 3045   match(ConI);
 3046 
 3047   op_cost(0);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immP()
 3054 %{
 3055   match(ConP);
 3056 
 3057   op_cost(10);
 3058   format %{ %}
 3059   interface(CONST_INTER);
 3060 %}
 3061 
 3062 // NULL Pointer Immediate
 3063 operand immP0()
 3064 %{
 3065   predicate(n->get_ptr() == 0);
 3066   match(ConP);
 3067 
 3068   op_cost(5);
 3069   format %{ %}
 3070   interface(CONST_INTER);
 3071 %}
 3072 
 3073 // Pointer Immediate
 3074 operand immN() %{
 3075   match(ConN);
 3076 
 3077   op_cost(10);
 3078   format %{ %}
 3079   interface(CONST_INTER);
 3080 %}
 3081 
 3082 operand immNKlass() %{
 3083   match(ConNKlass);
 3084 
 3085   op_cost(10);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 // NULL Pointer Immediate
 3091 operand immN0() %{
 3092   predicate(n->get_narrowcon() == 0);
 3093   match(ConN);
 3094 
 3095   op_cost(5);
 3096   format %{ %}
 3097   interface(CONST_INTER);
 3098 %}
 3099 
 3100 operand immP31()
 3101 %{
 3102   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3103             && (n->get_ptr() >> 31) == 0);
 3104   match(ConP);
 3105 
 3106   op_cost(5);
 3107   format %{ %}
 3108   interface(CONST_INTER);
 3109 %}
 3110 
 3111 
 3112 // Long Immediate
 3113 operand immL()
 3114 %{
 3115   match(ConL);
 3116 
 3117   op_cost(20);
 3118   format %{ %}
 3119   interface(CONST_INTER);
 3120 %}
 3121 
 3122 // Long Immediate 8-bit
 3123 operand immL8()
 3124 %{
 3125   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3126   match(ConL);
 3127 
 3128   op_cost(5);
 3129   format %{ %}
 3130   interface(CONST_INTER);
 3131 %}
 3132 
 3133 // Long Immediate 32-bit unsigned
 3134 operand immUL32()
 3135 %{
 3136   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(10);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 // Long Immediate 32-bit signed
 3145 operand immL32()
 3146 %{
 3147   predicate(n->get_long() == (int) (n->get_long()));
 3148   match(ConL);
 3149 
 3150   op_cost(15);
 3151   format %{ %}
 3152   interface(CONST_INTER);
 3153 %}
 3154 
 3155 operand immL_Pow2()
 3156 %{
 3157   predicate(is_power_of_2((julong)n->get_long()));
 3158   match(ConL);
 3159 
 3160   op_cost(15);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 operand immL_NotPow2()
 3166 %{
 3167   predicate(is_power_of_2((julong)~n->get_long()));
 3168   match(ConL);
 3169 
 3170   op_cost(15);
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Long Immediate zero
 3176 operand immL0()
 3177 %{
 3178   predicate(n->get_long() == 0L);
 3179   match(ConL);
 3180 
 3181   op_cost(10);
 3182   format %{ %}
 3183   interface(CONST_INTER);
 3184 %}
 3185 
 3186 // Constant for increment
 3187 operand immL1()
 3188 %{
 3189   predicate(n->get_long() == 1);
 3190   match(ConL);
 3191 
 3192   format %{ %}
 3193   interface(CONST_INTER);
 3194 %}
 3195 
 3196 // Constant for decrement
 3197 operand immL_M1()
 3198 %{
 3199   predicate(n->get_long() == -1);
 3200   match(ConL);
 3201 
 3202   format %{ %}
 3203   interface(CONST_INTER);
 3204 %}
 3205 
 3206 // Long Immediate: the value 10
 3207 operand immL10()
 3208 %{
 3209   predicate(n->get_long() == 10);
 3210   match(ConL);
 3211 
 3212   format %{ %}
 3213   interface(CONST_INTER);
 3214 %}
 3215 
 3216 // Long immediate from 0 to 127.
 3217 // Used for a shorter form of long mul by 10.
 3218 operand immL_127()
 3219 %{
 3220   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3221   match(ConL);
 3222 
 3223   op_cost(10);
 3224   format %{ %}
 3225   interface(CONST_INTER);
 3226 %}
 3227 
 3228 // Long Immediate: low 32-bit mask
 3229 operand immL_32bits()
 3230 %{
 3231   predicate(n->get_long() == 0xFFFFFFFFL);
 3232   match(ConL);
 3233   op_cost(20);
 3234 
 3235   format %{ %}
 3236   interface(CONST_INTER);
 3237 %}
 3238 
 3239 // Int Immediate: 2^n-1, postive
 3240 operand immI_Pow2M1()
 3241 %{
 3242   predicate((n->get_int() > 0)
 3243             && is_power_of_2(n->get_int() + 1));
 3244   match(ConI);
 3245 
 3246   op_cost(20);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Float Immediate zero
 3252 operand immF0()
 3253 %{
 3254   predicate(jint_cast(n->getf()) == 0);
 3255   match(ConF);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Float Immediate
 3263 operand immF()
 3264 %{
 3265   match(ConF);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Double Immediate zero
 3273 operand immD0()
 3274 %{
 3275   predicate(jlong_cast(n->getd()) == 0);
 3276   match(ConD);
 3277 
 3278   op_cost(5);
 3279   format %{ %}
 3280   interface(CONST_INTER);
 3281 %}
 3282 
 3283 // Double Immediate
 3284 operand immD()
 3285 %{
 3286   match(ConD);
 3287 
 3288   op_cost(15);
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Immediates for special shifts (sign extend)
 3294 
 3295 // Constants for increment
 3296 operand immI_16()
 3297 %{
 3298   predicate(n->get_int() == 16);
 3299   match(ConI);
 3300 
 3301   format %{ %}
 3302   interface(CONST_INTER);
 3303 %}
 3304 
 3305 operand immI_24()
 3306 %{
 3307   predicate(n->get_int() == 24);
 3308   match(ConI);
 3309 
 3310   format %{ %}
 3311   interface(CONST_INTER);
 3312 %}
 3313 
 3314 // Constant for byte-wide masking
 3315 operand immI_255()
 3316 %{
 3317   predicate(n->get_int() == 255);
 3318   match(ConI);
 3319 
 3320   format %{ %}
 3321   interface(CONST_INTER);
 3322 %}
 3323 
 3324 // Constant for short-wide masking
 3325 operand immI_65535()
 3326 %{
 3327   predicate(n->get_int() == 65535);
 3328   match(ConI);
 3329 
 3330   format %{ %}
 3331   interface(CONST_INTER);
 3332 %}
 3333 
 3334 // Constant for byte-wide masking
 3335 operand immL_255()
 3336 %{
 3337   predicate(n->get_long() == 255);
 3338   match(ConL);
 3339 
 3340   format %{ %}
 3341   interface(CONST_INTER);
 3342 %}
 3343 
 3344 // Constant for short-wide masking
 3345 operand immL_65535()
 3346 %{
 3347   predicate(n->get_long() == 65535);
 3348   match(ConL);
 3349 
 3350   format %{ %}
 3351   interface(CONST_INTER);
 3352 %}
 3353 
 3354 operand kReg()
 3355 %{
 3356   constraint(ALLOC_IN_RC(vectmask_reg));
 3357   match(RegVectMask);
 3358   format %{%}
 3359   interface(REG_INTER);
 3360 %}
 3361 
 3362 operand kReg_K1()
 3363 %{
 3364   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3365   match(RegVectMask);
 3366   format %{%}
 3367   interface(REG_INTER);
 3368 %}
 3369 
 3370 operand kReg_K2()
 3371 %{
 3372   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3373   match(RegVectMask);
 3374   format %{%}
 3375   interface(REG_INTER);
 3376 %}
 3377 
 3378 // Special Registers
 3379 operand kReg_K3()
 3380 %{
 3381   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3382   match(RegVectMask);
 3383   format %{%}
 3384   interface(REG_INTER);
 3385 %}
 3386 
 3387 operand kReg_K4()
 3388 %{
 3389   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3390   match(RegVectMask);
 3391   format %{%}
 3392   interface(REG_INTER);
 3393 %}
 3394 
 3395 operand kReg_K5()
 3396 %{
 3397   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3398   match(RegVectMask);
 3399   format %{%}
 3400   interface(REG_INTER);
 3401 %}
 3402 
 3403 operand kReg_K6()
 3404 %{
 3405   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3406   match(RegVectMask);
 3407   format %{%}
 3408   interface(REG_INTER);
 3409 %}
 3410 
 3411 // Special Registers
 3412 operand kReg_K7()
 3413 %{
 3414   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3415   match(RegVectMask);
 3416   format %{%}
 3417   interface(REG_INTER);
 3418 %}
 3419 
 3420 // Register Operands
 3421 // Integer Register
 3422 operand rRegI()
 3423 %{
 3424   constraint(ALLOC_IN_RC(int_reg));
 3425   match(RegI);
 3426 
 3427   match(rax_RegI);
 3428   match(rbx_RegI);
 3429   match(rcx_RegI);
 3430   match(rdx_RegI);
 3431   match(rdi_RegI);
 3432 
 3433   format %{ %}
 3434   interface(REG_INTER);
 3435 %}
 3436 
 3437 // Special Registers
 3438 operand rax_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rax_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RAX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 // Special Registers
 3449 operand rbx_RegI()
 3450 %{
 3451   constraint(ALLOC_IN_RC(int_rbx_reg));
 3452   match(RegI);
 3453   match(rRegI);
 3454 
 3455   format %{ "RBX" %}
 3456   interface(REG_INTER);
 3457 %}
 3458 
 3459 operand rcx_RegI()
 3460 %{
 3461   constraint(ALLOC_IN_RC(int_rcx_reg));
 3462   match(RegI);
 3463   match(rRegI);
 3464 
 3465   format %{ "RCX" %}
 3466   interface(REG_INTER);
 3467 %}
 3468 
 3469 operand rdx_RegI()
 3470 %{
 3471   constraint(ALLOC_IN_RC(int_rdx_reg));
 3472   match(RegI);
 3473   match(rRegI);
 3474 
 3475   format %{ "RDX" %}
 3476   interface(REG_INTER);
 3477 %}
 3478 
 3479 operand rdi_RegI()
 3480 %{
 3481   constraint(ALLOC_IN_RC(int_rdi_reg));
 3482   match(RegI);
 3483   match(rRegI);
 3484 
 3485   format %{ "RDI" %}
 3486   interface(REG_INTER);
 3487 %}
 3488 
 3489 operand no_rax_rdx_RegI()
 3490 %{
 3491   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3492   match(RegI);
 3493   match(rbx_RegI);
 3494   match(rcx_RegI);
 3495   match(rdi_RegI);
 3496 
 3497   format %{ %}
 3498   interface(REG_INTER);
 3499 %}
 3500 
 3501 // Pointer Register
 3502 operand any_RegP()
 3503 %{
 3504   constraint(ALLOC_IN_RC(any_reg));
 3505   match(RegP);
 3506   match(rax_RegP);
 3507   match(rbx_RegP);
 3508   match(rdi_RegP);
 3509   match(rsi_RegP);
 3510   match(rbp_RegP);
 3511   match(r15_RegP);
 3512   match(rRegP);
 3513 
 3514   format %{ %}
 3515   interface(REG_INTER);
 3516 %}
 3517 
 3518 operand rRegP()
 3519 %{
 3520   constraint(ALLOC_IN_RC(ptr_reg));
 3521   match(RegP);
 3522   match(rax_RegP);
 3523   match(rbx_RegP);
 3524   match(rdi_RegP);
 3525   match(rsi_RegP);
 3526   match(rbp_RegP);  // See Q&A below about
 3527   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3528 
 3529   format %{ %}
 3530   interface(REG_INTER);
 3531 %}
 3532 
 3533 operand rRegN() %{
 3534   constraint(ALLOC_IN_RC(int_reg));
 3535   match(RegN);
 3536 
 3537   format %{ %}
 3538   interface(REG_INTER);
 3539 %}
 3540 
 3541 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3542 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3543 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3544 // The output of an instruction is controlled by the allocator, which respects
 3545 // register class masks, not match rules.  Unless an instruction mentions
 3546 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3547 // by the allocator as an input.
 3548 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3549 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3550 // result, RBP is not included in the output of the instruction either.
 3551 
 3552 operand no_rax_RegP()
 3553 %{
 3554   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3555   match(RegP);
 3556   match(rbx_RegP);
 3557   match(rsi_RegP);
 3558   match(rdi_RegP);
 3559 
 3560   format %{ %}
 3561   interface(REG_INTER);
 3562 %}
 3563 
 3564 // This operand is not allowed to use RBP even if
 3565 // RBP is not used to hold the frame pointer.
 3566 operand no_rbp_RegP()
 3567 %{
 3568   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3569   match(RegP);
 3570   match(rbx_RegP);
 3571   match(rsi_RegP);
 3572   match(rdi_RegP);
 3573 
 3574   format %{ %}
 3575   interface(REG_INTER);
 3576 %}
 3577 
 3578 operand no_rax_rbx_RegP()
 3579 %{
 3580   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3581   match(RegP);
 3582   match(rsi_RegP);
 3583   match(rdi_RegP);
 3584 
 3585   format %{ %}
 3586   interface(REG_INTER);
 3587 %}
 3588 
 3589 // Special Registers
 3590 // Return a pointer value
 3591 operand rax_RegP()
 3592 %{
 3593   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3594   match(RegP);
 3595   match(rRegP);
 3596 
 3597   format %{ %}
 3598   interface(REG_INTER);
 3599 %}
 3600 
 3601 // Special Registers
 3602 // Return a compressed pointer value
 3603 operand rax_RegN()
 3604 %{
 3605   constraint(ALLOC_IN_RC(int_rax_reg));
 3606   match(RegN);
 3607   match(rRegN);
 3608 
 3609   format %{ %}
 3610   interface(REG_INTER);
 3611 %}
 3612 
 3613 // Used in AtomicAdd
 3614 operand rbx_RegP()
 3615 %{
 3616   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3617   match(RegP);
 3618   match(rRegP);
 3619 
 3620   format %{ %}
 3621   interface(REG_INTER);
 3622 %}
 3623 
 3624 operand rsi_RegP()
 3625 %{
 3626   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3627   match(RegP);
 3628   match(rRegP);
 3629 
 3630   format %{ %}
 3631   interface(REG_INTER);
 3632 %}
 3633 
 3634 operand rbp_RegP()
 3635 %{
 3636   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3637   match(RegP);
 3638   match(rRegP);
 3639 
 3640   format %{ %}
 3641   interface(REG_INTER);
 3642 %}
 3643 
 3644 // Used in rep stosq
 3645 operand rdi_RegP()
 3646 %{
 3647   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3648   match(RegP);
 3649   match(rRegP);
 3650 
 3651   format %{ %}
 3652   interface(REG_INTER);
 3653 %}
 3654 
 3655 operand r15_RegP()
 3656 %{
 3657   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3658   match(RegP);
 3659   match(rRegP);
 3660 
 3661   format %{ %}
 3662   interface(REG_INTER);
 3663 %}
 3664 
 3665 operand rRegL()
 3666 %{
 3667   constraint(ALLOC_IN_RC(long_reg));
 3668   match(RegL);
 3669   match(rax_RegL);
 3670   match(rdx_RegL);
 3671 
 3672   format %{ %}
 3673   interface(REG_INTER);
 3674 %}
 3675 
 3676 // Special Registers
 3677 operand no_rax_rdx_RegL()
 3678 %{
 3679   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3680   match(RegL);
 3681   match(rRegL);
 3682 
 3683   format %{ %}
 3684   interface(REG_INTER);
 3685 %}
 3686 
 3687 operand no_rax_RegL()
 3688 %{
 3689   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3690   match(RegL);
 3691   match(rRegL);
 3692   match(rdx_RegL);
 3693 
 3694   format %{ %}
 3695   interface(REG_INTER);
 3696 %}
 3697 
 3698 operand rax_RegL()
 3699 %{
 3700   constraint(ALLOC_IN_RC(long_rax_reg));
 3701   match(RegL);
 3702   match(rRegL);
 3703 
 3704   format %{ "RAX" %}
 3705   interface(REG_INTER);
 3706 %}
 3707 
 3708 operand rcx_RegL()
 3709 %{
 3710   constraint(ALLOC_IN_RC(long_rcx_reg));
 3711   match(RegL);
 3712   match(rRegL);
 3713 
 3714   format %{ %}
 3715   interface(REG_INTER);
 3716 %}
 3717 
 3718 operand rdx_RegL()
 3719 %{
 3720   constraint(ALLOC_IN_RC(long_rdx_reg));
 3721   match(RegL);
 3722   match(rRegL);
 3723 
 3724   format %{ %}
 3725   interface(REG_INTER);
 3726 %}
 3727 
 3728 // Flags register, used as output of compare instructions
 3729 operand rFlagsReg()
 3730 %{
 3731   constraint(ALLOC_IN_RC(int_flags));
 3732   match(RegFlags);
 3733 
 3734   format %{ "RFLAGS" %}
 3735   interface(REG_INTER);
 3736 %}
 3737 
 3738 // Flags register, used as output of FLOATING POINT compare instructions
 3739 operand rFlagsRegU()
 3740 %{
 3741   constraint(ALLOC_IN_RC(int_flags));
 3742   match(RegFlags);
 3743 
 3744   format %{ "RFLAGS_U" %}
 3745   interface(REG_INTER);
 3746 %}
 3747 
 3748 operand rFlagsRegUCF() %{
 3749   constraint(ALLOC_IN_RC(int_flags));
 3750   match(RegFlags);
 3751   predicate(false);
 3752 
 3753   format %{ "RFLAGS_U_CF" %}
 3754   interface(REG_INTER);
 3755 %}
 3756 
 3757 // Float register operands
 3758 operand regF() %{
 3759    constraint(ALLOC_IN_RC(float_reg));
 3760    match(RegF);
 3761 
 3762    format %{ %}
 3763    interface(REG_INTER);
 3764 %}
 3765 
 3766 // Float register operands
 3767 operand legRegF() %{
 3768    constraint(ALLOC_IN_RC(float_reg_legacy));
 3769    match(RegF);
 3770 
 3771    format %{ %}
 3772    interface(REG_INTER);
 3773 %}
 3774 
 3775 // Float register operands
 3776 operand vlRegF() %{
 3777    constraint(ALLOC_IN_RC(float_reg_vl));
 3778    match(RegF);
 3779 
 3780    format %{ %}
 3781    interface(REG_INTER);
 3782 %}
 3783 
 3784 // Double register operands
 3785 operand regD() %{
 3786    constraint(ALLOC_IN_RC(double_reg));
 3787    match(RegD);
 3788 
 3789    format %{ %}
 3790    interface(REG_INTER);
 3791 %}
 3792 
 3793 // Double register operands
 3794 operand legRegD() %{
 3795    constraint(ALLOC_IN_RC(double_reg_legacy));
 3796    match(RegD);
 3797 
 3798    format %{ %}
 3799    interface(REG_INTER);
 3800 %}
 3801 
 3802 // Double register operands
 3803 operand vlRegD() %{
 3804    constraint(ALLOC_IN_RC(double_reg_vl));
 3805    match(RegD);
 3806 
 3807    format %{ %}
 3808    interface(REG_INTER);
 3809 %}
 3810 
 3811 //----------Memory Operands----------------------------------------------------
 3812 // Direct Memory Operand
 3813 // operand direct(immP addr)
 3814 // %{
 3815 //   match(addr);
 3816 
 3817 //   format %{ "[$addr]" %}
 3818 //   interface(MEMORY_INTER) %{
 3819 //     base(0xFFFFFFFF);
 3820 //     index(0x4);
 3821 //     scale(0x0);
 3822 //     disp($addr);
 3823 //   %}
 3824 // %}
 3825 
 3826 // Indirect Memory Operand
 3827 operand indirect(any_RegP reg)
 3828 %{
 3829   constraint(ALLOC_IN_RC(ptr_reg));
 3830   match(reg);
 3831 
 3832   format %{ "[$reg]" %}
 3833   interface(MEMORY_INTER) %{
 3834     base($reg);
 3835     index(0x4);
 3836     scale(0x0);
 3837     disp(0x0);
 3838   %}
 3839 %}
 3840 
 3841 // Indirect Memory Plus Short Offset Operand
 3842 operand indOffset8(any_RegP reg, immL8 off)
 3843 %{
 3844   constraint(ALLOC_IN_RC(ptr_reg));
 3845   match(AddP reg off);
 3846 
 3847   format %{ "[$reg + $off (8-bit)]" %}
 3848   interface(MEMORY_INTER) %{
 3849     base($reg);
 3850     index(0x4);
 3851     scale(0x0);
 3852     disp($off);
 3853   %}
 3854 %}
 3855 
 3856 // Indirect Memory Plus Long Offset Operand
 3857 operand indOffset32(any_RegP reg, immL32 off)
 3858 %{
 3859   constraint(ALLOC_IN_RC(ptr_reg));
 3860   match(AddP reg off);
 3861 
 3862   format %{ "[$reg + $off (32-bit)]" %}
 3863   interface(MEMORY_INTER) %{
 3864     base($reg);
 3865     index(0x4);
 3866     scale(0x0);
 3867     disp($off);
 3868   %}
 3869 %}
 3870 
 3871 // Indirect Memory Plus Index Register Plus Offset Operand
 3872 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3873 %{
 3874   constraint(ALLOC_IN_RC(ptr_reg));
 3875   match(AddP (AddP reg lreg) off);
 3876 
 3877   op_cost(10);
 3878   format %{"[$reg + $off + $lreg]" %}
 3879   interface(MEMORY_INTER) %{
 3880     base($reg);
 3881     index($lreg);
 3882     scale(0x0);
 3883     disp($off);
 3884   %}
 3885 %}
 3886 
 3887 // Indirect Memory Plus Index Register Plus Offset Operand
 3888 operand indIndex(any_RegP reg, rRegL lreg)
 3889 %{
 3890   constraint(ALLOC_IN_RC(ptr_reg));
 3891   match(AddP reg lreg);
 3892 
 3893   op_cost(10);
 3894   format %{"[$reg + $lreg]" %}
 3895   interface(MEMORY_INTER) %{
 3896     base($reg);
 3897     index($lreg);
 3898     scale(0x0);
 3899     disp(0x0);
 3900   %}
 3901 %}
 3902 
 3903 // Indirect Memory Times Scale Plus Index Register
 3904 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3905 %{
 3906   constraint(ALLOC_IN_RC(ptr_reg));
 3907   match(AddP reg (LShiftL lreg scale));
 3908 
 3909   op_cost(10);
 3910   format %{"[$reg + $lreg << $scale]" %}
 3911   interface(MEMORY_INTER) %{
 3912     base($reg);
 3913     index($lreg);
 3914     scale($scale);
 3915     disp(0x0);
 3916   %}
 3917 %}
 3918 
 3919 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3920 %{
 3921   constraint(ALLOC_IN_RC(ptr_reg));
 3922   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3923   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3924 
 3925   op_cost(10);
 3926   format %{"[$reg + pos $idx << $scale]" %}
 3927   interface(MEMORY_INTER) %{
 3928     base($reg);
 3929     index($idx);
 3930     scale($scale);
 3931     disp(0x0);
 3932   %}
 3933 %}
 3934 
 3935 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3936 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3937 %{
 3938   constraint(ALLOC_IN_RC(ptr_reg));
 3939   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3940 
 3941   op_cost(10);
 3942   format %{"[$reg + $off + $lreg << $scale]" %}
 3943   interface(MEMORY_INTER) %{
 3944     base($reg);
 3945     index($lreg);
 3946     scale($scale);
 3947     disp($off);
 3948   %}
 3949 %}
 3950 
 3951 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3952 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3953 %{
 3954   constraint(ALLOC_IN_RC(ptr_reg));
 3955   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3956   match(AddP (AddP reg (ConvI2L idx)) off);
 3957 
 3958   op_cost(10);
 3959   format %{"[$reg + $off + $idx]" %}
 3960   interface(MEMORY_INTER) %{
 3961     base($reg);
 3962     index($idx);
 3963     scale(0x0);
 3964     disp($off);
 3965   %}
 3966 %}
 3967 
 3968 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3969 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3970 %{
 3971   constraint(ALLOC_IN_RC(ptr_reg));
 3972   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3973   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3974 
 3975   op_cost(10);
 3976   format %{"[$reg + $off + $idx << $scale]" %}
 3977   interface(MEMORY_INTER) %{
 3978     base($reg);
 3979     index($idx);
 3980     scale($scale);
 3981     disp($off);
 3982   %}
 3983 %}
 3984 
 3985 // Indirect Narrow Oop Plus Offset Operand
 3986 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3987 // we can't free r12 even with CompressedOops::base() == NULL.
 3988 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3989   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3990   constraint(ALLOC_IN_RC(ptr_reg));
 3991   match(AddP (DecodeN reg) off);
 3992 
 3993   op_cost(10);
 3994   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3995   interface(MEMORY_INTER) %{
 3996     base(0xc); // R12
 3997     index($reg);
 3998     scale(0x3);
 3999     disp($off);
 4000   %}
 4001 %}
 4002 
 4003 // Indirect Memory Operand
 4004 operand indirectNarrow(rRegN reg)
 4005 %{
 4006   predicate(CompressedOops::shift() == 0);
 4007   constraint(ALLOC_IN_RC(ptr_reg));
 4008   match(DecodeN reg);
 4009 
 4010   format %{ "[$reg]" %}
 4011   interface(MEMORY_INTER) %{
 4012     base($reg);
 4013     index(0x4);
 4014     scale(0x0);
 4015     disp(0x0);
 4016   %}
 4017 %}
 4018 
 4019 // Indirect Memory Plus Short Offset Operand
 4020 operand indOffset8Narrow(rRegN reg, immL8 off)
 4021 %{
 4022   predicate(CompressedOops::shift() == 0);
 4023   constraint(ALLOC_IN_RC(ptr_reg));
 4024   match(AddP (DecodeN reg) off);
 4025 
 4026   format %{ "[$reg + $off (8-bit)]" %}
 4027   interface(MEMORY_INTER) %{
 4028     base($reg);
 4029     index(0x4);
 4030     scale(0x0);
 4031     disp($off);
 4032   %}
 4033 %}
 4034 
 4035 // Indirect Memory Plus Long Offset Operand
 4036 operand indOffset32Narrow(rRegN reg, immL32 off)
 4037 %{
 4038   predicate(CompressedOops::shift() == 0);
 4039   constraint(ALLOC_IN_RC(ptr_reg));
 4040   match(AddP (DecodeN reg) off);
 4041 
 4042   format %{ "[$reg + $off (32-bit)]" %}
 4043   interface(MEMORY_INTER) %{
 4044     base($reg);
 4045     index(0x4);
 4046     scale(0x0);
 4047     disp($off);
 4048   %}
 4049 %}
 4050 
 4051 // Indirect Memory Plus Index Register Plus Offset Operand
 4052 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4053 %{
 4054   predicate(CompressedOops::shift() == 0);
 4055   constraint(ALLOC_IN_RC(ptr_reg));
 4056   match(AddP (AddP (DecodeN reg) lreg) off);
 4057 
 4058   op_cost(10);
 4059   format %{"[$reg + $off + $lreg]" %}
 4060   interface(MEMORY_INTER) %{
 4061     base($reg);
 4062     index($lreg);
 4063     scale(0x0);
 4064     disp($off);
 4065   %}
 4066 %}
 4067 
 4068 // Indirect Memory Plus Index Register Plus Offset Operand
 4069 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4070 %{
 4071   predicate(CompressedOops::shift() == 0);
 4072   constraint(ALLOC_IN_RC(ptr_reg));
 4073   match(AddP (DecodeN reg) lreg);
 4074 
 4075   op_cost(10);
 4076   format %{"[$reg + $lreg]" %}
 4077   interface(MEMORY_INTER) %{
 4078     base($reg);
 4079     index($lreg);
 4080     scale(0x0);
 4081     disp(0x0);
 4082   %}
 4083 %}
 4084 
 4085 // Indirect Memory Times Scale Plus Index Register
 4086 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4087 %{
 4088   predicate(CompressedOops::shift() == 0);
 4089   constraint(ALLOC_IN_RC(ptr_reg));
 4090   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4091 
 4092   op_cost(10);
 4093   format %{"[$reg + $lreg << $scale]" %}
 4094   interface(MEMORY_INTER) %{
 4095     base($reg);
 4096     index($lreg);
 4097     scale($scale);
 4098     disp(0x0);
 4099   %}
 4100 %}
 4101 
 4102 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4103 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4104 %{
 4105   predicate(CompressedOops::shift() == 0);
 4106   constraint(ALLOC_IN_RC(ptr_reg));
 4107   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4108 
 4109   op_cost(10);
 4110   format %{"[$reg + $off + $lreg << $scale]" %}
 4111   interface(MEMORY_INTER) %{
 4112     base($reg);
 4113     index($lreg);
 4114     scale($scale);
 4115     disp($off);
 4116   %}
 4117 %}
 4118 
 4119 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4120 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4121 %{
 4122   constraint(ALLOC_IN_RC(ptr_reg));
 4123   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4124   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4125 
 4126   op_cost(10);
 4127   format %{"[$reg + $off + $idx]" %}
 4128   interface(MEMORY_INTER) %{
 4129     base($reg);
 4130     index($idx);
 4131     scale(0x0);
 4132     disp($off);
 4133   %}
 4134 %}
 4135 
 4136 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4137 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4138 %{
 4139   constraint(ALLOC_IN_RC(ptr_reg));
 4140   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4141   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4142 
 4143   op_cost(10);
 4144   format %{"[$reg + $off + $idx << $scale]" %}
 4145   interface(MEMORY_INTER) %{
 4146     base($reg);
 4147     index($idx);
 4148     scale($scale);
 4149     disp($off);
 4150   %}
 4151 %}
 4152 
 4153 //----------Special Memory Operands--------------------------------------------
 4154 // Stack Slot Operand - This operand is used for loading and storing temporary
 4155 //                      values on the stack where a match requires a value to
 4156 //                      flow through memory.
 4157 operand stackSlotP(sRegP reg)
 4158 %{
 4159   constraint(ALLOC_IN_RC(stack_slots));
 4160   // No match rule because this operand is only generated in matching
 4161 
 4162   format %{ "[$reg]" %}
 4163   interface(MEMORY_INTER) %{
 4164     base(0x4);   // RSP
 4165     index(0x4);  // No Index
 4166     scale(0x0);  // No Scale
 4167     disp($reg);  // Stack Offset
 4168   %}
 4169 %}
 4170 
 4171 operand stackSlotI(sRegI reg)
 4172 %{
 4173   constraint(ALLOC_IN_RC(stack_slots));
 4174   // No match rule because this operand is only generated in matching
 4175 
 4176   format %{ "[$reg]" %}
 4177   interface(MEMORY_INTER) %{
 4178     base(0x4);   // RSP
 4179     index(0x4);  // No Index
 4180     scale(0x0);  // No Scale
 4181     disp($reg);  // Stack Offset
 4182   %}
 4183 %}
 4184 
 4185 operand stackSlotF(sRegF reg)
 4186 %{
 4187   constraint(ALLOC_IN_RC(stack_slots));
 4188   // No match rule because this operand is only generated in matching
 4189 
 4190   format %{ "[$reg]" %}
 4191   interface(MEMORY_INTER) %{
 4192     base(0x4);   // RSP
 4193     index(0x4);  // No Index
 4194     scale(0x0);  // No Scale
 4195     disp($reg);  // Stack Offset
 4196   %}
 4197 %}
 4198 
 4199 operand stackSlotD(sRegD reg)
 4200 %{
 4201   constraint(ALLOC_IN_RC(stack_slots));
 4202   // No match rule because this operand is only generated in matching
 4203 
 4204   format %{ "[$reg]" %}
 4205   interface(MEMORY_INTER) %{
 4206     base(0x4);   // RSP
 4207     index(0x4);  // No Index
 4208     scale(0x0);  // No Scale
 4209     disp($reg);  // Stack Offset
 4210   %}
 4211 %}
 4212 operand stackSlotL(sRegL reg)
 4213 %{
 4214   constraint(ALLOC_IN_RC(stack_slots));
 4215   // No match rule because this operand is only generated in matching
 4216 
 4217   format %{ "[$reg]" %}
 4218   interface(MEMORY_INTER) %{
 4219     base(0x4);   // RSP
 4220     index(0x4);  // No Index
 4221     scale(0x0);  // No Scale
 4222     disp($reg);  // Stack Offset
 4223   %}
 4224 %}
 4225 
 4226 //----------Conditional Branch Operands----------------------------------------
 4227 // Comparison Op  - This is the operation of the comparison, and is limited to
 4228 //                  the following set of codes:
 4229 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4230 //
 4231 // Other attributes of the comparison, such as unsignedness, are specified
 4232 // by the comparison instruction that sets a condition code flags register.
 4233 // That result is represented by a flags operand whose subtype is appropriate
 4234 // to the unsignedness (etc.) of the comparison.
 4235 //
 4236 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4237 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4238 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4239 
 4240 // Comparision Code
 4241 operand cmpOp()
 4242 %{
 4243   match(Bool);
 4244 
 4245   format %{ "" %}
 4246   interface(COND_INTER) %{
 4247     equal(0x4, "e");
 4248     not_equal(0x5, "ne");
 4249     less(0xC, "l");
 4250     greater_equal(0xD, "ge");
 4251     less_equal(0xE, "le");
 4252     greater(0xF, "g");
 4253     overflow(0x0, "o");
 4254     no_overflow(0x1, "no");
 4255   %}
 4256 %}
 4257 
 4258 // Comparison Code, unsigned compare.  Used by FP also, with
 4259 // C2 (unordered) turned into GT or LT already.  The other bits
 4260 // C0 and C3 are turned into Carry & Zero flags.
 4261 operand cmpOpU()
 4262 %{
 4263   match(Bool);
 4264 
 4265   format %{ "" %}
 4266   interface(COND_INTER) %{
 4267     equal(0x4, "e");
 4268     not_equal(0x5, "ne");
 4269     less(0x2, "b");
 4270     greater_equal(0x3, "nb");
 4271     less_equal(0x6, "be");
 4272     greater(0x7, "nbe");
 4273     overflow(0x0, "o");
 4274     no_overflow(0x1, "no");
 4275   %}
 4276 %}
 4277 
 4278 
 4279 // Floating comparisons that don't require any fixup for the unordered case
 4280 operand cmpOpUCF() %{
 4281   match(Bool);
 4282   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4283             n->as_Bool()->_test._test == BoolTest::ge ||
 4284             n->as_Bool()->_test._test == BoolTest::le ||
 4285             n->as_Bool()->_test._test == BoolTest::gt);
 4286   format %{ "" %}
 4287   interface(COND_INTER) %{
 4288     equal(0x4, "e");
 4289     not_equal(0x5, "ne");
 4290     less(0x2, "b");
 4291     greater_equal(0x3, "nb");
 4292     less_equal(0x6, "be");
 4293     greater(0x7, "nbe");
 4294     overflow(0x0, "o");
 4295     no_overflow(0x1, "no");
 4296   %}
 4297 %}
 4298 
 4299 
 4300 // Floating comparisons that can be fixed up with extra conditional jumps
 4301 operand cmpOpUCF2() %{
 4302   match(Bool);
 4303   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4304             n->as_Bool()->_test._test == BoolTest::eq);
 4305   format %{ "" %}
 4306   interface(COND_INTER) %{
 4307     equal(0x4, "e");
 4308     not_equal(0x5, "ne");
 4309     less(0x2, "b");
 4310     greater_equal(0x3, "nb");
 4311     less_equal(0x6, "be");
 4312     greater(0x7, "nbe");
 4313     overflow(0x0, "o");
 4314     no_overflow(0x1, "no");
 4315   %}
 4316 %}
 4317 
 4318 //----------OPERAND CLASSES----------------------------------------------------
 4319 // Operand Classes are groups of operands that are used as to simplify
 4320 // instruction definitions by not requiring the AD writer to specify separate
 4321 // instructions for every form of operand when the instruction accepts
 4322 // multiple operand types with the same basic encoding and format.  The classic
 4323 // case of this is memory operands.
 4324 
 4325 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4326                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4327                indCompressedOopOffset,
 4328                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4329                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4330                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4331 
 4332 //----------PIPELINE-----------------------------------------------------------
 4333 // Rules which define the behavior of the target architectures pipeline.
 4334 pipeline %{
 4335 
 4336 //----------ATTRIBUTES---------------------------------------------------------
 4337 attributes %{
 4338   variable_size_instructions;        // Fixed size instructions
 4339   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4340   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4341   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4342   instruction_fetch_units = 1;       // of 16 bytes
 4343 
 4344   // List of nop instructions
 4345   nops( MachNop );
 4346 %}
 4347 
 4348 //----------RESOURCES----------------------------------------------------------
 4349 // Resources are the functional units available to the machine
 4350 
 4351 // Generic P2/P3 pipeline
 4352 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4353 // 3 instructions decoded per cycle.
 4354 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4355 // 3 ALU op, only ALU0 handles mul instructions.
 4356 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4357            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4358            BR, FPU,
 4359            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4360 
 4361 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4362 // Pipeline Description specifies the stages in the machine's pipeline
 4363 
 4364 // Generic P2/P3 pipeline
 4365 pipe_desc(S0, S1, S2, S3, S4, S5);
 4366 
 4367 //----------PIPELINE CLASSES---------------------------------------------------
 4368 // Pipeline Classes describe the stages in which input and output are
 4369 // referenced by the hardware pipeline.
 4370 
 4371 // Naming convention: ialu or fpu
 4372 // Then: _reg
 4373 // Then: _reg if there is a 2nd register
 4374 // Then: _long if it's a pair of instructions implementing a long
 4375 // Then: _fat if it requires the big decoder
 4376 //   Or: _mem if it requires the big decoder and a memory unit.
 4377 
 4378 // Integer ALU reg operation
 4379 pipe_class ialu_reg(rRegI dst)
 4380 %{
 4381     single_instruction;
 4382     dst    : S4(write);
 4383     dst    : S3(read);
 4384     DECODE : S0;        // any decoder
 4385     ALU    : S3;        // any alu
 4386 %}
 4387 
 4388 // Long ALU reg operation
 4389 pipe_class ialu_reg_long(rRegL dst)
 4390 %{
 4391     instruction_count(2);
 4392     dst    : S4(write);
 4393     dst    : S3(read);
 4394     DECODE : S0(2);     // any 2 decoders
 4395     ALU    : S3(2);     // both alus
 4396 %}
 4397 
 4398 // Integer ALU reg operation using big decoder
 4399 pipe_class ialu_reg_fat(rRegI dst)
 4400 %{
 4401     single_instruction;
 4402     dst    : S4(write);
 4403     dst    : S3(read);
 4404     D0     : S0;        // big decoder only
 4405     ALU    : S3;        // any alu
 4406 %}
 4407 
 4408 // Integer ALU reg-reg operation
 4409 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4410 %{
 4411     single_instruction;
 4412     dst    : S4(write);
 4413     src    : S3(read);
 4414     DECODE : S0;        // any decoder
 4415     ALU    : S3;        // any alu
 4416 %}
 4417 
 4418 // Integer ALU reg-reg operation
 4419 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4420 %{
 4421     single_instruction;
 4422     dst    : S4(write);
 4423     src    : S3(read);
 4424     D0     : S0;        // big decoder only
 4425     ALU    : S3;        // any alu
 4426 %}
 4427 
 4428 // Integer ALU reg-mem operation
 4429 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4430 %{
 4431     single_instruction;
 4432     dst    : S5(write);
 4433     mem    : S3(read);
 4434     D0     : S0;        // big decoder only
 4435     ALU    : S4;        // any alu
 4436     MEM    : S3;        // any mem
 4437 %}
 4438 
 4439 // Integer mem operation (prefetch)
 4440 pipe_class ialu_mem(memory mem)
 4441 %{
 4442     single_instruction;
 4443     mem    : S3(read);
 4444     D0     : S0;        // big decoder only
 4445     MEM    : S3;        // any mem
 4446 %}
 4447 
 4448 // Integer Store to Memory
 4449 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4450 %{
 4451     single_instruction;
 4452     mem    : S3(read);
 4453     src    : S5(read);
 4454     D0     : S0;        // big decoder only
 4455     ALU    : S4;        // any alu
 4456     MEM    : S3;
 4457 %}
 4458 
 4459 // // Long Store to Memory
 4460 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4461 // %{
 4462 //     instruction_count(2);
 4463 //     mem    : S3(read);
 4464 //     src    : S5(read);
 4465 //     D0     : S0(2);          // big decoder only; twice
 4466 //     ALU    : S4(2);     // any 2 alus
 4467 //     MEM    : S3(2);  // Both mems
 4468 // %}
 4469 
 4470 // Integer Store to Memory
 4471 pipe_class ialu_mem_imm(memory mem)
 4472 %{
 4473     single_instruction;
 4474     mem    : S3(read);
 4475     D0     : S0;        // big decoder only
 4476     ALU    : S4;        // any alu
 4477     MEM    : S3;
 4478 %}
 4479 
 4480 // Integer ALU0 reg-reg operation
 4481 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4482 %{
 4483     single_instruction;
 4484     dst    : S4(write);
 4485     src    : S3(read);
 4486     D0     : S0;        // Big decoder only
 4487     ALU0   : S3;        // only alu0
 4488 %}
 4489 
 4490 // Integer ALU0 reg-mem operation
 4491 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4492 %{
 4493     single_instruction;
 4494     dst    : S5(write);
 4495     mem    : S3(read);
 4496     D0     : S0;        // big decoder only
 4497     ALU0   : S4;        // ALU0 only
 4498     MEM    : S3;        // any mem
 4499 %}
 4500 
 4501 // Integer ALU reg-reg operation
 4502 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4503 %{
 4504     single_instruction;
 4505     cr     : S4(write);
 4506     src1   : S3(read);
 4507     src2   : S3(read);
 4508     DECODE : S0;        // any decoder
 4509     ALU    : S3;        // any alu
 4510 %}
 4511 
 4512 // Integer ALU reg-imm operation
 4513 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     DECODE : S0;        // any decoder
 4519     ALU    : S3;        // any alu
 4520 %}
 4521 
 4522 // Integer ALU reg-mem operation
 4523 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4524 %{
 4525     single_instruction;
 4526     cr     : S4(write);
 4527     src1   : S3(read);
 4528     src2   : S3(read);
 4529     D0     : S0;        // big decoder only
 4530     ALU    : S4;        // any alu
 4531     MEM    : S3;
 4532 %}
 4533 
 4534 // Conditional move reg-reg
 4535 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4536 %{
 4537     instruction_count(4);
 4538     y      : S4(read);
 4539     q      : S3(read);
 4540     p      : S3(read);
 4541     DECODE : S0(4);     // any decoder
 4542 %}
 4543 
 4544 // Conditional move reg-reg
 4545 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4546 %{
 4547     single_instruction;
 4548     dst    : S4(write);
 4549     src    : S3(read);
 4550     cr     : S3(read);
 4551     DECODE : S0;        // any decoder
 4552 %}
 4553 
 4554 // Conditional move reg-mem
 4555 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4556 %{
 4557     single_instruction;
 4558     dst    : S4(write);
 4559     src    : S3(read);
 4560     cr     : S3(read);
 4561     DECODE : S0;        // any decoder
 4562     MEM    : S3;
 4563 %}
 4564 
 4565 // Conditional move reg-reg long
 4566 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0(2);     // any 2 decoders
 4573 %}
 4574 
 4575 // XXX
 4576 // // Conditional move double reg-reg
 4577 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4578 // %{
 4579 //     single_instruction;
 4580 //     dst    : S4(write);
 4581 //     src    : S3(read);
 4582 //     cr     : S3(read);
 4583 //     DECODE : S0;     // any decoder
 4584 // %}
 4585 
 4586 // Float reg-reg operation
 4587 pipe_class fpu_reg(regD dst)
 4588 %{
 4589     instruction_count(2);
 4590     dst    : S3(read);
 4591     DECODE : S0(2);     // any 2 decoders
 4592     FPU    : S3;
 4593 %}
 4594 
 4595 // Float reg-reg operation
 4596 pipe_class fpu_reg_reg(regD dst, regD src)
 4597 %{
 4598     instruction_count(2);
 4599     dst    : S4(write);
 4600     src    : S3(read);
 4601     DECODE : S0(2);     // any 2 decoders
 4602     FPU    : S3;
 4603 %}
 4604 
 4605 // Float reg-reg operation
 4606 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4607 %{
 4608     instruction_count(3);
 4609     dst    : S4(write);
 4610     src1   : S3(read);
 4611     src2   : S3(read);
 4612     DECODE : S0(3);     // any 3 decoders
 4613     FPU    : S3(2);
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4618 %{
 4619     instruction_count(4);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     src3   : S3(read);
 4624     DECODE : S0(4);     // any 3 decoders
 4625     FPU    : S3(2);
 4626 %}
 4627 
 4628 // Float reg-reg operation
 4629 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4630 %{
 4631     instruction_count(4);
 4632     dst    : S4(write);
 4633     src1   : S3(read);
 4634     src2   : S3(read);
 4635     src3   : S3(read);
 4636     DECODE : S1(3);     // any 3 decoders
 4637     D0     : S0;        // Big decoder only
 4638     FPU    : S3(2);
 4639     MEM    : S3;
 4640 %}
 4641 
 4642 // Float reg-mem operation
 4643 pipe_class fpu_reg_mem(regD dst, memory mem)
 4644 %{
 4645     instruction_count(2);
 4646     dst    : S5(write);
 4647     mem    : S3(read);
 4648     D0     : S0;        // big decoder only
 4649     DECODE : S1;        // any decoder for FPU POP
 4650     FPU    : S4;
 4651     MEM    : S3;        // any mem
 4652 %}
 4653 
 4654 // Float reg-mem operation
 4655 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4656 %{
 4657     instruction_count(3);
 4658     dst    : S5(write);
 4659     src1   : S3(read);
 4660     mem    : S3(read);
 4661     D0     : S0;        // big decoder only
 4662     DECODE : S1(2);     // any decoder for FPU POP
 4663     FPU    : S4;
 4664     MEM    : S3;        // any mem
 4665 %}
 4666 
 4667 // Float mem-reg operation
 4668 pipe_class fpu_mem_reg(memory mem, regD src)
 4669 %{
 4670     instruction_count(2);
 4671     src    : S5(read);
 4672     mem    : S3(read);
 4673     DECODE : S0;        // any decoder for FPU PUSH
 4674     D0     : S1;        // big decoder only
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4680 %{
 4681     instruction_count(3);
 4682     src1   : S3(read);
 4683     src2   : S3(read);
 4684     mem    : S3(read);
 4685     DECODE : S0(2);     // any decoder for FPU PUSH
 4686     D0     : S1;        // big decoder only
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4692 %{
 4693     instruction_count(3);
 4694     src1   : S3(read);
 4695     src2   : S3(read);
 4696     mem    : S4(read);
 4697     DECODE : S0;        // any decoder for FPU PUSH
 4698     D0     : S0(2);     // big decoder only
 4699     FPU    : S4;
 4700     MEM    : S3(2);     // any mem
 4701 %}
 4702 
 4703 pipe_class fpu_mem_mem(memory dst, memory src1)
 4704 %{
 4705     instruction_count(2);
 4706     src1   : S3(read);
 4707     dst    : S4(read);
 4708     D0     : S0(2);     // big decoder only
 4709     MEM    : S3(2);     // any mem
 4710 %}
 4711 
 4712 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4713 %{
 4714     instruction_count(3);
 4715     src1   : S3(read);
 4716     src2   : S3(read);
 4717     dst    : S4(read);
 4718     D0     : S0(3);     // big decoder only
 4719     FPU    : S4;
 4720     MEM    : S3(3);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S4(read);
 4727     mem    : S4(read);
 4728     DECODE : S0;        // any decoder for FPU PUSH
 4729     D0     : S0(2);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(2);     // any mem
 4732 %}
 4733 
 4734 // Float load constant
 4735 pipe_class fpu_reg_con(regD dst)
 4736 %{
 4737     instruction_count(2);
 4738     dst    : S5(write);
 4739     D0     : S0;        // big decoder only for the load
 4740     DECODE : S1;        // any decoder for FPU POP
 4741     FPU    : S4;
 4742     MEM    : S3;        // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4747 %{
 4748     instruction_count(3);
 4749     dst    : S5(write);
 4750     src    : S3(read);
 4751     D0     : S0;        // big decoder only for the load
 4752     DECODE : S1(2);     // any decoder for FPU POP
 4753     FPU    : S4;
 4754     MEM    : S3;        // any mem
 4755 %}
 4756 
 4757 // UnConditional branch
 4758 pipe_class pipe_jmp(label labl)
 4759 %{
 4760     single_instruction;
 4761     BR   : S3;
 4762 %}
 4763 
 4764 // Conditional branch
 4765 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4766 %{
 4767     single_instruction;
 4768     cr    : S1(read);
 4769     BR    : S3;
 4770 %}
 4771 
 4772 // Allocation idiom
 4773 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4774 %{
 4775     instruction_count(1); force_serialization;
 4776     fixed_latency(6);
 4777     heap_ptr : S3(read);
 4778     DECODE   : S0(3);
 4779     D0       : S2;
 4780     MEM      : S3;
 4781     ALU      : S3(2);
 4782     dst      : S5(write);
 4783     BR       : S5;
 4784 %}
 4785 
 4786 // Generic big/slow expanded idiom
 4787 pipe_class pipe_slow()
 4788 %{
 4789     instruction_count(10); multiple_bundles; force_serialization;
 4790     fixed_latency(100);
 4791     D0  : S0(2);
 4792     MEM : S3(2);
 4793 %}
 4794 
 4795 // The real do-nothing guy
 4796 pipe_class empty()
 4797 %{
 4798     instruction_count(0);
 4799 %}
 4800 
 4801 // Define the class for the Nop node
 4802 define
 4803 %{
 4804    MachNop = empty;
 4805 %}
 4806 
 4807 %}
 4808 
 4809 //----------INSTRUCTIONS-------------------------------------------------------
 4810 //
 4811 // match      -- States which machine-independent subtree may be replaced
 4812 //               by this instruction.
 4813 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4814 //               selection to identify a minimum cost tree of machine
 4815 //               instructions that matches a tree of machine-independent
 4816 //               instructions.
 4817 // format     -- A string providing the disassembly for this instruction.
 4818 //               The value of an instruction's operand may be inserted
 4819 //               by referring to it with a '$' prefix.
 4820 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4821 //               to within an encode class as $primary, $secondary, and $tertiary
 4822 //               rrspectively.  The primary opcode is commonly used to
 4823 //               indicate the type of machine instruction, while secondary
 4824 //               and tertiary are often used for prefix options or addressing
 4825 //               modes.
 4826 // ins_encode -- A list of encode classes with parameters. The encode class
 4827 //               name must have been defined in an 'enc_class' specification
 4828 //               in the encode section of the architecture description.
 4829 
 4830 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4831 // Load Float
 4832 instruct MoveF2VL(vlRegF dst, regF src) %{
 4833   match(Set dst src);
 4834   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4835   ins_encode %{
 4836     ShouldNotReachHere();
 4837   %}
 4838   ins_pipe( fpu_reg_reg );
 4839 %}
 4840 
 4841 // Load Float
 4842 instruct MoveF2LEG(legRegF dst, regF src) %{
 4843   match(Set dst src);
 4844   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4845   ins_encode %{
 4846     ShouldNotReachHere();
 4847   %}
 4848   ins_pipe( fpu_reg_reg );
 4849 %}
 4850 
 4851 // Load Float
 4852 instruct MoveVL2F(regF dst, vlRegF src) %{
 4853   match(Set dst src);
 4854   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4855   ins_encode %{
 4856     ShouldNotReachHere();
 4857   %}
 4858   ins_pipe( fpu_reg_reg );
 4859 %}
 4860 
 4861 // Load Float
 4862 instruct MoveLEG2F(regF dst, legRegF src) %{
 4863   match(Set dst src);
 4864   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4865   ins_encode %{
 4866     ShouldNotReachHere();
 4867   %}
 4868   ins_pipe( fpu_reg_reg );
 4869 %}
 4870 
 4871 // Load Double
 4872 instruct MoveD2VL(vlRegD dst, regD src) %{
 4873   match(Set dst src);
 4874   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4875   ins_encode %{
 4876     ShouldNotReachHere();
 4877   %}
 4878   ins_pipe( fpu_reg_reg );
 4879 %}
 4880 
 4881 // Load Double
 4882 instruct MoveD2LEG(legRegD dst, regD src) %{
 4883   match(Set dst src);
 4884   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4885   ins_encode %{
 4886     ShouldNotReachHere();
 4887   %}
 4888   ins_pipe( fpu_reg_reg );
 4889 %}
 4890 
 4891 // Load Double
 4892 instruct MoveVL2D(regD dst, vlRegD src) %{
 4893   match(Set dst src);
 4894   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4895   ins_encode %{
 4896     ShouldNotReachHere();
 4897   %}
 4898   ins_pipe( fpu_reg_reg );
 4899 %}
 4900 
 4901 // Load Double
 4902 instruct MoveLEG2D(regD dst, legRegD src) %{
 4903   match(Set dst src);
 4904   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4905   ins_encode %{
 4906     ShouldNotReachHere();
 4907   %}
 4908   ins_pipe( fpu_reg_reg );
 4909 %}
 4910 
 4911 //----------Load/Store/Move Instructions---------------------------------------
 4912 //----------Load Instructions--------------------------------------------------
 4913 
 4914 // Load Byte (8 bit signed)
 4915 instruct loadB(rRegI dst, memory mem)
 4916 %{
 4917   match(Set dst (LoadB mem));
 4918 
 4919   ins_cost(125);
 4920   format %{ "movsbl  $dst, $mem\t# byte" %}
 4921 
 4922   ins_encode %{
 4923     __ movsbl($dst$$Register, $mem$$Address);
 4924   %}
 4925 
 4926   ins_pipe(ialu_reg_mem);
 4927 %}
 4928 
 4929 // Load Byte (8 bit signed) into Long Register
 4930 instruct loadB2L(rRegL dst, memory mem)
 4931 %{
 4932   match(Set dst (ConvI2L (LoadB mem)));
 4933 
 4934   ins_cost(125);
 4935   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4936 
 4937   ins_encode %{
 4938     __ movsbq($dst$$Register, $mem$$Address);
 4939   %}
 4940 
 4941   ins_pipe(ialu_reg_mem);
 4942 %}
 4943 
 4944 // Load Unsigned Byte (8 bit UNsigned)
 4945 instruct loadUB(rRegI dst, memory mem)
 4946 %{
 4947   match(Set dst (LoadUB mem));
 4948 
 4949   ins_cost(125);
 4950   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4951 
 4952   ins_encode %{
 4953     __ movzbl($dst$$Register, $mem$$Address);
 4954   %}
 4955 
 4956   ins_pipe(ialu_reg_mem);
 4957 %}
 4958 
 4959 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4960 instruct loadUB2L(rRegL dst, memory mem)
 4961 %{
 4962   match(Set dst (ConvI2L (LoadUB mem)));
 4963 
 4964   ins_cost(125);
 4965   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4966 
 4967   ins_encode %{
 4968     __ movzbq($dst$$Register, $mem$$Address);
 4969   %}
 4970 
 4971   ins_pipe(ialu_reg_mem);
 4972 %}
 4973 
 4974 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4975 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4976   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4977   effect(KILL cr);
 4978 
 4979   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4980             "andl    $dst, right_n_bits($mask, 8)" %}
 4981   ins_encode %{
 4982     Register Rdst = $dst$$Register;
 4983     __ movzbq(Rdst, $mem$$Address);
 4984     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4985   %}
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 // Load Short (16 bit signed)
 4990 instruct loadS(rRegI dst, memory mem)
 4991 %{
 4992   match(Set dst (LoadS mem));
 4993 
 4994   ins_cost(125);
 4995   format %{ "movswl $dst, $mem\t# short" %}
 4996 
 4997   ins_encode %{
 4998     __ movswl($dst$$Register, $mem$$Address);
 4999   %}
 5000 
 5001   ins_pipe(ialu_reg_mem);
 5002 %}
 5003 
 5004 // Load Short (16 bit signed) to Byte (8 bit signed)
 5005 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5006   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5007 
 5008   ins_cost(125);
 5009   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5010   ins_encode %{
 5011     __ movsbl($dst$$Register, $mem$$Address);
 5012   %}
 5013   ins_pipe(ialu_reg_mem);
 5014 %}
 5015 
 5016 // Load Short (16 bit signed) into Long Register
 5017 instruct loadS2L(rRegL dst, memory mem)
 5018 %{
 5019   match(Set dst (ConvI2L (LoadS mem)));
 5020 
 5021   ins_cost(125);
 5022   format %{ "movswq $dst, $mem\t# short -> long" %}
 5023 
 5024   ins_encode %{
 5025     __ movswq($dst$$Register, $mem$$Address);
 5026   %}
 5027 
 5028   ins_pipe(ialu_reg_mem);
 5029 %}
 5030 
 5031 // Load Unsigned Short/Char (16 bit UNsigned)
 5032 instruct loadUS(rRegI dst, memory mem)
 5033 %{
 5034   match(Set dst (LoadUS mem));
 5035 
 5036   ins_cost(125);
 5037   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5038 
 5039   ins_encode %{
 5040     __ movzwl($dst$$Register, $mem$$Address);
 5041   %}
 5042 
 5043   ins_pipe(ialu_reg_mem);
 5044 %}
 5045 
 5046 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5047 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5048   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5049 
 5050   ins_cost(125);
 5051   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5052   ins_encode %{
 5053     __ movsbl($dst$$Register, $mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_reg_mem);
 5056 %}
 5057 
 5058 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5059 instruct loadUS2L(rRegL dst, memory mem)
 5060 %{
 5061   match(Set dst (ConvI2L (LoadUS mem)));
 5062 
 5063   ins_cost(125);
 5064   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5065 
 5066   ins_encode %{
 5067     __ movzwq($dst$$Register, $mem$$Address);
 5068   %}
 5069 
 5070   ins_pipe(ialu_reg_mem);
 5071 %}
 5072 
 5073 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5074 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5075   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5076 
 5077   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5078   ins_encode %{
 5079     __ movzbq($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5085 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087   effect(KILL cr);
 5088 
 5089   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5090             "andl    $dst, right_n_bits($mask, 16)" %}
 5091   ins_encode %{
 5092     Register Rdst = $dst$$Register;
 5093     __ movzwq(Rdst, $mem$$Address);
 5094     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5095   %}
 5096   ins_pipe(ialu_reg_mem);
 5097 %}
 5098 
 5099 // Load Integer
 5100 instruct loadI(rRegI dst, memory mem)
 5101 %{
 5102   match(Set dst (LoadI mem));
 5103 
 5104   ins_cost(125);
 5105   format %{ "movl    $dst, $mem\t# int" %}
 5106 
 5107   ins_encode %{
 5108     __ movl($dst$$Register, $mem$$Address);
 5109   %}
 5110 
 5111   ins_pipe(ialu_reg_mem);
 5112 %}
 5113 
 5114 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5115 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5116   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5117 
 5118   ins_cost(125);
 5119   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5120   ins_encode %{
 5121     __ movsbl($dst$$Register, $mem$$Address);
 5122   %}
 5123   ins_pipe(ialu_reg_mem);
 5124 %}
 5125 
 5126 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5127 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5128   match(Set dst (AndI (LoadI mem) mask));
 5129 
 5130   ins_cost(125);
 5131   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5132   ins_encode %{
 5133     __ movzbl($dst$$Register, $mem$$Address);
 5134   %}
 5135   ins_pipe(ialu_reg_mem);
 5136 %}
 5137 
 5138 // Load Integer (32 bit signed) to Short (16 bit signed)
 5139 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5140   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5141 
 5142   ins_cost(125);
 5143   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5144   ins_encode %{
 5145     __ movswl($dst$$Register, $mem$$Address);
 5146   %}
 5147   ins_pipe(ialu_reg_mem);
 5148 %}
 5149 
 5150 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5151 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5152   match(Set dst (AndI (LoadI mem) mask));
 5153 
 5154   ins_cost(125);
 5155   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5156   ins_encode %{
 5157     __ movzwl($dst$$Register, $mem$$Address);
 5158   %}
 5159   ins_pipe(ialu_reg_mem);
 5160 %}
 5161 
 5162 // Load Integer into Long Register
 5163 instruct loadI2L(rRegL dst, memory mem)
 5164 %{
 5165   match(Set dst (ConvI2L (LoadI mem)));
 5166 
 5167   ins_cost(125);
 5168   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5169 
 5170   ins_encode %{
 5171     __ movslq($dst$$Register, $mem$$Address);
 5172   %}
 5173 
 5174   ins_pipe(ialu_reg_mem);
 5175 %}
 5176 
 5177 // Load Integer with mask 0xFF into Long Register
 5178 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5179   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5180 
 5181   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5182   ins_encode %{
 5183     __ movzbq($dst$$Register, $mem$$Address);
 5184   %}
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFFFF into Long Register
 5189 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5193   ins_encode %{
 5194     __ movzwq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with a 31-bit mask into Long Register
 5200 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202   effect(KILL cr);
 5203 
 5204   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5205             "andl    $dst, $mask" %}
 5206   ins_encode %{
 5207     Register Rdst = $dst$$Register;
 5208     __ movl(Rdst, $mem$$Address);
 5209     __ andl(Rdst, $mask$$constant);
 5210   %}
 5211   ins_pipe(ialu_reg_mem);
 5212 %}
 5213 
 5214 // Load Unsigned Integer into Long Register
 5215 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5216 %{
 5217   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5218 
 5219   ins_cost(125);
 5220   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5221 
 5222   ins_encode %{
 5223     __ movl($dst$$Register, $mem$$Address);
 5224   %}
 5225 
 5226   ins_pipe(ialu_reg_mem);
 5227 %}
 5228 
 5229 // Load Long
 5230 instruct loadL(rRegL dst, memory mem)
 5231 %{
 5232   match(Set dst (LoadL mem));
 5233 
 5234   ins_cost(125);
 5235   format %{ "movq    $dst, $mem\t# long" %}
 5236 
 5237   ins_encode %{
 5238     __ movq($dst$$Register, $mem$$Address);
 5239   %}
 5240 
 5241   ins_pipe(ialu_reg_mem); // XXX
 5242 %}
 5243 
 5244 // Load Range
 5245 instruct loadRange(rRegI dst, memory mem)
 5246 %{
 5247   match(Set dst (LoadRange mem));
 5248 
 5249   ins_cost(125); // XXX
 5250   format %{ "movl    $dst, $mem\t# range" %}
 5251   ins_encode %{
 5252     __ movl($dst$$Register, $mem$$Address);
 5253   %}
 5254   ins_pipe(ialu_reg_mem);
 5255 %}
 5256 
 5257 // Load Pointer
 5258 instruct loadP(rRegP dst, memory mem)
 5259 %{
 5260   match(Set dst (LoadP mem));
 5261   predicate(n->as_Load()->barrier_data() == 0);
 5262 
 5263   ins_cost(125); // XXX
 5264   format %{ "movq    $dst, $mem\t# ptr" %}
 5265   ins_encode %{
 5266     __ movq($dst$$Register, $mem$$Address);
 5267   %}
 5268   ins_pipe(ialu_reg_mem); // XXX
 5269 %}
 5270 
 5271 // Load Compressed Pointer
 5272 instruct loadN(rRegN dst, memory mem)
 5273 %{
 5274    match(Set dst (LoadN mem));
 5275 
 5276    ins_cost(125); // XXX
 5277    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5278    ins_encode %{
 5279      __ movl($dst$$Register, $mem$$Address);
 5280    %}
 5281    ins_pipe(ialu_reg_mem); // XXX
 5282 %}
 5283 
 5284 
 5285 // Load Klass Pointer
 5286 instruct loadKlass(rRegP dst, memory mem)
 5287 %{
 5288   match(Set dst (LoadKlass mem));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movq    $dst, $mem\t# class" %}
 5292   ins_encode %{
 5293     __ movq($dst$$Register, $mem$$Address);
 5294   %}
 5295   ins_pipe(ialu_reg_mem); // XXX
 5296 %}
 5297 
 5298 // Load narrow Klass Pointer
 5299 instruct loadNKlass(rRegN dst, memory mem)
 5300 %{
 5301   match(Set dst (LoadNKlass mem));
 5302 
 5303   ins_cost(125); // XXX
 5304   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5305   ins_encode %{
 5306     __ movl($dst$$Register, $mem$$Address);
 5307   %}
 5308   ins_pipe(ialu_reg_mem); // XXX
 5309 %}
 5310 
 5311 // Load Float
 5312 instruct loadF(regF dst, memory mem)
 5313 %{
 5314   match(Set dst (LoadF mem));
 5315 
 5316   ins_cost(145); // XXX
 5317   format %{ "movss   $dst, $mem\t# float" %}
 5318   ins_encode %{
 5319     __ movflt($dst$$XMMRegister, $mem$$Address);
 5320   %}
 5321   ins_pipe(pipe_slow); // XXX
 5322 %}
 5323 
 5324 // Load Double
 5325 instruct loadD_partial(regD dst, memory mem)
 5326 %{
 5327   predicate(!UseXmmLoadAndClearUpper);
 5328   match(Set dst (LoadD mem));
 5329 
 5330   ins_cost(145); // XXX
 5331   format %{ "movlpd  $dst, $mem\t# double" %}
 5332   ins_encode %{
 5333     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5334   %}
 5335   ins_pipe(pipe_slow); // XXX
 5336 %}
 5337 
 5338 instruct loadD(regD dst, memory mem)
 5339 %{
 5340   predicate(UseXmmLoadAndClearUpper);
 5341   match(Set dst (LoadD mem));
 5342 
 5343   ins_cost(145); // XXX
 5344   format %{ "movsd   $dst, $mem\t# double" %}
 5345   ins_encode %{
 5346     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5347   %}
 5348   ins_pipe(pipe_slow); // XXX
 5349 %}
 5350 
 5351 
 5352 // Following pseudo code describes the algorithm for max[FD]:
 5353 // Min algorithm is on similar lines
 5354 //  btmp = (b < +0.0) ? a : b
 5355 //  atmp = (b < +0.0) ? b : a
 5356 //  Tmp  = Max_Float(atmp , btmp)
 5357 //  Res  = (atmp == NaN) ? atmp : Tmp
 5358 
 5359 // max = java.lang.Math.max(float a, float b)
 5360 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5361   predicate(UseAVX > 0 && !n->is_reduction());
 5362   match(Set dst (MaxF a b));
 5363   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5364   format %{
 5365      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5366      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5367      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5368      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5369      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5370   %}
 5371   ins_encode %{
 5372     int vector_len = Assembler::AVX_128bit;
 5373     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5374     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5375     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5376     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5377     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5378  %}
 5379   ins_pipe( pipe_slow );
 5380 %}
 5381 
 5382 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5383   predicate(UseAVX > 0 && n->is_reduction());
 5384   match(Set dst (MaxF a b));
 5385   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5386 
 5387   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5388   ins_encode %{
 5389     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5390                     false /*min*/, true /*single*/);
 5391   %}
 5392   ins_pipe( pipe_slow );
 5393 %}
 5394 
 5395 // max = java.lang.Math.max(double a, double b)
 5396 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5397   predicate(UseAVX > 0 && !n->is_reduction());
 5398   match(Set dst (MaxD a b));
 5399   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5400   format %{
 5401      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5402      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5403      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5404      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5405      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5406   %}
 5407   ins_encode %{
 5408     int vector_len = Assembler::AVX_128bit;
 5409     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5410     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5411     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5412     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5413     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5414   %}
 5415   ins_pipe( pipe_slow );
 5416 %}
 5417 
 5418 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5419   predicate(UseAVX > 0 && n->is_reduction());
 5420   match(Set dst (MaxD a b));
 5421   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5422 
 5423   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5424   ins_encode %{
 5425     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5426                     false /*min*/, false /*single*/);
 5427   %}
 5428   ins_pipe( pipe_slow );
 5429 %}
 5430 
 5431 // min = java.lang.Math.min(float a, float b)
 5432 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5433   predicate(UseAVX > 0 && !n->is_reduction());
 5434   match(Set dst (MinF a b));
 5435   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5436   format %{
 5437      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5438      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5439      "vminss           $tmp,$atmp,$btmp           \n\t"
 5440      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5441      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5442   %}
 5443   ins_encode %{
 5444     int vector_len = Assembler::AVX_128bit;
 5445     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5446     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5447     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5448     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5449     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5450   %}
 5451   ins_pipe( pipe_slow );
 5452 %}
 5453 
 5454 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5455   predicate(UseAVX > 0 && n->is_reduction());
 5456   match(Set dst (MinF a b));
 5457   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5458 
 5459   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5460   ins_encode %{
 5461     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5462                     true /*min*/, true /*single*/);
 5463   %}
 5464   ins_pipe( pipe_slow );
 5465 %}
 5466 
 5467 // min = java.lang.Math.min(double a, double b)
 5468 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5469   predicate(UseAVX > 0 && !n->is_reduction());
 5470   match(Set dst (MinD a b));
 5471   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5472   format %{
 5473      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5474      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5475      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5476      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5477      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5478   %}
 5479   ins_encode %{
 5480     int vector_len = Assembler::AVX_128bit;
 5481     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5482     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5483     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5484     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5485     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5486   %}
 5487   ins_pipe( pipe_slow );
 5488 %}
 5489 
 5490 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5491   predicate(UseAVX > 0 && n->is_reduction());
 5492   match(Set dst (MinD a b));
 5493   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5494 
 5495   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5496   ins_encode %{
 5497     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5498                     true /*min*/, false /*single*/);
 5499   %}
 5500   ins_pipe( pipe_slow );
 5501 %}
 5502 
 5503 // Load Effective Address
 5504 instruct leaP8(rRegP dst, indOffset8 mem)
 5505 %{
 5506   match(Set dst mem);
 5507 
 5508   ins_cost(110); // XXX
 5509   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5510   ins_encode %{
 5511     __ leaq($dst$$Register, $mem$$Address);
 5512   %}
 5513   ins_pipe(ialu_reg_reg_fat);
 5514 %}
 5515 
 5516 instruct leaP32(rRegP dst, indOffset32 mem)
 5517 %{
 5518   match(Set dst mem);
 5519 
 5520   ins_cost(110);
 5521   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5522   ins_encode %{
 5523     __ leaq($dst$$Register, $mem$$Address);
 5524   %}
 5525   ins_pipe(ialu_reg_reg_fat);
 5526 %}
 5527 
 5528 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5529 %{
 5530   match(Set dst mem);
 5531 
 5532   ins_cost(110);
 5533   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5534   ins_encode %{
 5535     __ leaq($dst$$Register, $mem$$Address);
 5536   %}
 5537   ins_pipe(ialu_reg_reg_fat);
 5538 %}
 5539 
 5540 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5541 %{
 5542   match(Set dst mem);
 5543 
 5544   ins_cost(110);
 5545   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5546   ins_encode %{
 5547     __ leaq($dst$$Register, $mem$$Address);
 5548   %}
 5549   ins_pipe(ialu_reg_reg_fat);
 5550 %}
 5551 
 5552 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5553 %{
 5554   match(Set dst mem);
 5555 
 5556   ins_cost(110);
 5557   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5558   ins_encode %{
 5559     __ leaq($dst$$Register, $mem$$Address);
 5560   %}
 5561   ins_pipe(ialu_reg_reg_fat);
 5562 %}
 5563 
 5564 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5565 %{
 5566   match(Set dst mem);
 5567 
 5568   ins_cost(110);
 5569   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5570   ins_encode %{
 5571     __ leaq($dst$$Register, $mem$$Address);
 5572   %}
 5573   ins_pipe(ialu_reg_reg_fat);
 5574 %}
 5575 
 5576 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5577 %{
 5578   match(Set dst mem);
 5579 
 5580   ins_cost(110);
 5581   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5582   ins_encode %{
 5583     __ leaq($dst$$Register, $mem$$Address);
 5584   %}
 5585   ins_pipe(ialu_reg_reg_fat);
 5586 %}
 5587 
 5588 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5589 %{
 5590   match(Set dst mem);
 5591 
 5592   ins_cost(110);
 5593   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5594   ins_encode %{
 5595     __ leaq($dst$$Register, $mem$$Address);
 5596   %}
 5597   ins_pipe(ialu_reg_reg_fat);
 5598 %}
 5599 
 5600 // Load Effective Address which uses Narrow (32-bits) oop
 5601 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5602 %{
 5603   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5604   match(Set dst mem);
 5605 
 5606   ins_cost(110);
 5607   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5608   ins_encode %{
 5609     __ leaq($dst$$Register, $mem$$Address);
 5610   %}
 5611   ins_pipe(ialu_reg_reg_fat);
 5612 %}
 5613 
 5614 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5615 %{
 5616   predicate(CompressedOops::shift() == 0);
 5617   match(Set dst mem);
 5618 
 5619   ins_cost(110); // XXX
 5620   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5621   ins_encode %{
 5622     __ leaq($dst$$Register, $mem$$Address);
 5623   %}
 5624   ins_pipe(ialu_reg_reg_fat);
 5625 %}
 5626 
 5627 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5628 %{
 5629   predicate(CompressedOops::shift() == 0);
 5630   match(Set dst mem);
 5631 
 5632   ins_cost(110);
 5633   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5634   ins_encode %{
 5635     __ leaq($dst$$Register, $mem$$Address);
 5636   %}
 5637   ins_pipe(ialu_reg_reg_fat);
 5638 %}
 5639 
 5640 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5641 %{
 5642   predicate(CompressedOops::shift() == 0);
 5643   match(Set dst mem);
 5644 
 5645   ins_cost(110);
 5646   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5647   ins_encode %{
 5648     __ leaq($dst$$Register, $mem$$Address);
 5649   %}
 5650   ins_pipe(ialu_reg_reg_fat);
 5651 %}
 5652 
 5653 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5654 %{
 5655   predicate(CompressedOops::shift() == 0);
 5656   match(Set dst mem);
 5657 
 5658   ins_cost(110);
 5659   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5660   ins_encode %{
 5661     __ leaq($dst$$Register, $mem$$Address);
 5662   %}
 5663   ins_pipe(ialu_reg_reg_fat);
 5664 %}
 5665 
 5666 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5667 %{
 5668   predicate(CompressedOops::shift() == 0);
 5669   match(Set dst mem);
 5670 
 5671   ins_cost(110);
 5672   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5673   ins_encode %{
 5674     __ leaq($dst$$Register, $mem$$Address);
 5675   %}
 5676   ins_pipe(ialu_reg_reg_fat);
 5677 %}
 5678 
 5679 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5680 %{
 5681   predicate(CompressedOops::shift() == 0);
 5682   match(Set dst mem);
 5683 
 5684   ins_cost(110);
 5685   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5686   ins_encode %{
 5687     __ leaq($dst$$Register, $mem$$Address);
 5688   %}
 5689   ins_pipe(ialu_reg_reg_fat);
 5690 %}
 5691 
 5692 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5693 %{
 5694   predicate(CompressedOops::shift() == 0);
 5695   match(Set dst mem);
 5696 
 5697   ins_cost(110);
 5698   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5699   ins_encode %{
 5700     __ leaq($dst$$Register, $mem$$Address);
 5701   %}
 5702   ins_pipe(ialu_reg_reg_fat);
 5703 %}
 5704 
 5705 instruct loadConI(rRegI dst, immI src)
 5706 %{
 5707   match(Set dst src);
 5708 
 5709   format %{ "movl    $dst, $src\t# int" %}
 5710   ins_encode %{
 5711     __ movl($dst$$Register, $src$$constant);
 5712   %}
 5713   ins_pipe(ialu_reg_fat); // XXX
 5714 %}
 5715 
 5716 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5717 %{
 5718   match(Set dst src);
 5719   effect(KILL cr);
 5720 
 5721   ins_cost(50);
 5722   format %{ "xorl    $dst, $dst\t# int" %}
 5723   ins_encode %{
 5724     __ xorl($dst$$Register, $dst$$Register);
 5725   %}
 5726   ins_pipe(ialu_reg);
 5727 %}
 5728 
 5729 instruct loadConL(rRegL dst, immL src)
 5730 %{
 5731   match(Set dst src);
 5732 
 5733   ins_cost(150);
 5734   format %{ "movq    $dst, $src\t# long" %}
 5735   ins_encode %{
 5736     __ mov64($dst$$Register, $src$$constant);
 5737   %}
 5738   ins_pipe(ialu_reg);
 5739 %}
 5740 
 5741 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5742 %{
 5743   match(Set dst src);
 5744   effect(KILL cr);
 5745 
 5746   ins_cost(50);
 5747   format %{ "xorl    $dst, $dst\t# long" %}
 5748   ins_encode %{
 5749     __ xorl($dst$$Register, $dst$$Register);
 5750   %}
 5751   ins_pipe(ialu_reg); // XXX
 5752 %}
 5753 
 5754 instruct loadConUL32(rRegL dst, immUL32 src)
 5755 %{
 5756   match(Set dst src);
 5757 
 5758   ins_cost(60);
 5759   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5760   ins_encode %{
 5761     __ movl($dst$$Register, $src$$constant);
 5762   %}
 5763   ins_pipe(ialu_reg);
 5764 %}
 5765 
 5766 instruct loadConL32(rRegL dst, immL32 src)
 5767 %{
 5768   match(Set dst src);
 5769 
 5770   ins_cost(70);
 5771   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5772   ins_encode %{
 5773     __ movq($dst$$Register, $src$$constant);
 5774   %}
 5775   ins_pipe(ialu_reg);
 5776 %}
 5777 
 5778 instruct loadConP(rRegP dst, immP con) %{
 5779   match(Set dst con);
 5780 
 5781   format %{ "movq    $dst, $con\t# ptr" %}
 5782   ins_encode %{
 5783     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5784   %}
 5785   ins_pipe(ialu_reg_fat); // XXX
 5786 %}
 5787 
 5788 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5789 %{
 5790   match(Set dst src);
 5791   effect(KILL cr);
 5792 
 5793   ins_cost(50);
 5794   format %{ "xorl    $dst, $dst\t# ptr" %}
 5795   ins_encode %{
 5796     __ xorl($dst$$Register, $dst$$Register);
 5797   %}
 5798   ins_pipe(ialu_reg);
 5799 %}
 5800 
 5801 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5802 %{
 5803   match(Set dst src);
 5804   effect(KILL cr);
 5805 
 5806   ins_cost(60);
 5807   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5808   ins_encode %{
 5809     __ movl($dst$$Register, $src$$constant);
 5810   %}
 5811   ins_pipe(ialu_reg);
 5812 %}
 5813 
 5814 instruct loadConF(regF dst, immF con) %{
 5815   match(Set dst con);
 5816   ins_cost(125);
 5817   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5818   ins_encode %{
 5819     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5820   %}
 5821   ins_pipe(pipe_slow);
 5822 %}
 5823 
 5824 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5825   match(Set dst src);
 5826   effect(KILL cr);
 5827   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5828   ins_encode %{
 5829     __ xorq($dst$$Register, $dst$$Register);
 5830   %}
 5831   ins_pipe(ialu_reg);
 5832 %}
 5833 
 5834 instruct loadConN(rRegN dst, immN src) %{
 5835   match(Set dst src);
 5836 
 5837   ins_cost(125);
 5838   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5839   ins_encode %{
 5840     address con = (address)$src$$constant;
 5841     if (con == NULL) {
 5842       ShouldNotReachHere();
 5843     } else {
 5844       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5845     }
 5846   %}
 5847   ins_pipe(ialu_reg_fat); // XXX
 5848 %}
 5849 
 5850 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5851   match(Set dst src);
 5852 
 5853   ins_cost(125);
 5854   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5855   ins_encode %{
 5856     address con = (address)$src$$constant;
 5857     if (con == NULL) {
 5858       ShouldNotReachHere();
 5859     } else {
 5860       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5861     }
 5862   %}
 5863   ins_pipe(ialu_reg_fat); // XXX
 5864 %}
 5865 
 5866 instruct loadConF0(regF dst, immF0 src)
 5867 %{
 5868   match(Set dst src);
 5869   ins_cost(100);
 5870 
 5871   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5872   ins_encode %{
 5873     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5874   %}
 5875   ins_pipe(pipe_slow);
 5876 %}
 5877 
 5878 // Use the same format since predicate() can not be used here.
 5879 instruct loadConD(regD dst, immD con) %{
 5880   match(Set dst con);
 5881   ins_cost(125);
 5882   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5883   ins_encode %{
 5884     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5885   %}
 5886   ins_pipe(pipe_slow);
 5887 %}
 5888 
 5889 instruct loadConD0(regD dst, immD0 src)
 5890 %{
 5891   match(Set dst src);
 5892   ins_cost(100);
 5893 
 5894   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5895   ins_encode %{
 5896     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5897   %}
 5898   ins_pipe(pipe_slow);
 5899 %}
 5900 
 5901 instruct loadSSI(rRegI dst, stackSlotI src)
 5902 %{
 5903   match(Set dst src);
 5904 
 5905   ins_cost(125);
 5906   format %{ "movl    $dst, $src\t# int stk" %}
 5907   opcode(0x8B);
 5908   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5909   ins_pipe(ialu_reg_mem);
 5910 %}
 5911 
 5912 instruct loadSSL(rRegL dst, stackSlotL src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movq    $dst, $src\t# long stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSP(rRegP dst, stackSlotP src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movq    $dst, $src\t# ptr stk" %}
 5929   opcode(0x8B);
 5930   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5931   ins_pipe(ialu_reg_mem);
 5932 %}
 5933 
 5934 instruct loadSSF(regF dst, stackSlotF src)
 5935 %{
 5936   match(Set dst src);
 5937 
 5938   ins_cost(125);
 5939   format %{ "movss   $dst, $src\t# float stk" %}
 5940   ins_encode %{
 5941     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5942   %}
 5943   ins_pipe(pipe_slow); // XXX
 5944 %}
 5945 
 5946 // Use the same format since predicate() can not be used here.
 5947 instruct loadSSD(regD dst, stackSlotD src)
 5948 %{
 5949   match(Set dst src);
 5950 
 5951   ins_cost(125);
 5952   format %{ "movsd   $dst, $src\t# double stk" %}
 5953   ins_encode  %{
 5954     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5955   %}
 5956   ins_pipe(pipe_slow); // XXX
 5957 %}
 5958 
 5959 // Prefetch instructions for allocation.
 5960 // Must be safe to execute with invalid address (cannot fault).
 5961 
 5962 instruct prefetchAlloc( memory mem ) %{
 5963   predicate(AllocatePrefetchInstr==3);
 5964   match(PrefetchAllocation mem);
 5965   ins_cost(125);
 5966 
 5967   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5968   ins_encode %{
 5969     __ prefetchw($mem$$Address);
 5970   %}
 5971   ins_pipe(ialu_mem);
 5972 %}
 5973 
 5974 instruct prefetchAllocNTA( memory mem ) %{
 5975   predicate(AllocatePrefetchInstr==0);
 5976   match(PrefetchAllocation mem);
 5977   ins_cost(125);
 5978 
 5979   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5980   ins_encode %{
 5981     __ prefetchnta($mem$$Address);
 5982   %}
 5983   ins_pipe(ialu_mem);
 5984 %}
 5985 
 5986 instruct prefetchAllocT0( memory mem ) %{
 5987   predicate(AllocatePrefetchInstr==1);
 5988   match(PrefetchAllocation mem);
 5989   ins_cost(125);
 5990 
 5991   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5992   ins_encode %{
 5993     __ prefetcht0($mem$$Address);
 5994   %}
 5995   ins_pipe(ialu_mem);
 5996 %}
 5997 
 5998 instruct prefetchAllocT2( memory mem ) %{
 5999   predicate(AllocatePrefetchInstr==2);
 6000   match(PrefetchAllocation mem);
 6001   ins_cost(125);
 6002 
 6003   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6004   ins_encode %{
 6005     __ prefetcht2($mem$$Address);
 6006   %}
 6007   ins_pipe(ialu_mem);
 6008 %}
 6009 
 6010 //----------Store Instructions-------------------------------------------------
 6011 
 6012 // Store Byte
 6013 instruct storeB(memory mem, rRegI src)
 6014 %{
 6015   match(Set mem (StoreB mem src));
 6016 
 6017   ins_cost(125); // XXX
 6018   format %{ "movb    $mem, $src\t# byte" %}
 6019   ins_encode %{
 6020     __ movb($mem$$Address, $src$$Register);
 6021   %}
 6022   ins_pipe(ialu_mem_reg);
 6023 %}
 6024 
 6025 // Store Char/Short
 6026 instruct storeC(memory mem, rRegI src)
 6027 %{
 6028   match(Set mem (StoreC mem src));
 6029 
 6030   ins_cost(125); // XXX
 6031   format %{ "movw    $mem, $src\t# char/short" %}
 6032   ins_encode %{
 6033     __ movw($mem$$Address, $src$$Register);
 6034   %}
 6035   ins_pipe(ialu_mem_reg);
 6036 %}
 6037 
 6038 // Store Integer
 6039 instruct storeI(memory mem, rRegI src)
 6040 %{
 6041   match(Set mem (StoreI mem src));
 6042 
 6043   ins_cost(125); // XXX
 6044   format %{ "movl    $mem, $src\t# int" %}
 6045   ins_encode %{
 6046     __ movl($mem$$Address, $src$$Register);
 6047   %}
 6048   ins_pipe(ialu_mem_reg);
 6049 %}
 6050 
 6051 // Store Long
 6052 instruct storeL(memory mem, rRegL src)
 6053 %{
 6054   match(Set mem (StoreL mem src));
 6055 
 6056   ins_cost(125); // XXX
 6057   format %{ "movq    $mem, $src\t# long" %}
 6058   ins_encode %{
 6059     __ movq($mem$$Address, $src$$Register);
 6060   %}
 6061   ins_pipe(ialu_mem_reg); // XXX
 6062 %}
 6063 
 6064 // Store Pointer
 6065 instruct storeP(memory mem, any_RegP src)
 6066 %{
 6067   match(Set mem (StoreP mem src));
 6068 
 6069   ins_cost(125); // XXX
 6070   format %{ "movq    $mem, $src\t# ptr" %}
 6071   ins_encode %{
 6072     __ movq($mem$$Address, $src$$Register);
 6073   %}
 6074   ins_pipe(ialu_mem_reg);
 6075 %}
 6076 
 6077 instruct storeImmP0(memory mem, immP0 zero)
 6078 %{
 6079   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6080   match(Set mem (StoreP mem zero));
 6081 
 6082   ins_cost(125); // XXX
 6083   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6084   ins_encode %{
 6085     __ movq($mem$$Address, r12);
 6086   %}
 6087   ins_pipe(ialu_mem_reg);
 6088 %}
 6089 
 6090 // Store NULL Pointer, mark word, or other simple pointer constant.
 6091 instruct storeImmP(memory mem, immP31 src)
 6092 %{
 6093   match(Set mem (StoreP mem src));
 6094 
 6095   ins_cost(150); // XXX
 6096   format %{ "movq    $mem, $src\t# ptr" %}
 6097   ins_encode %{
 6098     __ movq($mem$$Address, $src$$constant);
 6099   %}
 6100   ins_pipe(ialu_mem_imm);
 6101 %}
 6102 
 6103 // Store Compressed Pointer
 6104 instruct storeN(memory mem, rRegN src)
 6105 %{
 6106   match(Set mem (StoreN mem src));
 6107 
 6108   ins_cost(125); // XXX
 6109   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6110   ins_encode %{
 6111     __ movl($mem$$Address, $src$$Register);
 6112   %}
 6113   ins_pipe(ialu_mem_reg);
 6114 %}
 6115 
 6116 instruct storeNKlass(memory mem, rRegN src)
 6117 %{
 6118   match(Set mem (StoreNKlass mem src));
 6119 
 6120   ins_cost(125); // XXX
 6121   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6122   ins_encode %{
 6123     __ movl($mem$$Address, $src$$Register);
 6124   %}
 6125   ins_pipe(ialu_mem_reg);
 6126 %}
 6127 
 6128 instruct storeImmN0(memory mem, immN0 zero)
 6129 %{
 6130   predicate(CompressedOops::base() == NULL);
 6131   match(Set mem (StoreN mem zero));
 6132 
 6133   ins_cost(125); // XXX
 6134   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6135   ins_encode %{
 6136     __ movl($mem$$Address, r12);
 6137   %}
 6138   ins_pipe(ialu_mem_reg);
 6139 %}
 6140 
 6141 instruct storeImmN(memory mem, immN src)
 6142 %{
 6143   match(Set mem (StoreN mem src));
 6144 
 6145   ins_cost(150); // XXX
 6146   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6147   ins_encode %{
 6148     address con = (address)$src$$constant;
 6149     if (con == NULL) {
 6150       __ movl($mem$$Address, (int32_t)0);
 6151     } else {
 6152       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6153     }
 6154   %}
 6155   ins_pipe(ialu_mem_imm);
 6156 %}
 6157 
 6158 instruct storeImmNKlass(memory mem, immNKlass src)
 6159 %{
 6160   match(Set mem (StoreNKlass mem src));
 6161 
 6162   ins_cost(150); // XXX
 6163   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6164   ins_encode %{
 6165     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6166   %}
 6167   ins_pipe(ialu_mem_imm);
 6168 %}
 6169 
 6170 // Store Integer Immediate
 6171 instruct storeImmI0(memory mem, immI_0 zero)
 6172 %{
 6173   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6174   match(Set mem (StoreI mem zero));
 6175 
 6176   ins_cost(125); // XXX
 6177   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6178   ins_encode %{
 6179     __ movl($mem$$Address, r12);
 6180   %}
 6181   ins_pipe(ialu_mem_reg);
 6182 %}
 6183 
 6184 instruct storeImmI(memory mem, immI src)
 6185 %{
 6186   match(Set mem (StoreI mem src));
 6187 
 6188   ins_cost(150);
 6189   format %{ "movl    $mem, $src\t# int" %}
 6190   ins_encode %{
 6191     __ movl($mem$$Address, $src$$constant);
 6192   %}
 6193   ins_pipe(ialu_mem_imm);
 6194 %}
 6195 
 6196 // Store Long Immediate
 6197 instruct storeImmL0(memory mem, immL0 zero)
 6198 %{
 6199   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6200   match(Set mem (StoreL mem zero));
 6201 
 6202   ins_cost(125); // XXX
 6203   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6204   ins_encode %{
 6205     __ movq($mem$$Address, r12);
 6206   %}
 6207   ins_pipe(ialu_mem_reg);
 6208 %}
 6209 
 6210 instruct storeImmL(memory mem, immL32 src)
 6211 %{
 6212   match(Set mem (StoreL mem src));
 6213 
 6214   ins_cost(150);
 6215   format %{ "movq    $mem, $src\t# long" %}
 6216   ins_encode %{
 6217     __ movq($mem$$Address, $src$$constant);
 6218   %}
 6219   ins_pipe(ialu_mem_imm);
 6220 %}
 6221 
 6222 // Store Short/Char Immediate
 6223 instruct storeImmC0(memory mem, immI_0 zero)
 6224 %{
 6225   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6226   match(Set mem (StoreC mem zero));
 6227 
 6228   ins_cost(125); // XXX
 6229   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6230   ins_encode %{
 6231     __ movw($mem$$Address, r12);
 6232   %}
 6233   ins_pipe(ialu_mem_reg);
 6234 %}
 6235 
 6236 instruct storeImmI16(memory mem, immI16 src)
 6237 %{
 6238   predicate(UseStoreImmI16);
 6239   match(Set mem (StoreC mem src));
 6240 
 6241   ins_cost(150);
 6242   format %{ "movw    $mem, $src\t# short/char" %}
 6243   ins_encode %{
 6244     __ movw($mem$$Address, $src$$constant);
 6245   %}
 6246   ins_pipe(ialu_mem_imm);
 6247 %}
 6248 
 6249 // Store Byte Immediate
 6250 instruct storeImmB0(memory mem, immI_0 zero)
 6251 %{
 6252   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6253   match(Set mem (StoreB mem zero));
 6254 
 6255   ins_cost(125); // XXX
 6256   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6257   ins_encode %{
 6258     __ movb($mem$$Address, r12);
 6259   %}
 6260   ins_pipe(ialu_mem_reg);
 6261 %}
 6262 
 6263 instruct storeImmB(memory mem, immI8 src)
 6264 %{
 6265   match(Set mem (StoreB mem src));
 6266 
 6267   ins_cost(150); // XXX
 6268   format %{ "movb    $mem, $src\t# byte" %}
 6269   ins_encode %{
 6270     __ movb($mem$$Address, $src$$constant);
 6271   %}
 6272   ins_pipe(ialu_mem_imm);
 6273 %}
 6274 
 6275 // Store CMS card-mark Immediate
 6276 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6277 %{
 6278   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6279   match(Set mem (StoreCM mem zero));
 6280 
 6281   ins_cost(125); // XXX
 6282   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6283   ins_encode %{
 6284     __ movb($mem$$Address, r12);
 6285   %}
 6286   ins_pipe(ialu_mem_reg);
 6287 %}
 6288 
 6289 instruct storeImmCM0(memory mem, immI_0 src)
 6290 %{
 6291   match(Set mem (StoreCM mem src));
 6292 
 6293   ins_cost(150); // XXX
 6294   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6295   ins_encode %{
 6296     __ movb($mem$$Address, $src$$constant);
 6297   %}
 6298   ins_pipe(ialu_mem_imm);
 6299 %}
 6300 
 6301 // Store Float
 6302 instruct storeF(memory mem, regF src)
 6303 %{
 6304   match(Set mem (StoreF mem src));
 6305 
 6306   ins_cost(95); // XXX
 6307   format %{ "movss   $mem, $src\t# float" %}
 6308   ins_encode %{
 6309     __ movflt($mem$$Address, $src$$XMMRegister);
 6310   %}
 6311   ins_pipe(pipe_slow); // XXX
 6312 %}
 6313 
 6314 // Store immediate Float value (it is faster than store from XMM register)
 6315 instruct storeF0(memory mem, immF0 zero)
 6316 %{
 6317   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6318   match(Set mem (StoreF mem zero));
 6319 
 6320   ins_cost(25); // XXX
 6321   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6322   ins_encode %{
 6323     __ movl($mem$$Address, r12);
 6324   %}
 6325   ins_pipe(ialu_mem_reg);
 6326 %}
 6327 
 6328 instruct storeF_imm(memory mem, immF src)
 6329 %{
 6330   match(Set mem (StoreF mem src));
 6331 
 6332   ins_cost(50);
 6333   format %{ "movl    $mem, $src\t# float" %}
 6334   ins_encode %{
 6335     __ movl($mem$$Address, jint_cast($src$$constant));
 6336   %}
 6337   ins_pipe(ialu_mem_imm);
 6338 %}
 6339 
 6340 // Store Double
 6341 instruct storeD(memory mem, regD src)
 6342 %{
 6343   match(Set mem (StoreD mem src));
 6344 
 6345   ins_cost(95); // XXX
 6346   format %{ "movsd   $mem, $src\t# double" %}
 6347   ins_encode %{
 6348     __ movdbl($mem$$Address, $src$$XMMRegister);
 6349   %}
 6350   ins_pipe(pipe_slow); // XXX
 6351 %}
 6352 
 6353 // Store immediate double 0.0 (it is faster than store from XMM register)
 6354 instruct storeD0_imm(memory mem, immD0 src)
 6355 %{
 6356   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6357   match(Set mem (StoreD mem src));
 6358 
 6359   ins_cost(50);
 6360   format %{ "movq    $mem, $src\t# double 0." %}
 6361   ins_encode %{
 6362     __ movq($mem$$Address, $src$$constant);
 6363   %}
 6364   ins_pipe(ialu_mem_imm);
 6365 %}
 6366 
 6367 instruct storeD0(memory mem, immD0 zero)
 6368 %{
 6369   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6370   match(Set mem (StoreD mem zero));
 6371 
 6372   ins_cost(25); // XXX
 6373   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6374   ins_encode %{
 6375     __ movq($mem$$Address, r12);
 6376   %}
 6377   ins_pipe(ialu_mem_reg);
 6378 %}
 6379 
 6380 instruct storeSSI(stackSlotI dst, rRegI src)
 6381 %{
 6382   match(Set dst src);
 6383 
 6384   ins_cost(100);
 6385   format %{ "movl    $dst, $src\t# int stk" %}
 6386   opcode(0x89);
 6387   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6388   ins_pipe( ialu_mem_reg );
 6389 %}
 6390 
 6391 instruct storeSSL(stackSlotL dst, rRegL src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movq    $dst, $src\t# long stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe(ialu_mem_reg);
 6400 %}
 6401 
 6402 instruct storeSSP(stackSlotP dst, rRegP src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(100);
 6407   format %{ "movq    $dst, $src\t# ptr stk" %}
 6408   opcode(0x89);
 6409   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6410   ins_pipe(ialu_mem_reg);
 6411 %}
 6412 
 6413 instruct storeSSF(stackSlotF dst, regF src)
 6414 %{
 6415   match(Set dst src);
 6416 
 6417   ins_cost(95); // XXX
 6418   format %{ "movss   $dst, $src\t# float stk" %}
 6419   ins_encode %{
 6420     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6421   %}
 6422   ins_pipe(pipe_slow); // XXX
 6423 %}
 6424 
 6425 instruct storeSSD(stackSlotD dst, regD src)
 6426 %{
 6427   match(Set dst src);
 6428 
 6429   ins_cost(95); // XXX
 6430   format %{ "movsd   $dst, $src\t# double stk" %}
 6431   ins_encode %{
 6432     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6433   %}
 6434   ins_pipe(pipe_slow); // XXX
 6435 %}
 6436 
 6437 instruct cacheWB(indirect addr)
 6438 %{
 6439   predicate(VM_Version::supports_data_cache_line_flush());
 6440   match(CacheWB addr);
 6441 
 6442   ins_cost(100);
 6443   format %{"cache wb $addr" %}
 6444   ins_encode %{
 6445     assert($addr->index_position() < 0, "should be");
 6446     assert($addr$$disp == 0, "should be");
 6447     __ cache_wb(Address($addr$$base$$Register, 0));
 6448   %}
 6449   ins_pipe(pipe_slow); // XXX
 6450 %}
 6451 
 6452 instruct cacheWBPreSync()
 6453 %{
 6454   predicate(VM_Version::supports_data_cache_line_flush());
 6455   match(CacheWBPreSync);
 6456 
 6457   ins_cost(100);
 6458   format %{"cache wb presync" %}
 6459   ins_encode %{
 6460     __ cache_wbsync(true);
 6461   %}
 6462   ins_pipe(pipe_slow); // XXX
 6463 %}
 6464 
 6465 instruct cacheWBPostSync()
 6466 %{
 6467   predicate(VM_Version::supports_data_cache_line_flush());
 6468   match(CacheWBPostSync);
 6469 
 6470   ins_cost(100);
 6471   format %{"cache wb postsync" %}
 6472   ins_encode %{
 6473     __ cache_wbsync(false);
 6474   %}
 6475   ins_pipe(pipe_slow); // XXX
 6476 %}
 6477 
 6478 //----------BSWAP Instructions-------------------------------------------------
 6479 instruct bytes_reverse_int(rRegI dst) %{
 6480   match(Set dst (ReverseBytesI dst));
 6481 
 6482   format %{ "bswapl  $dst" %}
 6483   ins_encode %{
 6484     __ bswapl($dst$$Register);
 6485   %}
 6486   ins_pipe( ialu_reg );
 6487 %}
 6488 
 6489 instruct bytes_reverse_long(rRegL dst) %{
 6490   match(Set dst (ReverseBytesL dst));
 6491 
 6492   format %{ "bswapq  $dst" %}
 6493   ins_encode %{
 6494     __ bswapq($dst$$Register);
 6495   %}
 6496   ins_pipe( ialu_reg);
 6497 %}
 6498 
 6499 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6500   match(Set dst (ReverseBytesUS dst));
 6501   effect(KILL cr);
 6502 
 6503   format %{ "bswapl  $dst\n\t"
 6504             "shrl    $dst,16\n\t" %}
 6505   ins_encode %{
 6506     __ bswapl($dst$$Register);
 6507     __ shrl($dst$$Register, 16);
 6508   %}
 6509   ins_pipe( ialu_reg );
 6510 %}
 6511 
 6512 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6513   match(Set dst (ReverseBytesS dst));
 6514   effect(KILL cr);
 6515 
 6516   format %{ "bswapl  $dst\n\t"
 6517             "sar     $dst,16\n\t" %}
 6518   ins_encode %{
 6519     __ bswapl($dst$$Register);
 6520     __ sarl($dst$$Register, 16);
 6521   %}
 6522   ins_pipe( ialu_reg );
 6523 %}
 6524 
 6525 //---------- Zeros Count Instructions ------------------------------------------
 6526 
 6527 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6528   predicate(UseCountLeadingZerosInstruction);
 6529   match(Set dst (CountLeadingZerosI src));
 6530   effect(KILL cr);
 6531 
 6532   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6533   ins_encode %{
 6534     __ lzcntl($dst$$Register, $src$$Register);
 6535   %}
 6536   ins_pipe(ialu_reg);
 6537 %}
 6538 
 6539 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6540   predicate(!UseCountLeadingZerosInstruction);
 6541   match(Set dst (CountLeadingZerosI src));
 6542   effect(KILL cr);
 6543 
 6544   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6545             "jnz     skip\n\t"
 6546             "movl    $dst, -1\n"
 6547       "skip:\n\t"
 6548             "negl    $dst\n\t"
 6549             "addl    $dst, 31" %}
 6550   ins_encode %{
 6551     Register Rdst = $dst$$Register;
 6552     Register Rsrc = $src$$Register;
 6553     Label skip;
 6554     __ bsrl(Rdst, Rsrc);
 6555     __ jccb(Assembler::notZero, skip);
 6556     __ movl(Rdst, -1);
 6557     __ bind(skip);
 6558     __ negl(Rdst);
 6559     __ addl(Rdst, BitsPerInt - 1);
 6560   %}
 6561   ins_pipe(ialu_reg);
 6562 %}
 6563 
 6564 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6565   predicate(UseCountLeadingZerosInstruction);
 6566   match(Set dst (CountLeadingZerosL src));
 6567   effect(KILL cr);
 6568 
 6569   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6570   ins_encode %{
 6571     __ lzcntq($dst$$Register, $src$$Register);
 6572   %}
 6573   ins_pipe(ialu_reg);
 6574 %}
 6575 
 6576 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6577   predicate(!UseCountLeadingZerosInstruction);
 6578   match(Set dst (CountLeadingZerosL src));
 6579   effect(KILL cr);
 6580 
 6581   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6582             "jnz     skip\n\t"
 6583             "movl    $dst, -1\n"
 6584       "skip:\n\t"
 6585             "negl    $dst\n\t"
 6586             "addl    $dst, 63" %}
 6587   ins_encode %{
 6588     Register Rdst = $dst$$Register;
 6589     Register Rsrc = $src$$Register;
 6590     Label skip;
 6591     __ bsrq(Rdst, Rsrc);
 6592     __ jccb(Assembler::notZero, skip);
 6593     __ movl(Rdst, -1);
 6594     __ bind(skip);
 6595     __ negl(Rdst);
 6596     __ addl(Rdst, BitsPerLong - 1);
 6597   %}
 6598   ins_pipe(ialu_reg);
 6599 %}
 6600 
 6601 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6602   predicate(UseCountTrailingZerosInstruction);
 6603   match(Set dst (CountTrailingZerosI src));
 6604   effect(KILL cr);
 6605 
 6606   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6607   ins_encode %{
 6608     __ tzcntl($dst$$Register, $src$$Register);
 6609   %}
 6610   ins_pipe(ialu_reg);
 6611 %}
 6612 
 6613 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6614   predicate(!UseCountTrailingZerosInstruction);
 6615   match(Set dst (CountTrailingZerosI src));
 6616   effect(KILL cr);
 6617 
 6618   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6619             "jnz     done\n\t"
 6620             "movl    $dst, 32\n"
 6621       "done:" %}
 6622   ins_encode %{
 6623     Register Rdst = $dst$$Register;
 6624     Label done;
 6625     __ bsfl(Rdst, $src$$Register);
 6626     __ jccb(Assembler::notZero, done);
 6627     __ movl(Rdst, BitsPerInt);
 6628     __ bind(done);
 6629   %}
 6630   ins_pipe(ialu_reg);
 6631 %}
 6632 
 6633 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6634   predicate(UseCountTrailingZerosInstruction);
 6635   match(Set dst (CountTrailingZerosL src));
 6636   effect(KILL cr);
 6637 
 6638   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6639   ins_encode %{
 6640     __ tzcntq($dst$$Register, $src$$Register);
 6641   %}
 6642   ins_pipe(ialu_reg);
 6643 %}
 6644 
 6645 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6646   predicate(!UseCountTrailingZerosInstruction);
 6647   match(Set dst (CountTrailingZerosL src));
 6648   effect(KILL cr);
 6649 
 6650   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6651             "jnz     done\n\t"
 6652             "movl    $dst, 64\n"
 6653       "done:" %}
 6654   ins_encode %{
 6655     Register Rdst = $dst$$Register;
 6656     Label done;
 6657     __ bsfq(Rdst, $src$$Register);
 6658     __ jccb(Assembler::notZero, done);
 6659     __ movl(Rdst, BitsPerLong);
 6660     __ bind(done);
 6661   %}
 6662   ins_pipe(ialu_reg);
 6663 %}
 6664 
 6665 
 6666 //---------- Population Count Instructions -------------------------------------
 6667 
 6668 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6669   predicate(UsePopCountInstruction);
 6670   match(Set dst (PopCountI src));
 6671   effect(KILL cr);
 6672 
 6673   format %{ "popcnt  $dst, $src" %}
 6674   ins_encode %{
 6675     __ popcntl($dst$$Register, $src$$Register);
 6676   %}
 6677   ins_pipe(ialu_reg);
 6678 %}
 6679 
 6680 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6681   predicate(UsePopCountInstruction);
 6682   match(Set dst (PopCountI (LoadI mem)));
 6683   effect(KILL cr);
 6684 
 6685   format %{ "popcnt  $dst, $mem" %}
 6686   ins_encode %{
 6687     __ popcntl($dst$$Register, $mem$$Address);
 6688   %}
 6689   ins_pipe(ialu_reg);
 6690 %}
 6691 
 6692 // Note: Long.bitCount(long) returns an int.
 6693 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6694   predicate(UsePopCountInstruction);
 6695   match(Set dst (PopCountL src));
 6696   effect(KILL cr);
 6697 
 6698   format %{ "popcnt  $dst, $src" %}
 6699   ins_encode %{
 6700     __ popcntq($dst$$Register, $src$$Register);
 6701   %}
 6702   ins_pipe(ialu_reg);
 6703 %}
 6704 
 6705 // Note: Long.bitCount(long) returns an int.
 6706 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6707   predicate(UsePopCountInstruction);
 6708   match(Set dst (PopCountL (LoadL mem)));
 6709   effect(KILL cr);
 6710 
 6711   format %{ "popcnt  $dst, $mem" %}
 6712   ins_encode %{
 6713     __ popcntq($dst$$Register, $mem$$Address);
 6714   %}
 6715   ins_pipe(ialu_reg);
 6716 %}
 6717 
 6718 
 6719 //----------MemBar Instructions-----------------------------------------------
 6720 // Memory barrier flavors
 6721 
 6722 instruct membar_acquire()
 6723 %{
 6724   match(MemBarAcquire);
 6725   match(LoadFence);
 6726   ins_cost(0);
 6727 
 6728   size(0);
 6729   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6730   ins_encode();
 6731   ins_pipe(empty);
 6732 %}
 6733 
 6734 instruct membar_acquire_lock()
 6735 %{
 6736   match(MemBarAcquireLock);
 6737   ins_cost(0);
 6738 
 6739   size(0);
 6740   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6741   ins_encode();
 6742   ins_pipe(empty);
 6743 %}
 6744 
 6745 instruct membar_release()
 6746 %{
 6747   match(MemBarRelease);
 6748   match(StoreFence);
 6749   ins_cost(0);
 6750 
 6751   size(0);
 6752   format %{ "MEMBAR-release ! (empty encoding)" %}
 6753   ins_encode();
 6754   ins_pipe(empty);
 6755 %}
 6756 
 6757 instruct membar_release_lock()
 6758 %{
 6759   match(MemBarReleaseLock);
 6760   ins_cost(0);
 6761 
 6762   size(0);
 6763   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6764   ins_encode();
 6765   ins_pipe(empty);
 6766 %}
 6767 
 6768 instruct membar_volatile(rFlagsReg cr) %{
 6769   match(MemBarVolatile);
 6770   effect(KILL cr);
 6771   ins_cost(400);
 6772 
 6773   format %{
 6774     $$template
 6775     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6776   %}
 6777   ins_encode %{
 6778     __ membar(Assembler::StoreLoad);
 6779   %}
 6780   ins_pipe(pipe_slow);
 6781 %}
 6782 
 6783 instruct unnecessary_membar_volatile()
 6784 %{
 6785   match(MemBarVolatile);
 6786   predicate(Matcher::post_store_load_barrier(n));
 6787   ins_cost(0);
 6788 
 6789   size(0);
 6790   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6791   ins_encode();
 6792   ins_pipe(empty);
 6793 %}
 6794 
 6795 instruct membar_storestore() %{
 6796   match(MemBarStoreStore);
 6797   match(StoreStoreFence);
 6798   ins_cost(0);
 6799 
 6800   size(0);
 6801   format %{ "MEMBAR-storestore (empty encoding)" %}
 6802   ins_encode( );
 6803   ins_pipe(empty);
 6804 %}
 6805 
 6806 //----------Move Instructions--------------------------------------------------
 6807 
 6808 instruct castX2P(rRegP dst, rRegL src)
 6809 %{
 6810   match(Set dst (CastX2P src));
 6811 
 6812   format %{ "movq    $dst, $src\t# long->ptr" %}
 6813   ins_encode %{
 6814     if ($dst$$reg != $src$$reg) {
 6815       __ movptr($dst$$Register, $src$$Register);
 6816     }
 6817   %}
 6818   ins_pipe(ialu_reg_reg); // XXX
 6819 %}
 6820 
 6821 instruct castP2X(rRegL dst, rRegP src)
 6822 %{
 6823   match(Set dst (CastP2X src));
 6824 
 6825   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6826   ins_encode %{
 6827     if ($dst$$reg != $src$$reg) {
 6828       __ movptr($dst$$Register, $src$$Register);
 6829     }
 6830   %}
 6831   ins_pipe(ialu_reg_reg); // XXX
 6832 %}
 6833 
 6834 // Convert oop into int for vectors alignment masking
 6835 instruct convP2I(rRegI dst, rRegP src)
 6836 %{
 6837   match(Set dst (ConvL2I (CastP2X src)));
 6838 
 6839   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6840   ins_encode %{
 6841     __ movl($dst$$Register, $src$$Register);
 6842   %}
 6843   ins_pipe(ialu_reg_reg); // XXX
 6844 %}
 6845 
 6846 // Convert compressed oop into int for vectors alignment masking
 6847 // in case of 32bit oops (heap < 4Gb).
 6848 instruct convN2I(rRegI dst, rRegN src)
 6849 %{
 6850   predicate(CompressedOops::shift() == 0);
 6851   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6852 
 6853   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6854   ins_encode %{
 6855     __ movl($dst$$Register, $src$$Register);
 6856   %}
 6857   ins_pipe(ialu_reg_reg); // XXX
 6858 %}
 6859 
 6860 // Convert oop pointer into compressed form
 6861 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6862   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6863   match(Set dst (EncodeP src));
 6864   effect(KILL cr);
 6865   format %{ "encode_heap_oop $dst,$src" %}
 6866   ins_encode %{
 6867     Register s = $src$$Register;
 6868     Register d = $dst$$Register;
 6869     if (s != d) {
 6870       __ movq(d, s);
 6871     }
 6872     __ encode_heap_oop(d);
 6873   %}
 6874   ins_pipe(ialu_reg_long);
 6875 %}
 6876 
 6877 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6878   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6879   match(Set dst (EncodeP src));
 6880   effect(KILL cr);
 6881   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6882   ins_encode %{
 6883     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6884   %}
 6885   ins_pipe(ialu_reg_long);
 6886 %}
 6887 
 6888 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6889   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6890             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6891   match(Set dst (DecodeN src));
 6892   effect(KILL cr);
 6893   format %{ "decode_heap_oop $dst,$src" %}
 6894   ins_encode %{
 6895     Register s = $src$$Register;
 6896     Register d = $dst$$Register;
 6897     if (s != d) {
 6898       __ movq(d, s);
 6899     }
 6900     __ decode_heap_oop(d);
 6901   %}
 6902   ins_pipe(ialu_reg_long);
 6903 %}
 6904 
 6905 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6906   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6907             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6908   match(Set dst (DecodeN src));
 6909   effect(KILL cr);
 6910   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6911   ins_encode %{
 6912     Register s = $src$$Register;
 6913     Register d = $dst$$Register;
 6914     if (s != d) {
 6915       __ decode_heap_oop_not_null(d, s);
 6916     } else {
 6917       __ decode_heap_oop_not_null(d);
 6918     }
 6919   %}
 6920   ins_pipe(ialu_reg_long);
 6921 %}
 6922 
 6923 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6924   match(Set dst (EncodePKlass src));
 6925   effect(TEMP dst, KILL cr);
 6926   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6927   ins_encode %{
 6928     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6929   %}
 6930   ins_pipe(ialu_reg_long);
 6931 %}
 6932 
 6933 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6934   match(Set dst (DecodeNKlass src));
 6935   effect(TEMP dst, KILL cr);
 6936   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6937   ins_encode %{
 6938     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6939   %}
 6940   ins_pipe(ialu_reg_long);
 6941 %}
 6942 
 6943 //----------Conditional Move---------------------------------------------------
 6944 // Jump
 6945 // dummy instruction for generating temp registers
 6946 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6947   match(Jump (LShiftL switch_val shift));
 6948   ins_cost(350);
 6949   predicate(false);
 6950   effect(TEMP dest);
 6951 
 6952   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6953             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6954   ins_encode %{
 6955     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6956     // to do that and the compiler is using that register as one it can allocate.
 6957     // So we build it all by hand.
 6958     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6959     // ArrayAddress dispatch(table, index);
 6960     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6961     __ lea($dest$$Register, $constantaddress);
 6962     __ jmp(dispatch);
 6963   %}
 6964   ins_pipe(pipe_jmp);
 6965 %}
 6966 
 6967 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6968   match(Jump (AddL (LShiftL switch_val shift) offset));
 6969   ins_cost(350);
 6970   effect(TEMP dest);
 6971 
 6972   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6973             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6974   ins_encode %{
 6975     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6976     // to do that and the compiler is using that register as one it can allocate.
 6977     // So we build it all by hand.
 6978     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6979     // ArrayAddress dispatch(table, index);
 6980     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6981     __ lea($dest$$Register, $constantaddress);
 6982     __ jmp(dispatch);
 6983   %}
 6984   ins_pipe(pipe_jmp);
 6985 %}
 6986 
 6987 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6988   match(Jump switch_val);
 6989   ins_cost(350);
 6990   effect(TEMP dest);
 6991 
 6992   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6993             "jmp     [$dest + $switch_val]\n\t" %}
 6994   ins_encode %{
 6995     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6996     // to do that and the compiler is using that register as one it can allocate.
 6997     // So we build it all by hand.
 6998     // Address index(noreg, switch_reg, Address::times_1);
 6999     // ArrayAddress dispatch(table, index);
 7000     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7001     __ lea($dest$$Register, $constantaddress);
 7002     __ jmp(dispatch);
 7003   %}
 7004   ins_pipe(pipe_jmp);
 7005 %}
 7006 
 7007 // Conditional move
 7008 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7009 %{
 7010   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7011 
 7012   ins_cost(200); // XXX
 7013   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7014   ins_encode %{
 7015     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7016   %}
 7017   ins_pipe(pipe_cmov_reg);
 7018 %}
 7019 
 7020 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7021   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7022 
 7023   ins_cost(200); // XXX
 7024   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7025   ins_encode %{
 7026     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7027   %}
 7028   ins_pipe(pipe_cmov_reg);
 7029 %}
 7030 
 7031 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7032   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7033   ins_cost(200);
 7034   expand %{
 7035     cmovI_regU(cop, cr, dst, src);
 7036   %}
 7037 %}
 7038 
 7039 // Conditional move
 7040 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7041   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7042 
 7043   ins_cost(250); // XXX
 7044   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7045   ins_encode %{
 7046     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7047   %}
 7048   ins_pipe(pipe_cmov_mem);
 7049 %}
 7050 
 7051 // Conditional move
 7052 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7053 %{
 7054   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7055 
 7056   ins_cost(250); // XXX
 7057   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7058   ins_encode %{
 7059     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7060   %}
 7061   ins_pipe(pipe_cmov_mem);
 7062 %}
 7063 
 7064 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7065   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7066   ins_cost(250);
 7067   expand %{
 7068     cmovI_memU(cop, cr, dst, src);
 7069   %}
 7070 %}
 7071 
 7072 // Conditional move
 7073 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7074 %{
 7075   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7076 
 7077   ins_cost(200); // XXX
 7078   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7079   ins_encode %{
 7080     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7081   %}
 7082   ins_pipe(pipe_cmov_reg);
 7083 %}
 7084 
 7085 // Conditional move
 7086 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7087 %{
 7088   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7089 
 7090   ins_cost(200); // XXX
 7091   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7092   ins_encode %{
 7093     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7094   %}
 7095   ins_pipe(pipe_cmov_reg);
 7096 %}
 7097 
 7098 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7099   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7100   ins_cost(200);
 7101   expand %{
 7102     cmovN_regU(cop, cr, dst, src);
 7103   %}
 7104 %}
 7105 
 7106 // Conditional move
 7107 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7108 %{
 7109   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7110 
 7111   ins_cost(200); // XXX
 7112   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7113   ins_encode %{
 7114     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7115   %}
 7116   ins_pipe(pipe_cmov_reg);  // XXX
 7117 %}
 7118 
 7119 // Conditional move
 7120 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7121 %{
 7122   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7123 
 7124   ins_cost(200); // XXX
 7125   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7126   ins_encode %{
 7127     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7128   %}
 7129   ins_pipe(pipe_cmov_reg); // XXX
 7130 %}
 7131 
 7132 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7133   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7134   ins_cost(200);
 7135   expand %{
 7136     cmovP_regU(cop, cr, dst, src);
 7137   %}
 7138 %}
 7139 
 7140 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7141 // correctly meets the two pointer arguments; one is an incoming
 7142 // register but the other is a memory operand.  ALSO appears to
 7143 // be buggy with implicit null checks.
 7144 //
 7145 //// Conditional move
 7146 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7147 //%{
 7148 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7149 //  ins_cost(250);
 7150 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7151 //  opcode(0x0F,0x40);
 7152 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7153 //  ins_pipe( pipe_cmov_mem );
 7154 //%}
 7155 //
 7156 //// Conditional move
 7157 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7158 //%{
 7159 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7160 //  ins_cost(250);
 7161 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7162 //  opcode(0x0F,0x40);
 7163 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7164 //  ins_pipe( pipe_cmov_mem );
 7165 //%}
 7166 
 7167 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7168 %{
 7169   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7170 
 7171   ins_cost(200); // XXX
 7172   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7173   ins_encode %{
 7174     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7175   %}
 7176   ins_pipe(pipe_cmov_reg);  // XXX
 7177 %}
 7178 
 7179 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7180 %{
 7181   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7182 
 7183   ins_cost(200); // XXX
 7184   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7185   ins_encode %{
 7186     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7187   %}
 7188   ins_pipe(pipe_cmov_mem);  // XXX
 7189 %}
 7190 
 7191 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7192 %{
 7193   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7194 
 7195   ins_cost(200); // XXX
 7196   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7197   ins_encode %{
 7198     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7199   %}
 7200   ins_pipe(pipe_cmov_reg); // XXX
 7201 %}
 7202 
 7203 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7204   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7205   ins_cost(200);
 7206   expand %{
 7207     cmovL_regU(cop, cr, dst, src);
 7208   %}
 7209 %}
 7210 
 7211 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7212 %{
 7213   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7214 
 7215   ins_cost(200); // XXX
 7216   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7217   ins_encode %{
 7218     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7219   %}
 7220   ins_pipe(pipe_cmov_mem); // XXX
 7221 %}
 7222 
 7223 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7224   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7225   ins_cost(200);
 7226   expand %{
 7227     cmovL_memU(cop, cr, dst, src);
 7228   %}
 7229 %}
 7230 
 7231 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7232 %{
 7233   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7234 
 7235   ins_cost(200); // XXX
 7236   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7237             "movss     $dst, $src\n"
 7238     "skip:" %}
 7239   ins_encode %{
 7240     Label Lskip;
 7241     // Invert sense of branch from sense of CMOV
 7242     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7243     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7244     __ bind(Lskip);
 7245   %}
 7246   ins_pipe(pipe_slow);
 7247 %}
 7248 
 7249 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7250 // %{
 7251 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7252 
 7253 //   ins_cost(200); // XXX
 7254 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7255 //             "movss     $dst, $src\n"
 7256 //     "skip:" %}
 7257 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7258 //   ins_pipe(pipe_slow);
 7259 // %}
 7260 
 7261 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7262 %{
 7263   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7264 
 7265   ins_cost(200); // XXX
 7266   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7267             "movss     $dst, $src\n"
 7268     "skip:" %}
 7269   ins_encode %{
 7270     Label Lskip;
 7271     // Invert sense of branch from sense of CMOV
 7272     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7273     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7274     __ bind(Lskip);
 7275   %}
 7276   ins_pipe(pipe_slow);
 7277 %}
 7278 
 7279 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7280   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7281   ins_cost(200);
 7282   expand %{
 7283     cmovF_regU(cop, cr, dst, src);
 7284   %}
 7285 %}
 7286 
 7287 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7288 %{
 7289   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7290 
 7291   ins_cost(200); // XXX
 7292   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7293             "movsd     $dst, $src\n"
 7294     "skip:" %}
 7295   ins_encode %{
 7296     Label Lskip;
 7297     // Invert sense of branch from sense of CMOV
 7298     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7299     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7300     __ bind(Lskip);
 7301   %}
 7302   ins_pipe(pipe_slow);
 7303 %}
 7304 
 7305 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7306 %{
 7307   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7308 
 7309   ins_cost(200); // XXX
 7310   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7311             "movsd     $dst, $src\n"
 7312     "skip:" %}
 7313   ins_encode %{
 7314     Label Lskip;
 7315     // Invert sense of branch from sense of CMOV
 7316     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7317     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7318     __ bind(Lskip);
 7319   %}
 7320   ins_pipe(pipe_slow);
 7321 %}
 7322 
 7323 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7324   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7325   ins_cost(200);
 7326   expand %{
 7327     cmovD_regU(cop, cr, dst, src);
 7328   %}
 7329 %}
 7330 
 7331 //----------Arithmetic Instructions--------------------------------------------
 7332 //----------Addition Instructions----------------------------------------------
 7333 
 7334 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7335 %{
 7336   match(Set dst (AddI dst src));
 7337   effect(KILL cr);
 7338 
 7339   format %{ "addl    $dst, $src\t# int" %}
 7340   ins_encode %{
 7341     __ addl($dst$$Register, $src$$Register);
 7342   %}
 7343   ins_pipe(ialu_reg_reg);
 7344 %}
 7345 
 7346 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7347 %{
 7348   match(Set dst (AddI dst src));
 7349   effect(KILL cr);
 7350 
 7351   format %{ "addl    $dst, $src\t# int" %}
 7352   ins_encode %{
 7353     __ addl($dst$$Register, $src$$constant);
 7354   %}
 7355   ins_pipe( ialu_reg );
 7356 %}
 7357 
 7358 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7359 %{
 7360   match(Set dst (AddI dst (LoadI src)));
 7361   effect(KILL cr);
 7362 
 7363   ins_cost(125); // XXX
 7364   format %{ "addl    $dst, $src\t# int" %}
 7365   ins_encode %{
 7366     __ addl($dst$$Register, $src$$Address);
 7367   %}
 7368   ins_pipe(ialu_reg_mem);
 7369 %}
 7370 
 7371 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7372 %{
 7373   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7374   effect(KILL cr);
 7375 
 7376   ins_cost(150); // XXX
 7377   format %{ "addl    $dst, $src\t# int" %}
 7378   ins_encode %{
 7379     __ addl($dst$$Address, $src$$Register);
 7380   %}
 7381   ins_pipe(ialu_mem_reg);
 7382 %}
 7383 
 7384 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7385 %{
 7386   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7387   effect(KILL cr);
 7388 
 7389   ins_cost(125); // XXX
 7390   format %{ "addl    $dst, $src\t# int" %}
 7391   ins_encode %{
 7392     __ addl($dst$$Address, $src$$constant);
 7393   %}
 7394   ins_pipe(ialu_mem_imm);
 7395 %}
 7396 
 7397 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7398 %{
 7399   predicate(UseIncDec);
 7400   match(Set dst (AddI dst src));
 7401   effect(KILL cr);
 7402 
 7403   format %{ "incl    $dst\t# int" %}
 7404   ins_encode %{
 7405     __ incrementl($dst$$Register);
 7406   %}
 7407   ins_pipe(ialu_reg);
 7408 %}
 7409 
 7410 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7411 %{
 7412   predicate(UseIncDec);
 7413   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7414   effect(KILL cr);
 7415 
 7416   ins_cost(125); // XXX
 7417   format %{ "incl    $dst\t# int" %}
 7418   ins_encode %{
 7419     __ incrementl($dst$$Address);
 7420   %}
 7421   ins_pipe(ialu_mem_imm);
 7422 %}
 7423 
 7424 // XXX why does that use AddI
 7425 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7426 %{
 7427   predicate(UseIncDec);
 7428   match(Set dst (AddI dst src));
 7429   effect(KILL cr);
 7430 
 7431   format %{ "decl    $dst\t# int" %}
 7432   ins_encode %{
 7433     __ decrementl($dst$$Register);
 7434   %}
 7435   ins_pipe(ialu_reg);
 7436 %}
 7437 
 7438 // XXX why does that use AddI
 7439 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7440 %{
 7441   predicate(UseIncDec);
 7442   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7443   effect(KILL cr);
 7444 
 7445   ins_cost(125); // XXX
 7446   format %{ "decl    $dst\t# int" %}
 7447   ins_encode %{
 7448     __ decrementl($dst$$Address);
 7449   %}
 7450   ins_pipe(ialu_mem_imm);
 7451 %}
 7452 
 7453 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
 7454 %{
 7455   match(Set dst (AddI src0 src1));
 7456 
 7457   ins_cost(110);
 7458   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
 7459   ins_encode %{
 7460     __ leal($dst$$Register, Address($src0$$Register, $src1$$constant));
 7461   %}
 7462   ins_pipe(ialu_reg_reg);
 7463 %}
 7464 
 7465 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7466 %{
 7467   match(Set dst (AddL dst src));
 7468   effect(KILL cr);
 7469 
 7470   format %{ "addq    $dst, $src\t# long" %}
 7471   ins_encode %{
 7472     __ addq($dst$$Register, $src$$Register);
 7473   %}
 7474   ins_pipe(ialu_reg_reg);
 7475 %}
 7476 
 7477 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7478 %{
 7479   match(Set dst (AddL dst src));
 7480   effect(KILL cr);
 7481 
 7482   format %{ "addq    $dst, $src\t# long" %}
 7483   ins_encode %{
 7484     __ addq($dst$$Register, $src$$constant);
 7485   %}
 7486   ins_pipe( ialu_reg );
 7487 %}
 7488 
 7489 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7490 %{
 7491   match(Set dst (AddL dst (LoadL src)));
 7492   effect(KILL cr);
 7493 
 7494   ins_cost(125); // XXX
 7495   format %{ "addq    $dst, $src\t# long" %}
 7496   ins_encode %{
 7497     __ addq($dst$$Register, $src$$Address);
 7498   %}
 7499   ins_pipe(ialu_reg_mem);
 7500 %}
 7501 
 7502 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7503 %{
 7504   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7505   effect(KILL cr);
 7506 
 7507   ins_cost(150); // XXX
 7508   format %{ "addq    $dst, $src\t# long" %}
 7509   ins_encode %{
 7510     __ addq($dst$$Address, $src$$Register);
 7511   %}
 7512   ins_pipe(ialu_mem_reg);
 7513 %}
 7514 
 7515 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7516 %{
 7517   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7518   effect(KILL cr);
 7519 
 7520   ins_cost(125); // XXX
 7521   format %{ "addq    $dst, $src\t# long" %}
 7522   ins_encode %{
 7523     __ addq($dst$$Address, $src$$constant);
 7524   %}
 7525   ins_pipe(ialu_mem_imm);
 7526 %}
 7527 
 7528 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7529 %{
 7530   predicate(UseIncDec);
 7531   match(Set dst (AddL dst src));
 7532   effect(KILL cr);
 7533 
 7534   format %{ "incq    $dst\t# long" %}
 7535   ins_encode %{
 7536     __ incrementq($dst$$Register);
 7537   %}
 7538   ins_pipe(ialu_reg);
 7539 %}
 7540 
 7541 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7542 %{
 7543   predicate(UseIncDec);
 7544   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7545   effect(KILL cr);
 7546 
 7547   ins_cost(125); // XXX
 7548   format %{ "incq    $dst\t# long" %}
 7549   ins_encode %{
 7550     __ incrementq($dst$$Address);
 7551   %}
 7552   ins_pipe(ialu_mem_imm);
 7553 %}
 7554 
 7555 // XXX why does that use AddL
 7556 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7557 %{
 7558   predicate(UseIncDec);
 7559   match(Set dst (AddL dst src));
 7560   effect(KILL cr);
 7561 
 7562   format %{ "decq    $dst\t# long" %}
 7563   ins_encode %{
 7564     __ decrementq($dst$$Register);
 7565   %}
 7566   ins_pipe(ialu_reg);
 7567 %}
 7568 
 7569 // XXX why does that use AddL
 7570 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7571 %{
 7572   predicate(UseIncDec);
 7573   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7574   effect(KILL cr);
 7575 
 7576   ins_cost(125); // XXX
 7577   format %{ "decq    $dst\t# long" %}
 7578   ins_encode %{
 7579     __ decrementq($dst$$Address);
 7580   %}
 7581   ins_pipe(ialu_mem_imm);
 7582 %}
 7583 
 7584 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
 7585 %{
 7586   match(Set dst (AddL src0 src1));
 7587 
 7588   ins_cost(110);
 7589   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
 7590   ins_encode %{
 7591     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7592   %}
 7593   ins_pipe(ialu_reg_reg);
 7594 %}
 7595 
 7596 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7597 %{
 7598   match(Set dst (AddP dst src));
 7599   effect(KILL cr);
 7600 
 7601   format %{ "addq    $dst, $src\t# ptr" %}
 7602   ins_encode %{
 7603     __ addq($dst$$Register, $src$$Register);
 7604   %}
 7605   ins_pipe(ialu_reg_reg);
 7606 %}
 7607 
 7608 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7609 %{
 7610   match(Set dst (AddP dst src));
 7611   effect(KILL cr);
 7612 
 7613   format %{ "addq    $dst, $src\t# ptr" %}
 7614   ins_encode %{
 7615     __ addq($dst$$Register, $src$$constant);
 7616   %}
 7617   ins_pipe( ialu_reg );
 7618 %}
 7619 
 7620 // XXX addP mem ops ????
 7621 
 7622 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
 7623 %{
 7624   match(Set dst (AddP src0 src1));
 7625 
 7626   ins_cost(110);
 7627   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
 7628   ins_encode %{
 7629     __ leaq($dst$$Register, Address($src0$$Register, $src1$$constant));
 7630   %}
 7631   ins_pipe(ialu_reg_reg);
 7632 %}
 7633 
 7634 instruct checkCastPP(rRegP dst)
 7635 %{
 7636   match(Set dst (CheckCastPP dst));
 7637 
 7638   size(0);
 7639   format %{ "# checkcastPP of $dst" %}
 7640   ins_encode(/* empty encoding */);
 7641   ins_pipe(empty);
 7642 %}
 7643 
 7644 instruct castPP(rRegP dst)
 7645 %{
 7646   match(Set dst (CastPP dst));
 7647 
 7648   size(0);
 7649   format %{ "# castPP of $dst" %}
 7650   ins_encode(/* empty encoding */);
 7651   ins_pipe(empty);
 7652 %}
 7653 
 7654 instruct castII(rRegI dst)
 7655 %{
 7656   match(Set dst (CastII dst));
 7657 
 7658   size(0);
 7659   format %{ "# castII of $dst" %}
 7660   ins_encode(/* empty encoding */);
 7661   ins_cost(0);
 7662   ins_pipe(empty);
 7663 %}
 7664 
 7665 instruct castLL(rRegL dst)
 7666 %{
 7667   match(Set dst (CastLL dst));
 7668 
 7669   size(0);
 7670   format %{ "# castLL of $dst" %}
 7671   ins_encode(/* empty encoding */);
 7672   ins_cost(0);
 7673   ins_pipe(empty);
 7674 %}
 7675 
 7676 instruct castFF(regF dst)
 7677 %{
 7678   match(Set dst (CastFF dst));
 7679 
 7680   size(0);
 7681   format %{ "# castFF of $dst" %}
 7682   ins_encode(/* empty encoding */);
 7683   ins_cost(0);
 7684   ins_pipe(empty);
 7685 %}
 7686 
 7687 instruct castDD(regD dst)
 7688 %{
 7689   match(Set dst (CastDD dst));
 7690 
 7691   size(0);
 7692   format %{ "# castDD of $dst" %}
 7693   ins_encode(/* empty encoding */);
 7694   ins_cost(0);
 7695   ins_pipe(empty);
 7696 %}
 7697 
 7698 // LoadP-locked same as a regular LoadP when used with compare-swap
 7699 instruct loadPLocked(rRegP dst, memory mem)
 7700 %{
 7701   match(Set dst (LoadPLocked mem));
 7702 
 7703   ins_cost(125); // XXX
 7704   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7705   ins_encode %{
 7706     __ movq($dst$$Register, $mem$$Address);
 7707   %}
 7708   ins_pipe(ialu_reg_mem); // XXX
 7709 %}
 7710 
 7711 // Conditional-store of the updated heap-top.
 7712 // Used during allocation of the shared heap.
 7713 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7714 
 7715 instruct storePConditional(memory heap_top_ptr,
 7716                            rax_RegP oldval, rRegP newval,
 7717                            rFlagsReg cr)
 7718 %{
 7719   predicate(n->as_LoadStore()->barrier_data() == 0);
 7720   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7721 
 7722   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7723             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7724   ins_encode %{
 7725     __ lock();
 7726     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7727   %}
 7728   ins_pipe(pipe_cmpxchg);
 7729 %}
 7730 
 7731 // Conditional-store of an int value.
 7732 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7733 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7734 %{
 7735   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7736   effect(KILL oldval);
 7737 
 7738   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7739   opcode(0x0F, 0xB1);
 7740   ins_encode(lock_prefix,
 7741              REX_reg_mem(newval, mem),
 7742              OpcP, OpcS,
 7743              reg_mem(newval, mem));
 7744   ins_pipe(pipe_cmpxchg);
 7745 %}
 7746 
 7747 // Conditional-store of a long value.
 7748 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7749 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7750 %{
 7751   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7752   effect(KILL oldval);
 7753 
 7754   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7755   ins_encode %{
 7756     __ lock();
 7757     __ cmpxchgq($newval$$Register, $mem$$Address);
 7758   %}
 7759   ins_pipe(pipe_cmpxchg);
 7760 %}
 7761 
 7762 
 7763 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7764 instruct compareAndSwapP(rRegI res,
 7765                          memory mem_ptr,
 7766                          rax_RegP oldval, rRegP newval,
 7767                          rFlagsReg cr)
 7768 %{
 7769   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7770   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7771   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7772   effect(KILL cr, KILL oldval);
 7773 
 7774   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7775             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7776             "sete    $res\n\t"
 7777             "movzbl  $res, $res" %}
 7778   ins_encode %{
 7779     __ lock();
 7780     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7781     __ sete($res$$Register);
 7782     __ movzbl($res$$Register, $res$$Register);
 7783   %}
 7784   ins_pipe( pipe_cmpxchg );
 7785 %}
 7786 
 7787 instruct compareAndSwapL(rRegI res,
 7788                          memory mem_ptr,
 7789                          rax_RegL oldval, rRegL newval,
 7790                          rFlagsReg cr)
 7791 %{
 7792   predicate(VM_Version::supports_cx8());
 7793   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7794   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7795   effect(KILL cr, KILL oldval);
 7796 
 7797   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7798             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7799             "sete    $res\n\t"
 7800             "movzbl  $res, $res" %}
 7801   ins_encode %{
 7802     __ lock();
 7803     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7804     __ sete($res$$Register);
 7805     __ movzbl($res$$Register, $res$$Register);
 7806   %}
 7807   ins_pipe( pipe_cmpxchg );
 7808 %}
 7809 
 7810 instruct compareAndSwapI(rRegI res,
 7811                          memory mem_ptr,
 7812                          rax_RegI oldval, rRegI newval,
 7813                          rFlagsReg cr)
 7814 %{
 7815   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7816   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7817   effect(KILL cr, KILL oldval);
 7818 
 7819   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7820             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7821             "sete    $res\n\t"
 7822             "movzbl  $res, $res" %}
 7823   ins_encode %{
 7824     __ lock();
 7825     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7826     __ sete($res$$Register);
 7827     __ movzbl($res$$Register, $res$$Register);
 7828   %}
 7829   ins_pipe( pipe_cmpxchg );
 7830 %}
 7831 
 7832 instruct compareAndSwapB(rRegI res,
 7833                          memory mem_ptr,
 7834                          rax_RegI oldval, rRegI newval,
 7835                          rFlagsReg cr)
 7836 %{
 7837   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7838   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7839   effect(KILL cr, KILL oldval);
 7840 
 7841   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7842             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7843             "sete    $res\n\t"
 7844             "movzbl  $res, $res" %}
 7845   ins_encode %{
 7846     __ lock();
 7847     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7848     __ sete($res$$Register);
 7849     __ movzbl($res$$Register, $res$$Register);
 7850   %}
 7851   ins_pipe( pipe_cmpxchg );
 7852 %}
 7853 
 7854 instruct compareAndSwapS(rRegI res,
 7855                          memory mem_ptr,
 7856                          rax_RegI oldval, rRegI newval,
 7857                          rFlagsReg cr)
 7858 %{
 7859   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7860   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7861   effect(KILL cr, KILL oldval);
 7862 
 7863   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7864             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7865             "sete    $res\n\t"
 7866             "movzbl  $res, $res" %}
 7867   ins_encode %{
 7868     __ lock();
 7869     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7870     __ sete($res$$Register);
 7871     __ movzbl($res$$Register, $res$$Register);
 7872   %}
 7873   ins_pipe( pipe_cmpxchg );
 7874 %}
 7875 
 7876 instruct compareAndSwapN(rRegI res,
 7877                           memory mem_ptr,
 7878                           rax_RegN oldval, rRegN newval,
 7879                           rFlagsReg cr) %{
 7880   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7881   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7882   effect(KILL cr, KILL oldval);
 7883 
 7884   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7885             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7886             "sete    $res\n\t"
 7887             "movzbl  $res, $res" %}
 7888   ins_encode %{
 7889     __ lock();
 7890     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7891     __ sete($res$$Register);
 7892     __ movzbl($res$$Register, $res$$Register);
 7893   %}
 7894   ins_pipe( pipe_cmpxchg );
 7895 %}
 7896 
 7897 instruct compareAndExchangeB(
 7898                          memory mem_ptr,
 7899                          rax_RegI oldval, rRegI newval,
 7900                          rFlagsReg cr)
 7901 %{
 7902   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7903   effect(KILL cr);
 7904 
 7905   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7906             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7907   ins_encode %{
 7908     __ lock();
 7909     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7910   %}
 7911   ins_pipe( pipe_cmpxchg );
 7912 %}
 7913 
 7914 instruct compareAndExchangeS(
 7915                          memory mem_ptr,
 7916                          rax_RegI oldval, rRegI newval,
 7917                          rFlagsReg cr)
 7918 %{
 7919   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7920   effect(KILL cr);
 7921 
 7922   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7923             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7924   ins_encode %{
 7925     __ lock();
 7926     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7927   %}
 7928   ins_pipe( pipe_cmpxchg );
 7929 %}
 7930 
 7931 instruct compareAndExchangeI(
 7932                          memory mem_ptr,
 7933                          rax_RegI oldval, rRegI newval,
 7934                          rFlagsReg cr)
 7935 %{
 7936   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7937   effect(KILL cr);
 7938 
 7939   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7940             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7941   ins_encode %{
 7942     __ lock();
 7943     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7944   %}
 7945   ins_pipe( pipe_cmpxchg );
 7946 %}
 7947 
 7948 instruct compareAndExchangeL(
 7949                          memory mem_ptr,
 7950                          rax_RegL oldval, rRegL newval,
 7951                          rFlagsReg cr)
 7952 %{
 7953   predicate(VM_Version::supports_cx8());
 7954   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7955   effect(KILL cr);
 7956 
 7957   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7958             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7959   ins_encode %{
 7960     __ lock();
 7961     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7962   %}
 7963   ins_pipe( pipe_cmpxchg );
 7964 %}
 7965 
 7966 instruct compareAndExchangeN(
 7967                           memory mem_ptr,
 7968                           rax_RegN oldval, rRegN newval,
 7969                           rFlagsReg cr) %{
 7970   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7971   effect(KILL cr);
 7972 
 7973   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7974             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7975   ins_encode %{
 7976     __ lock();
 7977     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7978   %}
 7979   ins_pipe( pipe_cmpxchg );
 7980 %}
 7981 
 7982 instruct compareAndExchangeP(
 7983                          memory mem_ptr,
 7984                          rax_RegP oldval, rRegP newval,
 7985                          rFlagsReg cr)
 7986 %{
 7987   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7988   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7989   effect(KILL cr);
 7990 
 7991   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7992             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7993   ins_encode %{
 7994     __ lock();
 7995     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7996   %}
 7997   ins_pipe( pipe_cmpxchg );
 7998 %}
 7999 
 8000 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8001   predicate(n->as_LoadStore()->result_not_used());
 8002   match(Set dummy (GetAndAddB mem add));
 8003   effect(KILL cr);
 8004   format %{ "ADDB  [$mem],$add" %}
 8005   ins_encode %{
 8006     __ lock();
 8007     __ addb($mem$$Address, $add$$constant);
 8008   %}
 8009   ins_pipe( pipe_cmpxchg );
 8010 %}
 8011 
 8012 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8013   match(Set newval (GetAndAddB mem newval));
 8014   effect(KILL cr);
 8015   format %{ "XADDB  [$mem],$newval" %}
 8016   ins_encode %{
 8017     __ lock();
 8018     __ xaddb($mem$$Address, $newval$$Register);
 8019   %}
 8020   ins_pipe( pipe_cmpxchg );
 8021 %}
 8022 
 8023 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8024   predicate(n->as_LoadStore()->result_not_used());
 8025   match(Set dummy (GetAndAddS mem add));
 8026   effect(KILL cr);
 8027   format %{ "ADDW  [$mem],$add" %}
 8028   ins_encode %{
 8029     __ lock();
 8030     __ addw($mem$$Address, $add$$constant);
 8031   %}
 8032   ins_pipe( pipe_cmpxchg );
 8033 %}
 8034 
 8035 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8036   match(Set newval (GetAndAddS mem newval));
 8037   effect(KILL cr);
 8038   format %{ "XADDW  [$mem],$newval" %}
 8039   ins_encode %{
 8040     __ lock();
 8041     __ xaddw($mem$$Address, $newval$$Register);
 8042   %}
 8043   ins_pipe( pipe_cmpxchg );
 8044 %}
 8045 
 8046 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8047   predicate(n->as_LoadStore()->result_not_used());
 8048   match(Set dummy (GetAndAddI mem add));
 8049   effect(KILL cr);
 8050   format %{ "ADDL  [$mem],$add" %}
 8051   ins_encode %{
 8052     __ lock();
 8053     __ addl($mem$$Address, $add$$constant);
 8054   %}
 8055   ins_pipe( pipe_cmpxchg );
 8056 %}
 8057 
 8058 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8059   match(Set newval (GetAndAddI mem newval));
 8060   effect(KILL cr);
 8061   format %{ "XADDL  [$mem],$newval" %}
 8062   ins_encode %{
 8063     __ lock();
 8064     __ xaddl($mem$$Address, $newval$$Register);
 8065   %}
 8066   ins_pipe( pipe_cmpxchg );
 8067 %}
 8068 
 8069 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8070   predicate(n->as_LoadStore()->result_not_used());
 8071   match(Set dummy (GetAndAddL mem add));
 8072   effect(KILL cr);
 8073   format %{ "ADDQ  [$mem],$add" %}
 8074   ins_encode %{
 8075     __ lock();
 8076     __ addq($mem$$Address, $add$$constant);
 8077   %}
 8078   ins_pipe( pipe_cmpxchg );
 8079 %}
 8080 
 8081 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8082   match(Set newval (GetAndAddL mem newval));
 8083   effect(KILL cr);
 8084   format %{ "XADDQ  [$mem],$newval" %}
 8085   ins_encode %{
 8086     __ lock();
 8087     __ xaddq($mem$$Address, $newval$$Register);
 8088   %}
 8089   ins_pipe( pipe_cmpxchg );
 8090 %}
 8091 
 8092 instruct xchgB( memory mem, rRegI newval) %{
 8093   match(Set newval (GetAndSetB mem newval));
 8094   format %{ "XCHGB  $newval,[$mem]" %}
 8095   ins_encode %{
 8096     __ xchgb($newval$$Register, $mem$$Address);
 8097   %}
 8098   ins_pipe( pipe_cmpxchg );
 8099 %}
 8100 
 8101 instruct xchgS( memory mem, rRegI newval) %{
 8102   match(Set newval (GetAndSetS mem newval));
 8103   format %{ "XCHGW  $newval,[$mem]" %}
 8104   ins_encode %{
 8105     __ xchgw($newval$$Register, $mem$$Address);
 8106   %}
 8107   ins_pipe( pipe_cmpxchg );
 8108 %}
 8109 
 8110 instruct xchgI( memory mem, rRegI newval) %{
 8111   match(Set newval (GetAndSetI mem newval));
 8112   format %{ "XCHGL  $newval,[$mem]" %}
 8113   ins_encode %{
 8114     __ xchgl($newval$$Register, $mem$$Address);
 8115   %}
 8116   ins_pipe( pipe_cmpxchg );
 8117 %}
 8118 
 8119 instruct xchgL( memory mem, rRegL newval) %{
 8120   match(Set newval (GetAndSetL mem newval));
 8121   format %{ "XCHGL  $newval,[$mem]" %}
 8122   ins_encode %{
 8123     __ xchgq($newval$$Register, $mem$$Address);
 8124   %}
 8125   ins_pipe( pipe_cmpxchg );
 8126 %}
 8127 
 8128 instruct xchgP( memory mem, rRegP newval) %{
 8129   match(Set newval (GetAndSetP mem newval));
 8130   predicate(n->as_LoadStore()->barrier_data() == 0);
 8131   format %{ "XCHGQ  $newval,[$mem]" %}
 8132   ins_encode %{
 8133     __ xchgq($newval$$Register, $mem$$Address);
 8134   %}
 8135   ins_pipe( pipe_cmpxchg );
 8136 %}
 8137 
 8138 instruct xchgN( memory mem, rRegN newval) %{
 8139   match(Set newval (GetAndSetN mem newval));
 8140   format %{ "XCHGL  $newval,$mem]" %}
 8141   ins_encode %{
 8142     __ xchgl($newval$$Register, $mem$$Address);
 8143   %}
 8144   ins_pipe( pipe_cmpxchg );
 8145 %}
 8146 
 8147 //----------Abs Instructions-------------------------------------------
 8148 
 8149 // Integer Absolute Instructions
 8150 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8151 %{
 8152   match(Set dst (AbsI src));
 8153   effect(TEMP dst, TEMP tmp, KILL cr);
 8154   format %{ "movl $tmp, $src\n\t"
 8155             "sarl $tmp, 31\n\t"
 8156             "movl $dst, $src\n\t"
 8157             "xorl $dst, $tmp\n\t"
 8158             "subl $dst, $tmp\n"
 8159           %}
 8160   ins_encode %{
 8161     __ movl($tmp$$Register, $src$$Register);
 8162     __ sarl($tmp$$Register, 31);
 8163     __ movl($dst$$Register, $src$$Register);
 8164     __ xorl($dst$$Register, $tmp$$Register);
 8165     __ subl($dst$$Register, $tmp$$Register);
 8166   %}
 8167 
 8168   ins_pipe(ialu_reg_reg);
 8169 %}
 8170 
 8171 // Long Absolute Instructions
 8172 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8173 %{
 8174   match(Set dst (AbsL src));
 8175   effect(TEMP dst, TEMP tmp, KILL cr);
 8176   format %{ "movq $tmp, $src\n\t"
 8177             "sarq $tmp, 63\n\t"
 8178             "movq $dst, $src\n\t"
 8179             "xorq $dst, $tmp\n\t"
 8180             "subq $dst, $tmp\n"
 8181           %}
 8182   ins_encode %{
 8183     __ movq($tmp$$Register, $src$$Register);
 8184     __ sarq($tmp$$Register, 63);
 8185     __ movq($dst$$Register, $src$$Register);
 8186     __ xorq($dst$$Register, $tmp$$Register);
 8187     __ subq($dst$$Register, $tmp$$Register);
 8188   %}
 8189 
 8190   ins_pipe(ialu_reg_reg);
 8191 %}
 8192 
 8193 //----------Subtraction Instructions-------------------------------------------
 8194 
 8195 // Integer Subtraction Instructions
 8196 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8197 %{
 8198   match(Set dst (SubI dst src));
 8199   effect(KILL cr);
 8200 
 8201   format %{ "subl    $dst, $src\t# int" %}
 8202   ins_encode %{
 8203     __ subl($dst$$Register, $src$$Register);
 8204   %}
 8205   ins_pipe(ialu_reg_reg);
 8206 %}
 8207 
 8208 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8209 %{
 8210   match(Set dst (SubI dst src));
 8211   effect(KILL cr);
 8212 
 8213   format %{ "subl    $dst, $src\t# int" %}
 8214   ins_encode %{
 8215     __ subl($dst$$Register, $src$$constant);
 8216   %}
 8217   ins_pipe(ialu_reg);
 8218 %}
 8219 
 8220 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8221 %{
 8222   match(Set dst (SubI dst (LoadI src)));
 8223   effect(KILL cr);
 8224 
 8225   ins_cost(125);
 8226   format %{ "subl    $dst, $src\t# int" %}
 8227   ins_encode %{
 8228     __ subl($dst$$Register, $src$$Address);
 8229   %}
 8230   ins_pipe(ialu_reg_mem);
 8231 %}
 8232 
 8233 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8234 %{
 8235   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8236   effect(KILL cr);
 8237 
 8238   ins_cost(150);
 8239   format %{ "subl    $dst, $src\t# int" %}
 8240   ins_encode %{
 8241     __ subl($dst$$Address, $src$$Register);
 8242   %}
 8243   ins_pipe(ialu_mem_reg);
 8244 %}
 8245 
 8246 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8247 %{
 8248   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8249   effect(KILL cr);
 8250 
 8251   ins_cost(125); // XXX
 8252   format %{ "subl    $dst, $src\t# int" %}
 8253   ins_encode %{
 8254     __ subl($dst$$Address, $src$$constant);
 8255   %}
 8256   ins_pipe(ialu_mem_imm);
 8257 %}
 8258 
 8259 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8260 %{
 8261   match(Set dst (SubL dst src));
 8262   effect(KILL cr);
 8263 
 8264   format %{ "subq    $dst, $src\t# long" %}
 8265   ins_encode %{
 8266     __ subq($dst$$Register, $src$$Register);
 8267   %}
 8268   ins_pipe(ialu_reg_reg);
 8269 %}
 8270 
 8271 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8272 %{
 8273   match(Set dst (SubL dst src));
 8274   effect(KILL cr);
 8275 
 8276   format %{ "subq    $dst, $src\t# long" %}
 8277   ins_encode %{
 8278     __ subq($dst$$Register, $src$$constant);
 8279   %}
 8280   ins_pipe(ialu_reg);
 8281 %}
 8282 
 8283 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8284 %{
 8285   match(Set dst (SubL dst (LoadL src)));
 8286   effect(KILL cr);
 8287 
 8288   ins_cost(125);
 8289   format %{ "subq    $dst, $src\t# long" %}
 8290   ins_encode %{
 8291     __ subq($dst$$Register, $src$$Address);
 8292   %}
 8293   ins_pipe(ialu_reg_mem);
 8294 %}
 8295 
 8296 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8297 %{
 8298   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8299   effect(KILL cr);
 8300 
 8301   ins_cost(150);
 8302   format %{ "subq    $dst, $src\t# long" %}
 8303   ins_encode %{
 8304     __ subq($dst$$Address, $src$$Register);
 8305   %}
 8306   ins_pipe(ialu_mem_reg);
 8307 %}
 8308 
 8309 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8310 %{
 8311   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8312   effect(KILL cr);
 8313 
 8314   ins_cost(125); // XXX
 8315   format %{ "subq    $dst, $src\t# long" %}
 8316   ins_encode %{
 8317     __ subq($dst$$Address, $src$$constant);
 8318   %}
 8319   ins_pipe(ialu_mem_imm);
 8320 %}
 8321 
 8322 // Subtract from a pointer
 8323 // XXX hmpf???
 8324 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8325 %{
 8326   match(Set dst (AddP dst (SubI zero src)));
 8327   effect(KILL cr);
 8328 
 8329   format %{ "subq    $dst, $src\t# ptr - int" %}
 8330   opcode(0x2B);
 8331   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8332   ins_pipe(ialu_reg_reg);
 8333 %}
 8334 
 8335 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8336 %{
 8337   match(Set dst (SubI zero dst));
 8338   effect(KILL cr);
 8339 
 8340   format %{ "negl    $dst\t# int" %}
 8341   ins_encode %{
 8342     __ negl($dst$$Register);
 8343   %}
 8344   ins_pipe(ialu_reg);
 8345 %}
 8346 
 8347 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8348 %{
 8349   match(Set dst (NegI dst));
 8350   effect(KILL cr);
 8351 
 8352   format %{ "negl    $dst\t# int" %}
 8353   ins_encode %{
 8354     __ negl($dst$$Register);
 8355   %}
 8356   ins_pipe(ialu_reg);
 8357 %}
 8358 
 8359 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8360 %{
 8361   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8362   effect(KILL cr);
 8363 
 8364   format %{ "negl    $dst\t# int" %}
 8365   ins_encode %{
 8366     __ negl($dst$$Address);
 8367   %}
 8368   ins_pipe(ialu_reg);
 8369 %}
 8370 
 8371 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8372 %{
 8373   match(Set dst (SubL zero dst));
 8374   effect(KILL cr);
 8375 
 8376   format %{ "negq    $dst\t# long" %}
 8377   ins_encode %{
 8378     __ negq($dst$$Register);
 8379   %}
 8380   ins_pipe(ialu_reg);
 8381 %}
 8382 
 8383 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8384 %{
 8385   match(Set dst (NegL dst));
 8386   effect(KILL cr);
 8387 
 8388   format %{ "negq    $dst\t# int" %}
 8389   ins_encode %{
 8390     __ negq($dst$$Register);
 8391   %}
 8392   ins_pipe(ialu_reg);
 8393 %}
 8394 
 8395 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8396 %{
 8397   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8398   effect(KILL cr);
 8399 
 8400   format %{ "negq    $dst\t# long" %}
 8401   ins_encode %{
 8402     __ negq($dst$$Address);
 8403   %}
 8404   ins_pipe(ialu_reg);
 8405 %}
 8406 
 8407 //----------Multiplication/Division Instructions-------------------------------
 8408 // Integer Multiplication Instructions
 8409 // Multiply Register
 8410 
 8411 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8412 %{
 8413   match(Set dst (MulI dst src));
 8414   effect(KILL cr);
 8415 
 8416   ins_cost(300);
 8417   format %{ "imull   $dst, $src\t# int" %}
 8418   ins_encode %{
 8419     __ imull($dst$$Register, $src$$Register);
 8420   %}
 8421   ins_pipe(ialu_reg_reg_alu0);
 8422 %}
 8423 
 8424 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8425 %{
 8426   match(Set dst (MulI src imm));
 8427   effect(KILL cr);
 8428 
 8429   ins_cost(300);
 8430   format %{ "imull   $dst, $src, $imm\t# int" %}
 8431   ins_encode %{
 8432     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8433   %}
 8434   ins_pipe(ialu_reg_reg_alu0);
 8435 %}
 8436 
 8437 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8438 %{
 8439   match(Set dst (MulI dst (LoadI src)));
 8440   effect(KILL cr);
 8441 
 8442   ins_cost(350);
 8443   format %{ "imull   $dst, $src\t# int" %}
 8444   ins_encode %{
 8445     __ imull($dst$$Register, $src$$Address);
 8446   %}
 8447   ins_pipe(ialu_reg_mem_alu0);
 8448 %}
 8449 
 8450 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8451 %{
 8452   match(Set dst (MulI (LoadI src) imm));
 8453   effect(KILL cr);
 8454 
 8455   ins_cost(300);
 8456   format %{ "imull   $dst, $src, $imm\t# int" %}
 8457   ins_encode %{
 8458     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8459   %}
 8460   ins_pipe(ialu_reg_mem_alu0);
 8461 %}
 8462 
 8463 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8464 %{
 8465   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8466   effect(KILL cr, KILL src2);
 8467 
 8468   expand %{ mulI_rReg(dst, src1, cr);
 8469            mulI_rReg(src2, src3, cr);
 8470            addI_rReg(dst, src2, cr); %}
 8471 %}
 8472 
 8473 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8474 %{
 8475   match(Set dst (MulL dst src));
 8476   effect(KILL cr);
 8477 
 8478   ins_cost(300);
 8479   format %{ "imulq   $dst, $src\t# long" %}
 8480   ins_encode %{
 8481     __ imulq($dst$$Register, $src$$Register);
 8482   %}
 8483   ins_pipe(ialu_reg_reg_alu0);
 8484 %}
 8485 
 8486 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8487 %{
 8488   match(Set dst (MulL src imm));
 8489   effect(KILL cr);
 8490 
 8491   ins_cost(300);
 8492   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8493   ins_encode %{
 8494     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8495   %}
 8496   ins_pipe(ialu_reg_reg_alu0);
 8497 %}
 8498 
 8499 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8500 %{
 8501   match(Set dst (MulL dst (LoadL src)));
 8502   effect(KILL cr);
 8503 
 8504   ins_cost(350);
 8505   format %{ "imulq   $dst, $src\t# long" %}
 8506   ins_encode %{
 8507     __ imulq($dst$$Register, $src$$Address);
 8508   %}
 8509   ins_pipe(ialu_reg_mem_alu0);
 8510 %}
 8511 
 8512 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8513 %{
 8514   match(Set dst (MulL (LoadL src) imm));
 8515   effect(KILL cr);
 8516 
 8517   ins_cost(300);
 8518   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8519   ins_encode %{
 8520     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8521   %}
 8522   ins_pipe(ialu_reg_mem_alu0);
 8523 %}
 8524 
 8525 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8526 %{
 8527   match(Set dst (MulHiL src rax));
 8528   effect(USE_KILL rax, KILL cr);
 8529 
 8530   ins_cost(300);
 8531   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8532   ins_encode %{
 8533     __ imulq($src$$Register);
 8534   %}
 8535   ins_pipe(ialu_reg_reg_alu0);
 8536 %}
 8537 
 8538 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8539 %{
 8540   match(Set dst (UMulHiL src rax));
 8541   effect(USE_KILL rax, KILL cr);
 8542 
 8543   ins_cost(300);
 8544   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8545   ins_encode %{
 8546     __ mulq($src$$Register);
 8547   %}
 8548   ins_pipe(ialu_reg_reg_alu0);
 8549 %}
 8550 
 8551 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8552                    rFlagsReg cr)
 8553 %{
 8554   match(Set rax (DivI rax div));
 8555   effect(KILL rdx, KILL cr);
 8556 
 8557   ins_cost(30*100+10*100); // XXX
 8558   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8559             "jne,s   normal\n\t"
 8560             "xorl    rdx, rdx\n\t"
 8561             "cmpl    $div, -1\n\t"
 8562             "je,s    done\n"
 8563     "normal: cdql\n\t"
 8564             "idivl   $div\n"
 8565     "done:"        %}
 8566   ins_encode(cdql_enc(div));
 8567   ins_pipe(ialu_reg_reg_alu0);
 8568 %}
 8569 
 8570 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8571                    rFlagsReg cr)
 8572 %{
 8573   match(Set rax (DivL rax div));
 8574   effect(KILL rdx, KILL cr);
 8575 
 8576   ins_cost(30*100+10*100); // XXX
 8577   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8578             "cmpq    rax, rdx\n\t"
 8579             "jne,s   normal\n\t"
 8580             "xorl    rdx, rdx\n\t"
 8581             "cmpq    $div, -1\n\t"
 8582             "je,s    done\n"
 8583     "normal: cdqq\n\t"
 8584             "idivq   $div\n"
 8585     "done:"        %}
 8586   ins_encode(cdqq_enc(div));
 8587   ins_pipe(ialu_reg_reg_alu0);
 8588 %}
 8589 
 8590 // Integer DIVMOD with Register, both quotient and mod results
 8591 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8592                              rFlagsReg cr)
 8593 %{
 8594   match(DivModI rax div);
 8595   effect(KILL cr);
 8596 
 8597   ins_cost(30*100+10*100); // XXX
 8598   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8599             "jne,s   normal\n\t"
 8600             "xorl    rdx, rdx\n\t"
 8601             "cmpl    $div, -1\n\t"
 8602             "je,s    done\n"
 8603     "normal: cdql\n\t"
 8604             "idivl   $div\n"
 8605     "done:"        %}
 8606   ins_encode(cdql_enc(div));
 8607   ins_pipe(pipe_slow);
 8608 %}
 8609 
 8610 // Long DIVMOD with Register, both quotient and mod results
 8611 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8612                              rFlagsReg cr)
 8613 %{
 8614   match(DivModL rax div);
 8615   effect(KILL cr);
 8616 
 8617   ins_cost(30*100+10*100); // XXX
 8618   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8619             "cmpq    rax, rdx\n\t"
 8620             "jne,s   normal\n\t"
 8621             "xorl    rdx, rdx\n\t"
 8622             "cmpq    $div, -1\n\t"
 8623             "je,s    done\n"
 8624     "normal: cdqq\n\t"
 8625             "idivq   $div\n"
 8626     "done:"        %}
 8627   ins_encode(cdqq_enc(div));
 8628   ins_pipe(pipe_slow);
 8629 %}
 8630 
 8631 //----------- DivL-By-Constant-Expansions--------------------------------------
 8632 // DivI cases are handled by the compiler
 8633 
 8634 // Magic constant, reciprocal of 10
 8635 instruct loadConL_0x6666666666666667(rRegL dst)
 8636 %{
 8637   effect(DEF dst);
 8638 
 8639   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 8640   ins_encode(load_immL(dst, 0x6666666666666667));
 8641   ins_pipe(ialu_reg);
 8642 %}
 8643 
 8644 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8645 %{
 8646   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 8647 
 8648   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 8649   ins_encode %{
 8650     __ imulq($src$$Register);
 8651   %}
 8652   ins_pipe(ialu_reg_reg_alu0);
 8653 %}
 8654 
 8655 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 8656 %{
 8657   effect(USE_DEF dst, KILL cr);
 8658 
 8659   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 8660   ins_encode %{
 8661     __ sarq($dst$$Register, 63);
 8662   %}
 8663   ins_pipe(ialu_reg);
 8664 %}
 8665 
 8666 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 8667 %{
 8668   effect(USE_DEF dst, KILL cr);
 8669 
 8670   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 8671   ins_encode %{
 8672     __ sarq($dst$$Register, 2);
 8673   %}
 8674   ins_pipe(ialu_reg);
 8675 %}
 8676 
 8677 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 8678 %{
 8679   match(Set dst (DivL src div));
 8680 
 8681   ins_cost((5+8)*100);
 8682   expand %{
 8683     rax_RegL rax;                     // Killed temp
 8684     rFlagsReg cr;                     // Killed
 8685     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 8686     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 8687     sarL_rReg_63(src, cr);            // sarq  src, 63
 8688     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 8689     subL_rReg(dst, src, cr);          // subl  rdx, src
 8690   %}
 8691 %}
 8692 
 8693 //-----------------------------------------------------------------------------
 8694 
 8695 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8696                    rFlagsReg cr)
 8697 %{
 8698   match(Set rdx (ModI rax div));
 8699   effect(KILL rax, KILL cr);
 8700 
 8701   ins_cost(300); // XXX
 8702   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8703             "jne,s   normal\n\t"
 8704             "xorl    rdx, rdx\n\t"
 8705             "cmpl    $div, -1\n\t"
 8706             "je,s    done\n"
 8707     "normal: cdql\n\t"
 8708             "idivl   $div\n"
 8709     "done:"        %}
 8710   ins_encode(cdql_enc(div));
 8711   ins_pipe(ialu_reg_reg_alu0);
 8712 %}
 8713 
 8714 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8715                    rFlagsReg cr)
 8716 %{
 8717   match(Set rdx (ModL rax div));
 8718   effect(KILL rax, KILL cr);
 8719 
 8720   ins_cost(300); // XXX
 8721   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8722             "cmpq    rax, rdx\n\t"
 8723             "jne,s   normal\n\t"
 8724             "xorl    rdx, rdx\n\t"
 8725             "cmpq    $div, -1\n\t"
 8726             "je,s    done\n"
 8727     "normal: cdqq\n\t"
 8728             "idivq   $div\n"
 8729     "done:"        %}
 8730   ins_encode(cdqq_enc(div));
 8731   ins_pipe(ialu_reg_reg_alu0);
 8732 %}
 8733 
 8734 // Integer Shift Instructions
 8735 // Shift Left by one
 8736 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8737 %{
 8738   match(Set dst (LShiftI dst shift));
 8739   effect(KILL cr);
 8740 
 8741   format %{ "sall    $dst, $shift" %}
 8742   ins_encode %{
 8743     __ sall($dst$$Register, $shift$$constant);
 8744   %}
 8745   ins_pipe(ialu_reg);
 8746 %}
 8747 
 8748 // Shift Left by one
 8749 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8750 %{
 8751   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8752   effect(KILL cr);
 8753 
 8754   format %{ "sall    $dst, $shift\t" %}
 8755   ins_encode %{
 8756     __ sall($dst$$Address, $shift$$constant);
 8757   %}
 8758   ins_pipe(ialu_mem_imm);
 8759 %}
 8760 
 8761 // Shift Left by 8-bit immediate
 8762 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8763 %{
 8764   match(Set dst (LShiftI dst shift));
 8765   effect(KILL cr);
 8766 
 8767   format %{ "sall    $dst, $shift" %}
 8768   ins_encode %{
 8769     __ sall($dst$$Register, $shift$$constant);
 8770   %}
 8771   ins_pipe(ialu_reg);
 8772 %}
 8773 
 8774 // Shift Left by 8-bit immediate
 8775 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8776 %{
 8777   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8778   effect(KILL cr);
 8779 
 8780   format %{ "sall    $dst, $shift" %}
 8781   ins_encode %{
 8782     __ sall($dst$$Address, $shift$$constant);
 8783   %}
 8784   ins_pipe(ialu_mem_imm);
 8785 %}
 8786 
 8787 // Shift Left by variable
 8788 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8789 %{
 8790   match(Set dst (LShiftI dst shift));
 8791   effect(KILL cr);
 8792 
 8793   format %{ "sall    $dst, $shift" %}
 8794   ins_encode %{
 8795     __ sall($dst$$Register);
 8796   %}
 8797   ins_pipe(ialu_reg_reg);
 8798 %}
 8799 
 8800 // Shift Left by variable
 8801 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8802 %{
 8803   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8804   effect(KILL cr);
 8805 
 8806   format %{ "sall    $dst, $shift" %}
 8807   ins_encode %{
 8808     __ sall($dst$$Address);
 8809   %}
 8810   ins_pipe(ialu_mem_reg);
 8811 %}
 8812 
 8813 // Arithmetic shift right by one
 8814 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8815 %{
 8816   match(Set dst (RShiftI dst shift));
 8817   effect(KILL cr);
 8818 
 8819   format %{ "sarl    $dst, $shift" %}
 8820   ins_encode %{
 8821     __ sarl($dst$$Register, $shift$$constant);
 8822   %}
 8823   ins_pipe(ialu_reg);
 8824 %}
 8825 
 8826 // Arithmetic shift right by one
 8827 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8828 %{
 8829   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8830   effect(KILL cr);
 8831 
 8832   format %{ "sarl    $dst, $shift" %}
 8833   ins_encode %{
 8834     __ sarl($dst$$Address, $shift$$constant);
 8835   %}
 8836   ins_pipe(ialu_mem_imm);
 8837 %}
 8838 
 8839 // Arithmetic Shift Right by 8-bit immediate
 8840 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8841 %{
 8842   match(Set dst (RShiftI dst shift));
 8843   effect(KILL cr);
 8844 
 8845   format %{ "sarl    $dst, $shift" %}
 8846   ins_encode %{
 8847     __ sarl($dst$$Register, $shift$$constant);
 8848   %}
 8849   ins_pipe(ialu_mem_imm);
 8850 %}
 8851 
 8852 // Arithmetic Shift Right by 8-bit immediate
 8853 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8854 %{
 8855   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8856   effect(KILL cr);
 8857 
 8858   format %{ "sarl    $dst, $shift" %}
 8859   ins_encode %{
 8860     __ sarl($dst$$Address, $shift$$constant);
 8861   %}
 8862   ins_pipe(ialu_mem_imm);
 8863 %}
 8864 
 8865 // Arithmetic Shift Right by variable
 8866 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8867 %{
 8868   match(Set dst (RShiftI dst shift));
 8869   effect(KILL cr);
 8870   format %{ "sarl    $dst, $shift" %}
 8871   ins_encode %{
 8872     __ sarl($dst$$Register);
 8873   %}
 8874   ins_pipe(ialu_reg_reg);
 8875 %}
 8876 
 8877 // Arithmetic Shift Right by variable
 8878 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8879 %{
 8880   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8881   effect(KILL cr);
 8882 
 8883   format %{ "sarl    $dst, $shift" %}
 8884   ins_encode %{
 8885     __ sarl($dst$$Address);
 8886   %}
 8887   ins_pipe(ialu_mem_reg);
 8888 %}
 8889 
 8890 // Logical shift right by one
 8891 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8892 %{
 8893   match(Set dst (URShiftI dst shift));
 8894   effect(KILL cr);
 8895 
 8896   format %{ "shrl    $dst, $shift" %}
 8897   ins_encode %{
 8898     __ shrl($dst$$Register, $shift$$constant);
 8899   %}
 8900   ins_pipe(ialu_reg);
 8901 %}
 8902 
 8903 // Logical shift right by one
 8904 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8905 %{
 8906   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8907   effect(KILL cr);
 8908 
 8909   format %{ "shrl    $dst, $shift" %}
 8910   ins_encode %{
 8911     __ shrl($dst$$Address, $shift$$constant);
 8912   %}
 8913   ins_pipe(ialu_mem_imm);
 8914 %}
 8915 
 8916 // Logical Shift Right by 8-bit immediate
 8917 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8918 %{
 8919   match(Set dst (URShiftI dst shift));
 8920   effect(KILL cr);
 8921 
 8922   format %{ "shrl    $dst, $shift" %}
 8923   ins_encode %{
 8924     __ shrl($dst$$Register, $shift$$constant);
 8925   %}
 8926   ins_pipe(ialu_reg);
 8927 %}
 8928 
 8929 // Logical Shift Right by 8-bit immediate
 8930 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8931 %{
 8932   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8933   effect(KILL cr);
 8934 
 8935   format %{ "shrl    $dst, $shift" %}
 8936   ins_encode %{
 8937     __ shrl($dst$$Address, $shift$$constant);
 8938   %}
 8939   ins_pipe(ialu_mem_imm);
 8940 %}
 8941 
 8942 // Logical Shift Right by variable
 8943 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8944 %{
 8945   match(Set dst (URShiftI dst shift));
 8946   effect(KILL cr);
 8947 
 8948   format %{ "shrl    $dst, $shift" %}
 8949   ins_encode %{
 8950     __ shrl($dst$$Register);
 8951   %}
 8952   ins_pipe(ialu_reg_reg);
 8953 %}
 8954 
 8955 // Logical Shift Right by variable
 8956 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8957 %{
 8958   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8959   effect(KILL cr);
 8960 
 8961   format %{ "shrl    $dst, $shift" %}
 8962   ins_encode %{
 8963     __ shrl($dst$$Address);
 8964   %}
 8965   ins_pipe(ialu_mem_reg);
 8966 %}
 8967 
 8968 // Long Shift Instructions
 8969 // Shift Left by one
 8970 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 8971 %{
 8972   match(Set dst (LShiftL dst shift));
 8973   effect(KILL cr);
 8974 
 8975   format %{ "salq    $dst, $shift" %}
 8976   ins_encode %{
 8977     __ salq($dst$$Register, $shift$$constant);
 8978   %}
 8979   ins_pipe(ialu_reg);
 8980 %}
 8981 
 8982 // Shift Left by one
 8983 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8984 %{
 8985   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8986   effect(KILL cr);
 8987 
 8988   format %{ "salq    $dst, $shift" %}
 8989   ins_encode %{
 8990     __ salq($dst$$Address, $shift$$constant);
 8991   %}
 8992   ins_pipe(ialu_mem_imm);
 8993 %}
 8994 
 8995 // Shift Left by 8-bit immediate
 8996 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8997 %{
 8998   match(Set dst (LShiftL dst shift));
 8999   effect(KILL cr);
 9000 
 9001   format %{ "salq    $dst, $shift" %}
 9002   ins_encode %{
 9003     __ salq($dst$$Register, $shift$$constant);
 9004   %}
 9005   ins_pipe(ialu_reg);
 9006 %}
 9007 
 9008 // Shift Left by 8-bit immediate
 9009 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9010 %{
 9011   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9012   effect(KILL cr);
 9013 
 9014   format %{ "salq    $dst, $shift" %}
 9015   ins_encode %{
 9016     __ salq($dst$$Address, $shift$$constant);
 9017   %}
 9018   ins_pipe(ialu_mem_imm);
 9019 %}
 9020 
 9021 // Shift Left by variable
 9022 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9023 %{
 9024   match(Set dst (LShiftL dst shift));
 9025   effect(KILL cr);
 9026 
 9027   format %{ "salq    $dst, $shift" %}
 9028   ins_encode %{
 9029     __ salq($dst$$Register);
 9030   %}
 9031   ins_pipe(ialu_reg_reg);
 9032 %}
 9033 
 9034 // Shift Left by variable
 9035 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9036 %{
 9037   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9038   effect(KILL cr);
 9039 
 9040   format %{ "salq    $dst, $shift" %}
 9041   ins_encode %{
 9042     __ salq($dst$$Address);
 9043   %}
 9044   ins_pipe(ialu_mem_reg);
 9045 %}
 9046 
 9047 // Arithmetic shift right by one
 9048 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9049 %{
 9050   match(Set dst (RShiftL dst shift));
 9051   effect(KILL cr);
 9052 
 9053   format %{ "sarq    $dst, $shift" %}
 9054   ins_encode %{
 9055     __ sarq($dst$$Register, $shift$$constant);
 9056   %}
 9057   ins_pipe(ialu_reg);
 9058 %}
 9059 
 9060 // Arithmetic shift right by one
 9061 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9062 %{
 9063   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9064   effect(KILL cr);
 9065 
 9066   format %{ "sarq    $dst, $shift" %}
 9067   ins_encode %{
 9068     __ sarq($dst$$Address, $shift$$constant);
 9069   %}
 9070   ins_pipe(ialu_mem_imm);
 9071 %}
 9072 
 9073 // Arithmetic Shift Right by 8-bit immediate
 9074 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9075 %{
 9076   match(Set dst (RShiftL dst shift));
 9077   effect(KILL cr);
 9078 
 9079   format %{ "sarq    $dst, $shift" %}
 9080   ins_encode %{
 9081     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9082   %}
 9083   ins_pipe(ialu_mem_imm);
 9084 %}
 9085 
 9086 // Arithmetic Shift Right by 8-bit immediate
 9087 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9088 %{
 9089   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9090   effect(KILL cr);
 9091 
 9092   format %{ "sarq    $dst, $shift" %}
 9093   ins_encode %{
 9094     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9095   %}
 9096   ins_pipe(ialu_mem_imm);
 9097 %}
 9098 
 9099 // Arithmetic Shift Right by variable
 9100 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9101 %{
 9102   match(Set dst (RShiftL dst shift));
 9103   effect(KILL cr);
 9104 
 9105   format %{ "sarq    $dst, $shift" %}
 9106   ins_encode %{
 9107     __ sarq($dst$$Register);
 9108   %}
 9109   ins_pipe(ialu_reg_reg);
 9110 %}
 9111 
 9112 // Arithmetic Shift Right by variable
 9113 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9114 %{
 9115   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9116   effect(KILL cr);
 9117 
 9118   format %{ "sarq    $dst, $shift" %}
 9119   ins_encode %{
 9120     __ sarq($dst$$Address);
 9121   %}
 9122   ins_pipe(ialu_mem_reg);
 9123 %}
 9124 
 9125 // Logical shift right by one
 9126 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9127 %{
 9128   match(Set dst (URShiftL dst shift));
 9129   effect(KILL cr);
 9130 
 9131   format %{ "shrq    $dst, $shift" %}
 9132   ins_encode %{
 9133     __ shrq($dst$$Register, $shift$$constant);
 9134   %}
 9135   ins_pipe(ialu_reg);
 9136 %}
 9137 
 9138 // Logical shift right by one
 9139 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9140 %{
 9141   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9142   effect(KILL cr);
 9143 
 9144   format %{ "shrq    $dst, $shift" %}
 9145   ins_encode %{
 9146     __ shrq($dst$$Address, $shift$$constant);
 9147   %}
 9148   ins_pipe(ialu_mem_imm);
 9149 %}
 9150 
 9151 // Logical Shift Right by 8-bit immediate
 9152 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (URShiftL dst shift));
 9155   effect(KILL cr);
 9156 
 9157   format %{ "shrq    $dst, $shift" %}
 9158   ins_encode %{
 9159     __ shrq($dst$$Register, $shift$$constant);
 9160   %}
 9161   ins_pipe(ialu_reg);
 9162 %}
 9163 
 9164 // Logical Shift Right by 8-bit immediate
 9165 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9166 %{
 9167   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9168   effect(KILL cr);
 9169 
 9170   format %{ "shrq    $dst, $shift" %}
 9171   ins_encode %{
 9172     __ shrq($dst$$Address, $shift$$constant);
 9173   %}
 9174   ins_pipe(ialu_mem_imm);
 9175 %}
 9176 
 9177 // Logical Shift Right by variable
 9178 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9179 %{
 9180   match(Set dst (URShiftL dst shift));
 9181   effect(KILL cr);
 9182 
 9183   format %{ "shrq    $dst, $shift" %}
 9184   ins_encode %{
 9185     __ shrq($dst$$Register);
 9186   %}
 9187   ins_pipe(ialu_reg_reg);
 9188 %}
 9189 
 9190 // Logical Shift Right by variable
 9191 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9192 %{
 9193   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9194   effect(KILL cr);
 9195 
 9196   format %{ "shrq    $dst, $shift" %}
 9197   ins_encode %{
 9198     __ shrq($dst$$Address);
 9199   %}
 9200   ins_pipe(ialu_mem_reg);
 9201 %}
 9202 
 9203 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9204 // This idiom is used by the compiler for the i2b bytecode.
 9205 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9206 %{
 9207   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9208 
 9209   format %{ "movsbl  $dst, $src\t# i2b" %}
 9210   ins_encode %{
 9211     __ movsbl($dst$$Register, $src$$Register);
 9212   %}
 9213   ins_pipe(ialu_reg_reg);
 9214 %}
 9215 
 9216 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9217 // This idiom is used by the compiler the i2s bytecode.
 9218 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9219 %{
 9220   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9221 
 9222   format %{ "movswl  $dst, $src\t# i2s" %}
 9223   ins_encode %{
 9224     __ movswl($dst$$Register, $src$$Register);
 9225   %}
 9226   ins_pipe(ialu_reg_reg);
 9227 %}
 9228 
 9229 // ROL/ROR instructions
 9230 
 9231 // Rotate left by constant.
 9232 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9233 %{
 9234   predicate(n->bottom_type()->basic_type() == T_INT);
 9235   match(Set dst (RotateLeft dst shift));
 9236   effect(KILL cr);
 9237   format %{ "roll    $dst, $shift" %}
 9238   ins_encode %{
 9239     __ roll($dst$$Register, $shift$$constant);
 9240   %}
 9241   ins_pipe(ialu_reg);
 9242 %}
 9243 
 9244 // Rotate Left by variable
 9245 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9246 %{
 9247   predicate(n->bottom_type()->basic_type() == T_INT);
 9248   match(Set dst (RotateLeft dst shift));
 9249   effect(KILL cr);
 9250   format %{ "roll    $dst, $shift" %}
 9251   ins_encode %{
 9252     __ roll($dst$$Register);
 9253   %}
 9254   ins_pipe(ialu_reg_reg);
 9255 %}
 9256 
 9257 // Rotate Right by constant.
 9258 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9259 %{
 9260   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9261   match(Set dst (RotateRight dst shift));
 9262   effect(KILL cr);
 9263   format %{ "rorl    $dst, $shift" %}
 9264   ins_encode %{
 9265     __ rorl($dst$$Register, $shift$$constant);
 9266   %}
 9267   ins_pipe(ialu_reg);
 9268 %}
 9269 
 9270 // Rotate Right by constant.
 9271 instruct rorI_immI8(rRegI dst, immI8 shift)
 9272 %{
 9273   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9274   match(Set dst (RotateRight dst shift));
 9275   format %{ "rorxd     $dst, $shift" %}
 9276   ins_encode %{
 9277     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9278   %}
 9279   ins_pipe(ialu_reg_reg);
 9280 %}
 9281 
 9282 // Rotate Right by variable
 9283 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9284 %{
 9285   predicate(n->bottom_type()->basic_type() == T_INT);
 9286   match(Set dst (RotateRight dst shift));
 9287   effect(KILL cr);
 9288   format %{ "rorl    $dst, $shift" %}
 9289   ins_encode %{
 9290     __ rorl($dst$$Register);
 9291   %}
 9292   ins_pipe(ialu_reg_reg);
 9293 %}
 9294 
 9295 
 9296 // Rotate Left by constant.
 9297 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9298 %{
 9299   predicate(n->bottom_type()->basic_type() == T_LONG);
 9300   match(Set dst (RotateLeft dst shift));
 9301   effect(KILL cr);
 9302   format %{ "rolq    $dst, $shift" %}
 9303   ins_encode %{
 9304     __ rolq($dst$$Register, $shift$$constant);
 9305   %}
 9306   ins_pipe(ialu_reg);
 9307 %}
 9308 
 9309 // Rotate Left by variable
 9310 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9311 %{
 9312   predicate(n->bottom_type()->basic_type() == T_LONG);
 9313   match(Set dst (RotateLeft dst shift));
 9314   effect(KILL cr);
 9315   format %{ "rolq    $dst, $shift" %}
 9316   ins_encode %{
 9317     __ rolq($dst$$Register);
 9318   %}
 9319   ins_pipe(ialu_reg_reg);
 9320 %}
 9321 
 9322 
 9323 // Rotate Right by constant.
 9324 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9325 %{
 9326   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9327   match(Set dst (RotateRight dst shift));
 9328   effect(KILL cr);
 9329   format %{ "rorq    $dst, $shift" %}
 9330   ins_encode %{
 9331     __ rorq($dst$$Register, $shift$$constant);
 9332   %}
 9333   ins_pipe(ialu_reg);
 9334 %}
 9335 
 9336 
 9337 // Rotate Right by constant
 9338 instruct rorL_immI8(rRegL dst, immI8 shift)
 9339 %{
 9340   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9341   match(Set dst (RotateRight dst shift));
 9342   format %{ "rorxq    $dst, $shift" %}
 9343   ins_encode %{
 9344     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9345   %}
 9346   ins_pipe(ialu_reg_reg);
 9347 %}
 9348 
 9349 // Rotate Right by variable
 9350 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9351 %{
 9352   predicate(n->bottom_type()->basic_type() == T_LONG);
 9353   match(Set dst (RotateRight dst shift));
 9354   effect(KILL cr);
 9355   format %{ "rorq    $dst, $shift" %}
 9356   ins_encode %{
 9357     __ rorq($dst$$Register);
 9358   %}
 9359   ins_pipe(ialu_reg_reg);
 9360 %}
 9361 
 9362 
 9363 // Logical Instructions
 9364 
 9365 // Integer Logical Instructions
 9366 
 9367 // And Instructions
 9368 // And Register with Register
 9369 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9370 %{
 9371   match(Set dst (AndI dst src));
 9372   effect(KILL cr);
 9373 
 9374   format %{ "andl    $dst, $src\t# int" %}
 9375   ins_encode %{
 9376     __ andl($dst$$Register, $src$$Register);
 9377   %}
 9378   ins_pipe(ialu_reg_reg);
 9379 %}
 9380 
 9381 // And Register with Immediate 255
 9382 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9383 %{
 9384   match(Set dst (AndI dst src));
 9385 
 9386   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9387   ins_encode %{
 9388     __ movzbl($dst$$Register, $dst$$Register);
 9389   %}
 9390   ins_pipe(ialu_reg);
 9391 %}
 9392 
 9393 // And Register with Immediate 255 and promote to long
 9394 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9395 %{
 9396   match(Set dst (ConvI2L (AndI src mask)));
 9397 
 9398   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9399   ins_encode %{
 9400     __ movzbl($dst$$Register, $src$$Register);
 9401   %}
 9402   ins_pipe(ialu_reg);
 9403 %}
 9404 
 9405 // And Register with Immediate 65535
 9406 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9407 %{
 9408   match(Set dst (AndI dst src));
 9409 
 9410   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9411   ins_encode %{
 9412     __ movzwl($dst$$Register, $dst$$Register);
 9413   %}
 9414   ins_pipe(ialu_reg);
 9415 %}
 9416 
 9417 // And Register with Immediate 65535 and promote to long
 9418 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9419 %{
 9420   match(Set dst (ConvI2L (AndI src mask)));
 9421 
 9422   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9423   ins_encode %{
 9424     __ movzwl($dst$$Register, $src$$Register);
 9425   %}
 9426   ins_pipe(ialu_reg);
 9427 %}
 9428 
 9429 // Can skip int2long conversions after AND with small bitmask
 9430 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9431 %{
 9432   predicate(VM_Version::supports_bmi2());
 9433   ins_cost(125);
 9434   effect(TEMP tmp, KILL cr);
 9435   match(Set dst (ConvI2L (AndI src mask)));
 9436   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9437   ins_encode %{
 9438     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9439     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9440   %}
 9441   ins_pipe(ialu_reg_reg);
 9442 %}
 9443 
 9444 // And Register with Immediate
 9445 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9446 %{
 9447   match(Set dst (AndI dst src));
 9448   effect(KILL cr);
 9449 
 9450   format %{ "andl    $dst, $src\t# int" %}
 9451   ins_encode %{
 9452     __ andl($dst$$Register, $src$$constant);
 9453   %}
 9454   ins_pipe(ialu_reg);
 9455 %}
 9456 
 9457 // And Register with Memory
 9458 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9459 %{
 9460   match(Set dst (AndI dst (LoadI src)));
 9461   effect(KILL cr);
 9462 
 9463   ins_cost(125);
 9464   format %{ "andl    $dst, $src\t# int" %}
 9465   ins_encode %{
 9466     __ andl($dst$$Register, $src$$Address);
 9467   %}
 9468   ins_pipe(ialu_reg_mem);
 9469 %}
 9470 
 9471 // And Memory with Register
 9472 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9473 %{
 9474   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9475   effect(KILL cr);
 9476 
 9477   ins_cost(150);
 9478   format %{ "andb    $dst, $src\t# byte" %}
 9479   ins_encode %{
 9480     __ andb($dst$$Address, $src$$Register);
 9481   %}
 9482   ins_pipe(ialu_mem_reg);
 9483 %}
 9484 
 9485 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9486 %{
 9487   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9488   effect(KILL cr);
 9489 
 9490   ins_cost(150);
 9491   format %{ "andl    $dst, $src\t# int" %}
 9492   ins_encode %{
 9493     __ andl($dst$$Address, $src$$Register);
 9494   %}
 9495   ins_pipe(ialu_mem_reg);
 9496 %}
 9497 
 9498 // And Memory with Immediate
 9499 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9500 %{
 9501   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9502   effect(KILL cr);
 9503 
 9504   ins_cost(125);
 9505   format %{ "andl    $dst, $src\t# int" %}
 9506   ins_encode %{
 9507     __ andl($dst$$Address, $src$$constant);
 9508   %}
 9509   ins_pipe(ialu_mem_imm);
 9510 %}
 9511 
 9512 // BMI1 instructions
 9513 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9514   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9515   predicate(UseBMI1Instructions);
 9516   effect(KILL cr);
 9517 
 9518   ins_cost(125);
 9519   format %{ "andnl  $dst, $src1, $src2" %}
 9520 
 9521   ins_encode %{
 9522     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9523   %}
 9524   ins_pipe(ialu_reg_mem);
 9525 %}
 9526 
 9527 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9528   match(Set dst (AndI (XorI src1 minus_1) src2));
 9529   predicate(UseBMI1Instructions);
 9530   effect(KILL cr);
 9531 
 9532   format %{ "andnl  $dst, $src1, $src2" %}
 9533 
 9534   ins_encode %{
 9535     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9536   %}
 9537   ins_pipe(ialu_reg);
 9538 %}
 9539 
 9540 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9541   match(Set dst (AndI (SubI imm_zero src) src));
 9542   predicate(UseBMI1Instructions);
 9543   effect(KILL cr);
 9544 
 9545   format %{ "blsil  $dst, $src" %}
 9546 
 9547   ins_encode %{
 9548     __ blsil($dst$$Register, $src$$Register);
 9549   %}
 9550   ins_pipe(ialu_reg);
 9551 %}
 9552 
 9553 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9554   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9555   predicate(UseBMI1Instructions);
 9556   effect(KILL cr);
 9557 
 9558   ins_cost(125);
 9559   format %{ "blsil  $dst, $src" %}
 9560 
 9561   ins_encode %{
 9562     __ blsil($dst$$Register, $src$$Address);
 9563   %}
 9564   ins_pipe(ialu_reg_mem);
 9565 %}
 9566 
 9567 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9568 %{
 9569   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9570   predicate(UseBMI1Instructions);
 9571   effect(KILL cr);
 9572 
 9573   ins_cost(125);
 9574   format %{ "blsmskl $dst, $src" %}
 9575 
 9576   ins_encode %{
 9577     __ blsmskl($dst$$Register, $src$$Address);
 9578   %}
 9579   ins_pipe(ialu_reg_mem);
 9580 %}
 9581 
 9582 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9583 %{
 9584   match(Set dst (XorI (AddI src minus_1) src));
 9585   predicate(UseBMI1Instructions);
 9586   effect(KILL cr);
 9587 
 9588   format %{ "blsmskl $dst, $src" %}
 9589 
 9590   ins_encode %{
 9591     __ blsmskl($dst$$Register, $src$$Register);
 9592   %}
 9593 
 9594   ins_pipe(ialu_reg);
 9595 %}
 9596 
 9597 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9598 %{
 9599   match(Set dst (AndI (AddI src minus_1) src) );
 9600   predicate(UseBMI1Instructions);
 9601   effect(KILL cr);
 9602 
 9603   format %{ "blsrl  $dst, $src" %}
 9604 
 9605   ins_encode %{
 9606     __ blsrl($dst$$Register, $src$$Register);
 9607   %}
 9608 
 9609   ins_pipe(ialu_reg_mem);
 9610 %}
 9611 
 9612 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9613 %{
 9614   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9615   predicate(UseBMI1Instructions);
 9616   effect(KILL cr);
 9617 
 9618   ins_cost(125);
 9619   format %{ "blsrl  $dst, $src" %}
 9620 
 9621   ins_encode %{
 9622     __ blsrl($dst$$Register, $src$$Address);
 9623   %}
 9624 
 9625   ins_pipe(ialu_reg);
 9626 %}
 9627 
 9628 // Or Instructions
 9629 // Or Register with Register
 9630 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9631 %{
 9632   match(Set dst (OrI dst src));
 9633   effect(KILL cr);
 9634 
 9635   format %{ "orl     $dst, $src\t# int" %}
 9636   ins_encode %{
 9637     __ orl($dst$$Register, $src$$Register);
 9638   %}
 9639   ins_pipe(ialu_reg_reg);
 9640 %}
 9641 
 9642 // Or Register with Immediate
 9643 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9644 %{
 9645   match(Set dst (OrI dst src));
 9646   effect(KILL cr);
 9647 
 9648   format %{ "orl     $dst, $src\t# int" %}
 9649   ins_encode %{
 9650     __ orl($dst$$Register, $src$$constant);
 9651   %}
 9652   ins_pipe(ialu_reg);
 9653 %}
 9654 
 9655 // Or Register with Memory
 9656 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9657 %{
 9658   match(Set dst (OrI dst (LoadI src)));
 9659   effect(KILL cr);
 9660 
 9661   ins_cost(125);
 9662   format %{ "orl     $dst, $src\t# int" %}
 9663   ins_encode %{
 9664     __ orl($dst$$Register, $src$$Address);
 9665   %}
 9666   ins_pipe(ialu_reg_mem);
 9667 %}
 9668 
 9669 // Or Memory with Register
 9670 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9671 %{
 9672   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9673   effect(KILL cr);
 9674 
 9675   ins_cost(150);
 9676   format %{ "orb    $dst, $src\t# byte" %}
 9677   ins_encode %{
 9678     __ orb($dst$$Address, $src$$Register);
 9679   %}
 9680   ins_pipe(ialu_mem_reg);
 9681 %}
 9682 
 9683 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9684 %{
 9685   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9686   effect(KILL cr);
 9687 
 9688   ins_cost(150);
 9689   format %{ "orl     $dst, $src\t# int" %}
 9690   ins_encode %{
 9691     __ orl($dst$$Address, $src$$Register);
 9692   %}
 9693   ins_pipe(ialu_mem_reg);
 9694 %}
 9695 
 9696 // Or Memory with Immediate
 9697 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9698 %{
 9699   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9700   effect(KILL cr);
 9701 
 9702   ins_cost(125);
 9703   format %{ "orl     $dst, $src\t# int" %}
 9704   ins_encode %{
 9705     __ orl($dst$$Address, $src$$constant);
 9706   %}
 9707   ins_pipe(ialu_mem_imm);
 9708 %}
 9709 
 9710 // Xor Instructions
 9711 // Xor Register with Register
 9712 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9713 %{
 9714   match(Set dst (XorI dst src));
 9715   effect(KILL cr);
 9716 
 9717   format %{ "xorl    $dst, $src\t# int" %}
 9718   ins_encode %{
 9719     __ xorl($dst$$Register, $src$$Register);
 9720   %}
 9721   ins_pipe(ialu_reg_reg);
 9722 %}
 9723 
 9724 // Xor Register with Immediate -1
 9725 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9726   match(Set dst (XorI dst imm));
 9727 
 9728   format %{ "not    $dst" %}
 9729   ins_encode %{
 9730      __ notl($dst$$Register);
 9731   %}
 9732   ins_pipe(ialu_reg);
 9733 %}
 9734 
 9735 // Xor Register with Immediate
 9736 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9737 %{
 9738   match(Set dst (XorI dst src));
 9739   effect(KILL cr);
 9740 
 9741   format %{ "xorl    $dst, $src\t# int" %}
 9742   ins_encode %{
 9743     __ xorl($dst$$Register, $src$$constant);
 9744   %}
 9745   ins_pipe(ialu_reg);
 9746 %}
 9747 
 9748 // Xor Register with Memory
 9749 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9750 %{
 9751   match(Set dst (XorI dst (LoadI src)));
 9752   effect(KILL cr);
 9753 
 9754   ins_cost(125);
 9755   format %{ "xorl    $dst, $src\t# int" %}
 9756   ins_encode %{
 9757     __ xorl($dst$$Register, $src$$Address);
 9758   %}
 9759   ins_pipe(ialu_reg_mem);
 9760 %}
 9761 
 9762 // Xor Memory with Register
 9763 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9764 %{
 9765   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9766   effect(KILL cr);
 9767 
 9768   ins_cost(150);
 9769   format %{ "xorb    $dst, $src\t# byte" %}
 9770   ins_encode %{
 9771     __ xorb($dst$$Address, $src$$Register);
 9772   %}
 9773   ins_pipe(ialu_mem_reg);
 9774 %}
 9775 
 9776 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9777 %{
 9778   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9779   effect(KILL cr);
 9780 
 9781   ins_cost(150);
 9782   format %{ "xorl    $dst, $src\t# int" %}
 9783   ins_encode %{
 9784     __ xorl($dst$$Address, $src$$Register);
 9785   %}
 9786   ins_pipe(ialu_mem_reg);
 9787 %}
 9788 
 9789 // Xor Memory with Immediate
 9790 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9791 %{
 9792   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9793   effect(KILL cr);
 9794 
 9795   ins_cost(125);
 9796   format %{ "xorl    $dst, $src\t# int" %}
 9797   ins_encode %{
 9798     __ xorl($dst$$Address, $src$$constant);
 9799   %}
 9800   ins_pipe(ialu_mem_imm);
 9801 %}
 9802 
 9803 
 9804 // Long Logical Instructions
 9805 
 9806 // And Instructions
 9807 // And Register with Register
 9808 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9809 %{
 9810   match(Set dst (AndL dst src));
 9811   effect(KILL cr);
 9812 
 9813   format %{ "andq    $dst, $src\t# long" %}
 9814   ins_encode %{
 9815     __ andq($dst$$Register, $src$$Register);
 9816   %}
 9817   ins_pipe(ialu_reg_reg);
 9818 %}
 9819 
 9820 // And Register with Immediate 255
 9821 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9822 %{
 9823   match(Set dst (AndL dst src));
 9824 
 9825   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9826   ins_encode %{
 9827     __ movzbq($dst$$Register, $dst$$Register);
 9828   %}
 9829   ins_pipe(ialu_reg);
 9830 %}
 9831 
 9832 // And Register with Immediate 65535
 9833 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9834 %{
 9835   match(Set dst (AndL dst src));
 9836 
 9837   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9838   ins_encode %{
 9839     __ movzwq($dst$$Register, $dst$$Register);
 9840   %}
 9841   ins_pipe(ialu_reg);
 9842 %}
 9843 
 9844 // And Register with Immediate
 9845 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9846 %{
 9847   match(Set dst (AndL dst src));
 9848   effect(KILL cr);
 9849 
 9850   format %{ "andq    $dst, $src\t# long" %}
 9851   ins_encode %{
 9852     __ andq($dst$$Register, $src$$constant);
 9853   %}
 9854   ins_pipe(ialu_reg);
 9855 %}
 9856 
 9857 // And Register with Memory
 9858 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9859 %{
 9860   match(Set dst (AndL dst (LoadL src)));
 9861   effect(KILL cr);
 9862 
 9863   ins_cost(125);
 9864   format %{ "andq    $dst, $src\t# long" %}
 9865   ins_encode %{
 9866     __ andq($dst$$Register, $src$$Address);
 9867   %}
 9868   ins_pipe(ialu_reg_mem);
 9869 %}
 9870 
 9871 // And Memory with Register
 9872 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9873 %{
 9874   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9875   effect(KILL cr);
 9876 
 9877   ins_cost(150);
 9878   format %{ "andq    $dst, $src\t# long" %}
 9879   ins_encode %{
 9880     __ andq($dst$$Address, $src$$Register);
 9881   %}
 9882   ins_pipe(ialu_mem_reg);
 9883 %}
 9884 
 9885 // And Memory with Immediate
 9886 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9887 %{
 9888   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9889   effect(KILL cr);
 9890 
 9891   ins_cost(125);
 9892   format %{ "andq    $dst, $src\t# long" %}
 9893   ins_encode %{
 9894     __ andq($dst$$Address, $src$$constant);
 9895   %}
 9896   ins_pipe(ialu_mem_imm);
 9897 %}
 9898 
 9899 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9900 %{
 9901   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9902   // because AND/OR works well enough for 8/32-bit values.
 9903   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9904 
 9905   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9906   effect(KILL cr);
 9907 
 9908   ins_cost(125);
 9909   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9910   ins_encode %{
 9911     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9912   %}
 9913   ins_pipe(ialu_mem_imm);
 9914 %}
 9915 
 9916 // BMI1 instructions
 9917 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9918   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9919   predicate(UseBMI1Instructions);
 9920   effect(KILL cr);
 9921 
 9922   ins_cost(125);
 9923   format %{ "andnq  $dst, $src1, $src2" %}
 9924 
 9925   ins_encode %{
 9926     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9927   %}
 9928   ins_pipe(ialu_reg_mem);
 9929 %}
 9930 
 9931 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9932   match(Set dst (AndL (XorL src1 minus_1) src2));
 9933   predicate(UseBMI1Instructions);
 9934   effect(KILL cr);
 9935 
 9936   format %{ "andnq  $dst, $src1, $src2" %}
 9937 
 9938   ins_encode %{
 9939   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9940   %}
 9941   ins_pipe(ialu_reg_mem);
 9942 %}
 9943 
 9944 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9945   match(Set dst (AndL (SubL imm_zero src) src));
 9946   predicate(UseBMI1Instructions);
 9947   effect(KILL cr);
 9948 
 9949   format %{ "blsiq  $dst, $src" %}
 9950 
 9951   ins_encode %{
 9952     __ blsiq($dst$$Register, $src$$Register);
 9953   %}
 9954   ins_pipe(ialu_reg);
 9955 %}
 9956 
 9957 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9958   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9959   predicate(UseBMI1Instructions);
 9960   effect(KILL cr);
 9961 
 9962   ins_cost(125);
 9963   format %{ "blsiq  $dst, $src" %}
 9964 
 9965   ins_encode %{
 9966     __ blsiq($dst$$Register, $src$$Address);
 9967   %}
 9968   ins_pipe(ialu_reg_mem);
 9969 %}
 9970 
 9971 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9972 %{
 9973   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9974   predicate(UseBMI1Instructions);
 9975   effect(KILL cr);
 9976 
 9977   ins_cost(125);
 9978   format %{ "blsmskq $dst, $src" %}
 9979 
 9980   ins_encode %{
 9981     __ blsmskq($dst$$Register, $src$$Address);
 9982   %}
 9983   ins_pipe(ialu_reg_mem);
 9984 %}
 9985 
 9986 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9987 %{
 9988   match(Set dst (XorL (AddL src minus_1) src));
 9989   predicate(UseBMI1Instructions);
 9990   effect(KILL cr);
 9991 
 9992   format %{ "blsmskq $dst, $src" %}
 9993 
 9994   ins_encode %{
 9995     __ blsmskq($dst$$Register, $src$$Register);
 9996   %}
 9997 
 9998   ins_pipe(ialu_reg);
 9999 %}
10000 
10001 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10002 %{
10003   match(Set dst (AndL (AddL src minus_1) src) );
10004   predicate(UseBMI1Instructions);
10005   effect(KILL cr);
10006 
10007   format %{ "blsrq  $dst, $src" %}
10008 
10009   ins_encode %{
10010     __ blsrq($dst$$Register, $src$$Register);
10011   %}
10012 
10013   ins_pipe(ialu_reg);
10014 %}
10015 
10016 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10017 %{
10018   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10019   predicate(UseBMI1Instructions);
10020   effect(KILL cr);
10021 
10022   ins_cost(125);
10023   format %{ "blsrq  $dst, $src" %}
10024 
10025   ins_encode %{
10026     __ blsrq($dst$$Register, $src$$Address);
10027   %}
10028 
10029   ins_pipe(ialu_reg);
10030 %}
10031 
10032 // Or Instructions
10033 // Or Register with Register
10034 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10035 %{
10036   match(Set dst (OrL dst src));
10037   effect(KILL cr);
10038 
10039   format %{ "orq     $dst, $src\t# long" %}
10040   ins_encode %{
10041     __ orq($dst$$Register, $src$$Register);
10042   %}
10043   ins_pipe(ialu_reg_reg);
10044 %}
10045 
10046 // Use any_RegP to match R15 (TLS register) without spilling.
10047 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10048   match(Set dst (OrL dst (CastP2X src)));
10049   effect(KILL cr);
10050 
10051   format %{ "orq     $dst, $src\t# long" %}
10052   ins_encode %{
10053     __ orq($dst$$Register, $src$$Register);
10054   %}
10055   ins_pipe(ialu_reg_reg);
10056 %}
10057 
10058 
10059 // Or Register with Immediate
10060 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10061 %{
10062   match(Set dst (OrL dst src));
10063   effect(KILL cr);
10064 
10065   format %{ "orq     $dst, $src\t# long" %}
10066   ins_encode %{
10067     __ orq($dst$$Register, $src$$constant);
10068   %}
10069   ins_pipe(ialu_reg);
10070 %}
10071 
10072 // Or Register with Memory
10073 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10074 %{
10075   match(Set dst (OrL dst (LoadL src)));
10076   effect(KILL cr);
10077 
10078   ins_cost(125);
10079   format %{ "orq     $dst, $src\t# long" %}
10080   ins_encode %{
10081     __ orq($dst$$Register, $src$$Address);
10082   %}
10083   ins_pipe(ialu_reg_mem);
10084 %}
10085 
10086 // Or Memory with Register
10087 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10088 %{
10089   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10090   effect(KILL cr);
10091 
10092   ins_cost(150);
10093   format %{ "orq     $dst, $src\t# long" %}
10094   ins_encode %{
10095     __ orq($dst$$Address, $src$$Register);
10096   %}
10097   ins_pipe(ialu_mem_reg);
10098 %}
10099 
10100 // Or Memory with Immediate
10101 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10102 %{
10103   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10104   effect(KILL cr);
10105 
10106   ins_cost(125);
10107   format %{ "orq     $dst, $src\t# long" %}
10108   ins_encode %{
10109     __ orq($dst$$Address, $src$$constant);
10110   %}
10111   ins_pipe(ialu_mem_imm);
10112 %}
10113 
10114 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10115 %{
10116   // con should be a pure 64-bit power of 2 immediate
10117   // because AND/OR works well enough for 8/32-bit values.
10118   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10119 
10120   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10121   effect(KILL cr);
10122 
10123   ins_cost(125);
10124   format %{ "btsq    $dst, log2($con)\t# long" %}
10125   ins_encode %{
10126     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10127   %}
10128   ins_pipe(ialu_mem_imm);
10129 %}
10130 
10131 // Xor Instructions
10132 // Xor Register with Register
10133 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10134 %{
10135   match(Set dst (XorL dst src));
10136   effect(KILL cr);
10137 
10138   format %{ "xorq    $dst, $src\t# long" %}
10139   ins_encode %{
10140     __ xorq($dst$$Register, $src$$Register);
10141   %}
10142   ins_pipe(ialu_reg_reg);
10143 %}
10144 
10145 // Xor Register with Immediate -1
10146 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10147   match(Set dst (XorL dst imm));
10148 
10149   format %{ "notq   $dst" %}
10150   ins_encode %{
10151      __ notq($dst$$Register);
10152   %}
10153   ins_pipe(ialu_reg);
10154 %}
10155 
10156 // Xor Register with Immediate
10157 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10158 %{
10159   match(Set dst (XorL dst src));
10160   effect(KILL cr);
10161 
10162   format %{ "xorq    $dst, $src\t# long" %}
10163   ins_encode %{
10164     __ xorq($dst$$Register, $src$$constant);
10165   %}
10166   ins_pipe(ialu_reg);
10167 %}
10168 
10169 // Xor Register with Memory
10170 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10171 %{
10172   match(Set dst (XorL dst (LoadL src)));
10173   effect(KILL cr);
10174 
10175   ins_cost(125);
10176   format %{ "xorq    $dst, $src\t# long" %}
10177   ins_encode %{
10178     __ xorq($dst$$Register, $src$$Address);
10179   %}
10180   ins_pipe(ialu_reg_mem);
10181 %}
10182 
10183 // Xor Memory with Register
10184 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10185 %{
10186   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10187   effect(KILL cr);
10188 
10189   ins_cost(150);
10190   format %{ "xorq    $dst, $src\t# long" %}
10191   ins_encode %{
10192     __ xorq($dst$$Address, $src$$Register);
10193   %}
10194   ins_pipe(ialu_mem_reg);
10195 %}
10196 
10197 // Xor Memory with Immediate
10198 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10199 %{
10200   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10201   effect(KILL cr);
10202 
10203   ins_cost(125);
10204   format %{ "xorq    $dst, $src\t# long" %}
10205   ins_encode %{
10206     __ xorq($dst$$Address, $src$$constant);
10207   %}
10208   ins_pipe(ialu_mem_imm);
10209 %}
10210 
10211 // Convert Int to Boolean
10212 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10213 %{
10214   match(Set dst (Conv2B src));
10215   effect(KILL cr);
10216 
10217   format %{ "testl   $src, $src\t# ci2b\n\t"
10218             "setnz   $dst\n\t"
10219             "movzbl  $dst, $dst" %}
10220   ins_encode %{
10221     __ testl($src$$Register, $src$$Register);
10222     __ set_byte_if_not_zero($dst$$Register);
10223     __ movzbl($dst$$Register, $dst$$Register);
10224   %}
10225   ins_pipe(pipe_slow); // XXX
10226 %}
10227 
10228 // Convert Pointer to Boolean
10229 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10230 %{
10231   match(Set dst (Conv2B src));
10232   effect(KILL cr);
10233 
10234   format %{ "testq   $src, $src\t# cp2b\n\t"
10235             "setnz   $dst\n\t"
10236             "movzbl  $dst, $dst" %}
10237   ins_encode %{
10238     __ testq($src$$Register, $src$$Register);
10239     __ set_byte_if_not_zero($dst$$Register);
10240     __ movzbl($dst$$Register, $dst$$Register);
10241   %}
10242   ins_pipe(pipe_slow); // XXX
10243 %}
10244 
10245 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10246 %{
10247   match(Set dst (CmpLTMask p q));
10248   effect(KILL cr);
10249 
10250   ins_cost(400);
10251   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10252             "setlt   $dst\n\t"
10253             "movzbl  $dst, $dst\n\t"
10254             "negl    $dst" %}
10255   ins_encode %{
10256     __ cmpl($p$$Register, $q$$Register);
10257     __ setl($dst$$Register);
10258     __ movzbl($dst$$Register, $dst$$Register);
10259     __ negl($dst$$Register);
10260   %}
10261   ins_pipe(pipe_slow);
10262 %}
10263 
10264 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10265 %{
10266   match(Set dst (CmpLTMask dst zero));
10267   effect(KILL cr);
10268 
10269   ins_cost(100);
10270   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10271   ins_encode %{
10272     __ sarl($dst$$Register, 31);
10273   %}
10274   ins_pipe(ialu_reg);
10275 %}
10276 
10277 /* Better to save a register than avoid a branch */
10278 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10279 %{
10280   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10281   effect(KILL cr);
10282   ins_cost(300);
10283   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10284             "jge     done\n\t"
10285             "addl    $p,$y\n"
10286             "done:   " %}
10287   ins_encode %{
10288     Register Rp = $p$$Register;
10289     Register Rq = $q$$Register;
10290     Register Ry = $y$$Register;
10291     Label done;
10292     __ subl(Rp, Rq);
10293     __ jccb(Assembler::greaterEqual, done);
10294     __ addl(Rp, Ry);
10295     __ bind(done);
10296   %}
10297   ins_pipe(pipe_cmplt);
10298 %}
10299 
10300 /* Better to save a register than avoid a branch */
10301 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10302 %{
10303   match(Set y (AndI (CmpLTMask p q) y));
10304   effect(KILL cr);
10305 
10306   ins_cost(300);
10307 
10308   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10309             "jlt     done\n\t"
10310             "xorl    $y, $y\n"
10311             "done:   " %}
10312   ins_encode %{
10313     Register Rp = $p$$Register;
10314     Register Rq = $q$$Register;
10315     Register Ry = $y$$Register;
10316     Label done;
10317     __ cmpl(Rp, Rq);
10318     __ jccb(Assembler::less, done);
10319     __ xorl(Ry, Ry);
10320     __ bind(done);
10321   %}
10322   ins_pipe(pipe_cmplt);
10323 %}
10324 
10325 
10326 //---------- FP Instructions------------------------------------------------
10327 
10328 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10329 %{
10330   match(Set cr (CmpF src1 src2));
10331 
10332   ins_cost(145);
10333   format %{ "ucomiss $src1, $src2\n\t"
10334             "jnp,s   exit\n\t"
10335             "pushfq\t# saw NaN, set CF\n\t"
10336             "andq    [rsp], #0xffffff2b\n\t"
10337             "popfq\n"
10338     "exit:" %}
10339   ins_encode %{
10340     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10341     emit_cmpfp_fixup(_masm);
10342   %}
10343   ins_pipe(pipe_slow);
10344 %}
10345 
10346 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10347   match(Set cr (CmpF src1 src2));
10348 
10349   ins_cost(100);
10350   format %{ "ucomiss $src1, $src2" %}
10351   ins_encode %{
10352     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10353   %}
10354   ins_pipe(pipe_slow);
10355 %}
10356 
10357 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10358 %{
10359   match(Set cr (CmpF src1 (LoadF src2)));
10360 
10361   ins_cost(145);
10362   format %{ "ucomiss $src1, $src2\n\t"
10363             "jnp,s   exit\n\t"
10364             "pushfq\t# saw NaN, set CF\n\t"
10365             "andq    [rsp], #0xffffff2b\n\t"
10366             "popfq\n"
10367     "exit:" %}
10368   ins_encode %{
10369     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10370     emit_cmpfp_fixup(_masm);
10371   %}
10372   ins_pipe(pipe_slow);
10373 %}
10374 
10375 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10376   match(Set cr (CmpF src1 (LoadF src2)));
10377 
10378   ins_cost(100);
10379   format %{ "ucomiss $src1, $src2" %}
10380   ins_encode %{
10381     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10382   %}
10383   ins_pipe(pipe_slow);
10384 %}
10385 
10386 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10387   match(Set cr (CmpF src con));
10388 
10389   ins_cost(145);
10390   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10391             "jnp,s   exit\n\t"
10392             "pushfq\t# saw NaN, set CF\n\t"
10393             "andq    [rsp], #0xffffff2b\n\t"
10394             "popfq\n"
10395     "exit:" %}
10396   ins_encode %{
10397     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10398     emit_cmpfp_fixup(_masm);
10399   %}
10400   ins_pipe(pipe_slow);
10401 %}
10402 
10403 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10404   match(Set cr (CmpF src con));
10405   ins_cost(100);
10406   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10407   ins_encode %{
10408     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10409   %}
10410   ins_pipe(pipe_slow);
10411 %}
10412 
10413 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10414 %{
10415   match(Set cr (CmpD src1 src2));
10416 
10417   ins_cost(145);
10418   format %{ "ucomisd $src1, $src2\n\t"
10419             "jnp,s   exit\n\t"
10420             "pushfq\t# saw NaN, set CF\n\t"
10421             "andq    [rsp], #0xffffff2b\n\t"
10422             "popfq\n"
10423     "exit:" %}
10424   ins_encode %{
10425     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10426     emit_cmpfp_fixup(_masm);
10427   %}
10428   ins_pipe(pipe_slow);
10429 %}
10430 
10431 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10432   match(Set cr (CmpD src1 src2));
10433 
10434   ins_cost(100);
10435   format %{ "ucomisd $src1, $src2 test" %}
10436   ins_encode %{
10437     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10438   %}
10439   ins_pipe(pipe_slow);
10440 %}
10441 
10442 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10443 %{
10444   match(Set cr (CmpD src1 (LoadD src2)));
10445 
10446   ins_cost(145);
10447   format %{ "ucomisd $src1, $src2\n\t"
10448             "jnp,s   exit\n\t"
10449             "pushfq\t# saw NaN, set CF\n\t"
10450             "andq    [rsp], #0xffffff2b\n\t"
10451             "popfq\n"
10452     "exit:" %}
10453   ins_encode %{
10454     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10455     emit_cmpfp_fixup(_masm);
10456   %}
10457   ins_pipe(pipe_slow);
10458 %}
10459 
10460 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10461   match(Set cr (CmpD src1 (LoadD src2)));
10462 
10463   ins_cost(100);
10464   format %{ "ucomisd $src1, $src2" %}
10465   ins_encode %{
10466     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10467   %}
10468   ins_pipe(pipe_slow);
10469 %}
10470 
10471 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10472   match(Set cr (CmpD src con));
10473 
10474   ins_cost(145);
10475   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10476             "jnp,s   exit\n\t"
10477             "pushfq\t# saw NaN, set CF\n\t"
10478             "andq    [rsp], #0xffffff2b\n\t"
10479             "popfq\n"
10480     "exit:" %}
10481   ins_encode %{
10482     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10483     emit_cmpfp_fixup(_masm);
10484   %}
10485   ins_pipe(pipe_slow);
10486 %}
10487 
10488 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10489   match(Set cr (CmpD src con));
10490   ins_cost(100);
10491   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10492   ins_encode %{
10493     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10494   %}
10495   ins_pipe(pipe_slow);
10496 %}
10497 
10498 // Compare into -1,0,1
10499 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10500 %{
10501   match(Set dst (CmpF3 src1 src2));
10502   effect(KILL cr);
10503 
10504   ins_cost(275);
10505   format %{ "ucomiss $src1, $src2\n\t"
10506             "movl    $dst, #-1\n\t"
10507             "jp,s    done\n\t"
10508             "jb,s    done\n\t"
10509             "setne   $dst\n\t"
10510             "movzbl  $dst, $dst\n"
10511     "done:" %}
10512   ins_encode %{
10513     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10514     emit_cmpfp3(_masm, $dst$$Register);
10515   %}
10516   ins_pipe(pipe_slow);
10517 %}
10518 
10519 // Compare into -1,0,1
10520 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10521 %{
10522   match(Set dst (CmpF3 src1 (LoadF src2)));
10523   effect(KILL cr);
10524 
10525   ins_cost(275);
10526   format %{ "ucomiss $src1, $src2\n\t"
10527             "movl    $dst, #-1\n\t"
10528             "jp,s    done\n\t"
10529             "jb,s    done\n\t"
10530             "setne   $dst\n\t"
10531             "movzbl  $dst, $dst\n"
10532     "done:" %}
10533   ins_encode %{
10534     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10535     emit_cmpfp3(_masm, $dst$$Register);
10536   %}
10537   ins_pipe(pipe_slow);
10538 %}
10539 
10540 // Compare into -1,0,1
10541 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10542   match(Set dst (CmpF3 src con));
10543   effect(KILL cr);
10544 
10545   ins_cost(275);
10546   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10547             "movl    $dst, #-1\n\t"
10548             "jp,s    done\n\t"
10549             "jb,s    done\n\t"
10550             "setne   $dst\n\t"
10551             "movzbl  $dst, $dst\n"
10552     "done:" %}
10553   ins_encode %{
10554     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10555     emit_cmpfp3(_masm, $dst$$Register);
10556   %}
10557   ins_pipe(pipe_slow);
10558 %}
10559 
10560 // Compare into -1,0,1
10561 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10562 %{
10563   match(Set dst (CmpD3 src1 src2));
10564   effect(KILL cr);
10565 
10566   ins_cost(275);
10567   format %{ "ucomisd $src1, $src2\n\t"
10568             "movl    $dst, #-1\n\t"
10569             "jp,s    done\n\t"
10570             "jb,s    done\n\t"
10571             "setne   $dst\n\t"
10572             "movzbl  $dst, $dst\n"
10573     "done:" %}
10574   ins_encode %{
10575     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10576     emit_cmpfp3(_masm, $dst$$Register);
10577   %}
10578   ins_pipe(pipe_slow);
10579 %}
10580 
10581 // Compare into -1,0,1
10582 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10583 %{
10584   match(Set dst (CmpD3 src1 (LoadD src2)));
10585   effect(KILL cr);
10586 
10587   ins_cost(275);
10588   format %{ "ucomisd $src1, $src2\n\t"
10589             "movl    $dst, #-1\n\t"
10590             "jp,s    done\n\t"
10591             "jb,s    done\n\t"
10592             "setne   $dst\n\t"
10593             "movzbl  $dst, $dst\n"
10594     "done:" %}
10595   ins_encode %{
10596     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10597     emit_cmpfp3(_masm, $dst$$Register);
10598   %}
10599   ins_pipe(pipe_slow);
10600 %}
10601 
10602 // Compare into -1,0,1
10603 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10604   match(Set dst (CmpD3 src con));
10605   effect(KILL cr);
10606 
10607   ins_cost(275);
10608   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10609             "movl    $dst, #-1\n\t"
10610             "jp,s    done\n\t"
10611             "jb,s    done\n\t"
10612             "setne   $dst\n\t"
10613             "movzbl  $dst, $dst\n"
10614     "done:" %}
10615   ins_encode %{
10616     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10617     emit_cmpfp3(_masm, $dst$$Register);
10618   %}
10619   ins_pipe(pipe_slow);
10620 %}
10621 
10622 //----------Arithmetic Conversion Instructions---------------------------------
10623 
10624 instruct convF2D_reg_reg(regD dst, regF src)
10625 %{
10626   match(Set dst (ConvF2D src));
10627 
10628   format %{ "cvtss2sd $dst, $src" %}
10629   ins_encode %{
10630     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10631   %}
10632   ins_pipe(pipe_slow); // XXX
10633 %}
10634 
10635 instruct convF2D_reg_mem(regD dst, memory src)
10636 %{
10637   match(Set dst (ConvF2D (LoadF src)));
10638 
10639   format %{ "cvtss2sd $dst, $src" %}
10640   ins_encode %{
10641     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10642   %}
10643   ins_pipe(pipe_slow); // XXX
10644 %}
10645 
10646 instruct convD2F_reg_reg(regF dst, regD src)
10647 %{
10648   match(Set dst (ConvD2F src));
10649 
10650   format %{ "cvtsd2ss $dst, $src" %}
10651   ins_encode %{
10652     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10653   %}
10654   ins_pipe(pipe_slow); // XXX
10655 %}
10656 
10657 instruct convD2F_reg_mem(regF dst, memory src)
10658 %{
10659   match(Set dst (ConvD2F (LoadD src)));
10660 
10661   format %{ "cvtsd2ss $dst, $src" %}
10662   ins_encode %{
10663     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10664   %}
10665   ins_pipe(pipe_slow); // XXX
10666 %}
10667 
10668 // XXX do mem variants
10669 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10670 %{
10671   match(Set dst (ConvF2I src));
10672   effect(KILL cr);
10673   format %{ "convert_f2i $dst,$src" %}
10674   ins_encode %{
10675     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10676   %}
10677   ins_pipe(pipe_slow);
10678 %}
10679 
10680 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10681 %{
10682   match(Set dst (ConvF2L src));
10683   effect(KILL cr);
10684   format %{ "convert_f2l $dst,$src"%}
10685   ins_encode %{
10686     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10687   %}
10688   ins_pipe(pipe_slow);
10689 %}
10690 
10691 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10692 %{
10693   match(Set dst (ConvD2I src));
10694   effect(KILL cr);
10695   format %{ "convert_d2i $dst,$src"%}
10696   ins_encode %{
10697     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10698   %}
10699   ins_pipe(pipe_slow);
10700 %}
10701 
10702 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10703 %{
10704   match(Set dst (ConvD2L src));
10705   effect(KILL cr);
10706   format %{ "convert_d2l $dst,$src"%}
10707   ins_encode %{
10708     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10709   %}
10710   ins_pipe(pipe_slow);
10711 %}
10712 
10713 instruct convI2F_reg_reg(regF dst, rRegI src)
10714 %{
10715   predicate(!UseXmmI2F);
10716   match(Set dst (ConvI2F src));
10717 
10718   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10719   ins_encode %{
10720     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10721   %}
10722   ins_pipe(pipe_slow); // XXX
10723 %}
10724 
10725 instruct convI2F_reg_mem(regF dst, memory src)
10726 %{
10727   match(Set dst (ConvI2F (LoadI src)));
10728 
10729   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10730   ins_encode %{
10731     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10732   %}
10733   ins_pipe(pipe_slow); // XXX
10734 %}
10735 
10736 instruct convI2D_reg_reg(regD dst, rRegI src)
10737 %{
10738   predicate(!UseXmmI2D);
10739   match(Set dst (ConvI2D src));
10740 
10741   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10742   ins_encode %{
10743     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10744   %}
10745   ins_pipe(pipe_slow); // XXX
10746 %}
10747 
10748 instruct convI2D_reg_mem(regD dst, memory src)
10749 %{
10750   match(Set dst (ConvI2D (LoadI src)));
10751 
10752   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10753   ins_encode %{
10754     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10755   %}
10756   ins_pipe(pipe_slow); // XXX
10757 %}
10758 
10759 instruct convXI2F_reg(regF dst, rRegI src)
10760 %{
10761   predicate(UseXmmI2F);
10762   match(Set dst (ConvI2F src));
10763 
10764   format %{ "movdl $dst, $src\n\t"
10765             "cvtdq2psl $dst, $dst\t# i2f" %}
10766   ins_encode %{
10767     __ movdl($dst$$XMMRegister, $src$$Register);
10768     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10769   %}
10770   ins_pipe(pipe_slow); // XXX
10771 %}
10772 
10773 instruct convXI2D_reg(regD dst, rRegI src)
10774 %{
10775   predicate(UseXmmI2D);
10776   match(Set dst (ConvI2D src));
10777 
10778   format %{ "movdl $dst, $src\n\t"
10779             "cvtdq2pdl $dst, $dst\t# i2d" %}
10780   ins_encode %{
10781     __ movdl($dst$$XMMRegister, $src$$Register);
10782     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10783   %}
10784   ins_pipe(pipe_slow); // XXX
10785 %}
10786 
10787 instruct convL2F_reg_reg(regF dst, rRegL src)
10788 %{
10789   match(Set dst (ConvL2F src));
10790 
10791   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10792   ins_encode %{
10793     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10794   %}
10795   ins_pipe(pipe_slow); // XXX
10796 %}
10797 
10798 instruct convL2F_reg_mem(regF dst, memory src)
10799 %{
10800   match(Set dst (ConvL2F (LoadL src)));
10801 
10802   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10803   ins_encode %{
10804     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10805   %}
10806   ins_pipe(pipe_slow); // XXX
10807 %}
10808 
10809 instruct convL2D_reg_reg(regD dst, rRegL src)
10810 %{
10811   match(Set dst (ConvL2D src));
10812 
10813   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10814   ins_encode %{
10815     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10816   %}
10817   ins_pipe(pipe_slow); // XXX
10818 %}
10819 
10820 instruct convL2D_reg_mem(regD dst, memory src)
10821 %{
10822   match(Set dst (ConvL2D (LoadL src)));
10823 
10824   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10825   ins_encode %{
10826     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10827   %}
10828   ins_pipe(pipe_slow); // XXX
10829 %}
10830 
10831 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10832 %{
10833   match(Set dst (ConvI2L src));
10834 
10835   ins_cost(125);
10836   format %{ "movslq  $dst, $src\t# i2l" %}
10837   ins_encode %{
10838     __ movslq($dst$$Register, $src$$Register);
10839   %}
10840   ins_pipe(ialu_reg_reg);
10841 %}
10842 
10843 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10844 // %{
10845 //   match(Set dst (ConvI2L src));
10846 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10847 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10848 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10849 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10850 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10851 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10852 
10853 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10854 //   ins_encode(enc_copy(dst, src));
10855 // //   opcode(0x63); // needs REX.W
10856 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10857 //   ins_pipe(ialu_reg_reg);
10858 // %}
10859 
10860 // Zero-extend convert int to long
10861 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10862 %{
10863   match(Set dst (AndL (ConvI2L src) mask));
10864 
10865   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10866   ins_encode %{
10867     if ($dst$$reg != $src$$reg) {
10868       __ movl($dst$$Register, $src$$Register);
10869     }
10870   %}
10871   ins_pipe(ialu_reg_reg);
10872 %}
10873 
10874 // Zero-extend convert int to long
10875 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10876 %{
10877   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10878 
10879   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10880   ins_encode %{
10881     __ movl($dst$$Register, $src$$Address);
10882   %}
10883   ins_pipe(ialu_reg_mem);
10884 %}
10885 
10886 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10887 %{
10888   match(Set dst (AndL src mask));
10889 
10890   format %{ "movl    $dst, $src\t# zero-extend long" %}
10891   ins_encode %{
10892     __ movl($dst$$Register, $src$$Register);
10893   %}
10894   ins_pipe(ialu_reg_reg);
10895 %}
10896 
10897 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10898 %{
10899   match(Set dst (ConvL2I src));
10900 
10901   format %{ "movl    $dst, $src\t# l2i" %}
10902   ins_encode %{
10903     __ movl($dst$$Register, $src$$Register);
10904   %}
10905   ins_pipe(ialu_reg_reg);
10906 %}
10907 
10908 
10909 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10910   match(Set dst (MoveF2I src));
10911   effect(DEF dst, USE src);
10912 
10913   ins_cost(125);
10914   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10915   ins_encode %{
10916     __ movl($dst$$Register, Address(rsp, $src$$disp));
10917   %}
10918   ins_pipe(ialu_reg_mem);
10919 %}
10920 
10921 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10922   match(Set dst (MoveI2F src));
10923   effect(DEF dst, USE src);
10924 
10925   ins_cost(125);
10926   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10927   ins_encode %{
10928     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10929   %}
10930   ins_pipe(pipe_slow);
10931 %}
10932 
10933 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10934   match(Set dst (MoveD2L src));
10935   effect(DEF dst, USE src);
10936 
10937   ins_cost(125);
10938   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10939   ins_encode %{
10940     __ movq($dst$$Register, Address(rsp, $src$$disp));
10941   %}
10942   ins_pipe(ialu_reg_mem);
10943 %}
10944 
10945 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10946   predicate(!UseXmmLoadAndClearUpper);
10947   match(Set dst (MoveL2D src));
10948   effect(DEF dst, USE src);
10949 
10950   ins_cost(125);
10951   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10952   ins_encode %{
10953     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10954   %}
10955   ins_pipe(pipe_slow);
10956 %}
10957 
10958 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10959   predicate(UseXmmLoadAndClearUpper);
10960   match(Set dst (MoveL2D src));
10961   effect(DEF dst, USE src);
10962 
10963   ins_cost(125);
10964   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10965   ins_encode %{
10966     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10967   %}
10968   ins_pipe(pipe_slow);
10969 %}
10970 
10971 
10972 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10973   match(Set dst (MoveF2I src));
10974   effect(DEF dst, USE src);
10975 
10976   ins_cost(95); // XXX
10977   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10978   ins_encode %{
10979     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10980   %}
10981   ins_pipe(pipe_slow);
10982 %}
10983 
10984 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10985   match(Set dst (MoveI2F src));
10986   effect(DEF dst, USE src);
10987 
10988   ins_cost(100);
10989   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10990   ins_encode %{
10991     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10992   %}
10993   ins_pipe( ialu_mem_reg );
10994 %}
10995 
10996 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10997   match(Set dst (MoveD2L src));
10998   effect(DEF dst, USE src);
10999 
11000   ins_cost(95); // XXX
11001   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11002   ins_encode %{
11003     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11004   %}
11005   ins_pipe(pipe_slow);
11006 %}
11007 
11008 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11009   match(Set dst (MoveL2D src));
11010   effect(DEF dst, USE src);
11011 
11012   ins_cost(100);
11013   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11014   ins_encode %{
11015     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11016   %}
11017   ins_pipe(ialu_mem_reg);
11018 %}
11019 
11020 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11021   match(Set dst (MoveF2I src));
11022   effect(DEF dst, USE src);
11023   ins_cost(85);
11024   format %{ "movd    $dst,$src\t# MoveF2I" %}
11025   ins_encode %{
11026     __ movdl($dst$$Register, $src$$XMMRegister);
11027   %}
11028   ins_pipe( pipe_slow );
11029 %}
11030 
11031 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11032   match(Set dst (MoveD2L src));
11033   effect(DEF dst, USE src);
11034   ins_cost(85);
11035   format %{ "movd    $dst,$src\t# MoveD2L" %}
11036   ins_encode %{
11037     __ movdq($dst$$Register, $src$$XMMRegister);
11038   %}
11039   ins_pipe( pipe_slow );
11040 %}
11041 
11042 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11043   match(Set dst (MoveI2F src));
11044   effect(DEF dst, USE src);
11045   ins_cost(100);
11046   format %{ "movd    $dst,$src\t# MoveI2F" %}
11047   ins_encode %{
11048     __ movdl($dst$$XMMRegister, $src$$Register);
11049   %}
11050   ins_pipe( pipe_slow );
11051 %}
11052 
11053 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11054   match(Set dst (MoveL2D src));
11055   effect(DEF dst, USE src);
11056   ins_cost(100);
11057   format %{ "movd    $dst,$src\t# MoveL2D" %}
11058   ins_encode %{
11059      __ movdq($dst$$XMMRegister, $src$$Register);
11060   %}
11061   ins_pipe( pipe_slow );
11062 %}
11063 
11064 // Fast clearing of an array
11065 // Small ClearArray non-AVX512.
11066 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11067                   Universe dummy, rFlagsReg cr)
11068 %{
11069   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11070   match(Set dummy (ClearArray cnt base));
11071   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11072 
11073   format %{ $$template
11074     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11075     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11076     $$emit$$"jg      LARGE\n\t"
11077     $$emit$$"dec     rcx\n\t"
11078     $$emit$$"js      DONE\t# Zero length\n\t"
11079     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11080     $$emit$$"dec     rcx\n\t"
11081     $$emit$$"jge     LOOP\n\t"
11082     $$emit$$"jmp     DONE\n\t"
11083     $$emit$$"# LARGE:\n\t"
11084     if (UseFastStosb) {
11085        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11086        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11087     } else if (UseXMMForObjInit) {
11088        $$emit$$"mov     rdi,rax\n\t"
11089        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11090        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11091        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11092        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11093        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11094        $$emit$$"add     0x40,rax\n\t"
11095        $$emit$$"# L_zero_64_bytes:\n\t"
11096        $$emit$$"sub     0x8,rcx\n\t"
11097        $$emit$$"jge     L_loop\n\t"
11098        $$emit$$"add     0x4,rcx\n\t"
11099        $$emit$$"jl      L_tail\n\t"
11100        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11101        $$emit$$"add     0x20,rax\n\t"
11102        $$emit$$"sub     0x4,rcx\n\t"
11103        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11104        $$emit$$"add     0x4,rcx\n\t"
11105        $$emit$$"jle     L_end\n\t"
11106        $$emit$$"dec     rcx\n\t"
11107        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11108        $$emit$$"vmovq   xmm0,(rax)\n\t"
11109        $$emit$$"add     0x8,rax\n\t"
11110        $$emit$$"dec     rcx\n\t"
11111        $$emit$$"jge     L_sloop\n\t"
11112        $$emit$$"# L_end:\n\t"
11113     } else {
11114        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11115     }
11116     $$emit$$"# DONE"
11117   %}
11118   ins_encode %{
11119     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11120                  $tmp$$XMMRegister, false, knoreg);
11121   %}
11122   ins_pipe(pipe_slow);
11123 %}
11124 
11125 // Small ClearArray AVX512 non-constant length.
11126 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11127                        Universe dummy, rFlagsReg cr)
11128 %{
11129   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11130   match(Set dummy (ClearArray cnt base));
11131   ins_cost(125);
11132   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11133 
11134   format %{ $$template
11135     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11136     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11137     $$emit$$"jg      LARGE\n\t"
11138     $$emit$$"dec     rcx\n\t"
11139     $$emit$$"js      DONE\t# Zero length\n\t"
11140     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11141     $$emit$$"dec     rcx\n\t"
11142     $$emit$$"jge     LOOP\n\t"
11143     $$emit$$"jmp     DONE\n\t"
11144     $$emit$$"# LARGE:\n\t"
11145     if (UseFastStosb) {
11146        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11147        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11148     } else if (UseXMMForObjInit) {
11149        $$emit$$"mov     rdi,rax\n\t"
11150        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11151        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11152        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11153        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11154        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11155        $$emit$$"add     0x40,rax\n\t"
11156        $$emit$$"# L_zero_64_bytes:\n\t"
11157        $$emit$$"sub     0x8,rcx\n\t"
11158        $$emit$$"jge     L_loop\n\t"
11159        $$emit$$"add     0x4,rcx\n\t"
11160        $$emit$$"jl      L_tail\n\t"
11161        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11162        $$emit$$"add     0x20,rax\n\t"
11163        $$emit$$"sub     0x4,rcx\n\t"
11164        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11165        $$emit$$"add     0x4,rcx\n\t"
11166        $$emit$$"jle     L_end\n\t"
11167        $$emit$$"dec     rcx\n\t"
11168        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11169        $$emit$$"vmovq   xmm0,(rax)\n\t"
11170        $$emit$$"add     0x8,rax\n\t"
11171        $$emit$$"dec     rcx\n\t"
11172        $$emit$$"jge     L_sloop\n\t"
11173        $$emit$$"# L_end:\n\t"
11174     } else {
11175        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11176     }
11177     $$emit$$"# DONE"
11178   %}
11179   ins_encode %{
11180     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11181                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11182   %}
11183   ins_pipe(pipe_slow);
11184 %}
11185 
11186 // Large ClearArray non-AVX512.
11187 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11188                         Universe dummy, rFlagsReg cr)
11189 %{
11190   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11191   match(Set dummy (ClearArray cnt base));
11192   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11193 
11194   format %{ $$template
11195     if (UseFastStosb) {
11196        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11197        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11198        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11199     } else if (UseXMMForObjInit) {
11200        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11201        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11202        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11203        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11204        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11205        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11206        $$emit$$"add     0x40,rax\n\t"
11207        $$emit$$"# L_zero_64_bytes:\n\t"
11208        $$emit$$"sub     0x8,rcx\n\t"
11209        $$emit$$"jge     L_loop\n\t"
11210        $$emit$$"add     0x4,rcx\n\t"
11211        $$emit$$"jl      L_tail\n\t"
11212        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11213        $$emit$$"add     0x20,rax\n\t"
11214        $$emit$$"sub     0x4,rcx\n\t"
11215        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11216        $$emit$$"add     0x4,rcx\n\t"
11217        $$emit$$"jle     L_end\n\t"
11218        $$emit$$"dec     rcx\n\t"
11219        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11220        $$emit$$"vmovq   xmm0,(rax)\n\t"
11221        $$emit$$"add     0x8,rax\n\t"
11222        $$emit$$"dec     rcx\n\t"
11223        $$emit$$"jge     L_sloop\n\t"
11224        $$emit$$"# L_end:\n\t"
11225     } else {
11226        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11227        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11228     }
11229   %}
11230   ins_encode %{
11231     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11232                  $tmp$$XMMRegister, true, knoreg);
11233   %}
11234   ins_pipe(pipe_slow);
11235 %}
11236 
11237 // Large ClearArray AVX512.
11238 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11239                              Universe dummy, rFlagsReg cr)
11240 %{
11241   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11242   match(Set dummy (ClearArray cnt base));
11243   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11244 
11245   format %{ $$template
11246     if (UseFastStosb) {
11247        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11248        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11249        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11250     } else if (UseXMMForObjInit) {
11251        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11252        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11253        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11254        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11255        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11256        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11257        $$emit$$"add     0x40,rax\n\t"
11258        $$emit$$"# L_zero_64_bytes:\n\t"
11259        $$emit$$"sub     0x8,rcx\n\t"
11260        $$emit$$"jge     L_loop\n\t"
11261        $$emit$$"add     0x4,rcx\n\t"
11262        $$emit$$"jl      L_tail\n\t"
11263        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11264        $$emit$$"add     0x20,rax\n\t"
11265        $$emit$$"sub     0x4,rcx\n\t"
11266        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11267        $$emit$$"add     0x4,rcx\n\t"
11268        $$emit$$"jle     L_end\n\t"
11269        $$emit$$"dec     rcx\n\t"
11270        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11271        $$emit$$"vmovq   xmm0,(rax)\n\t"
11272        $$emit$$"add     0x8,rax\n\t"
11273        $$emit$$"dec     rcx\n\t"
11274        $$emit$$"jge     L_sloop\n\t"
11275        $$emit$$"# L_end:\n\t"
11276     } else {
11277        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11278        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11279     }
11280   %}
11281   ins_encode %{
11282     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11283                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11284   %}
11285   ins_pipe(pipe_slow);
11286 %}
11287 
11288 // Small ClearArray AVX512 constant length.
11289 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11290 %{
11291   predicate(!((ClearArrayNode*)n)->is_large() &&
11292               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11293   match(Set dummy (ClearArray cnt base));
11294   ins_cost(100);
11295   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11296   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11297   ins_encode %{
11298    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11299   %}
11300   ins_pipe(pipe_slow);
11301 %}
11302 
11303 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11304                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11305 %{
11306   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11307   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11308   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11309 
11310   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11311   ins_encode %{
11312     __ string_compare($str1$$Register, $str2$$Register,
11313                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11314                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11315   %}
11316   ins_pipe( pipe_slow );
11317 %}
11318 
11319 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11320                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11321 %{
11322   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11323   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11324   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11325 
11326   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11327   ins_encode %{
11328     __ string_compare($str1$$Register, $str2$$Register,
11329                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11330                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11331   %}
11332   ins_pipe( pipe_slow );
11333 %}
11334 
11335 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11336                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11337 %{
11338   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11339   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11340   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11341 
11342   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11343   ins_encode %{
11344     __ string_compare($str1$$Register, $str2$$Register,
11345                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11346                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11347   %}
11348   ins_pipe( pipe_slow );
11349 %}
11350 
11351 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11352                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11353 %{
11354   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11355   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11356   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11357 
11358   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11359   ins_encode %{
11360     __ string_compare($str1$$Register, $str2$$Register,
11361                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11362                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11363   %}
11364   ins_pipe( pipe_slow );
11365 %}
11366 
11367 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11368                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11369 %{
11370   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11371   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11372   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11373 
11374   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11375   ins_encode %{
11376     __ string_compare($str1$$Register, $str2$$Register,
11377                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11378                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11379   %}
11380   ins_pipe( pipe_slow );
11381 %}
11382 
11383 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11384                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11385 %{
11386   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11387   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11388   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11389 
11390   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11391   ins_encode %{
11392     __ string_compare($str1$$Register, $str2$$Register,
11393                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11394                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11395   %}
11396   ins_pipe( pipe_slow );
11397 %}
11398 
11399 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11400                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11401 %{
11402   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11403   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11404   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11405 
11406   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11407   ins_encode %{
11408     __ string_compare($str2$$Register, $str1$$Register,
11409                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11410                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11411   %}
11412   ins_pipe( pipe_slow );
11413 %}
11414 
11415 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11416                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11417 %{
11418   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11419   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11420   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11421 
11422   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11423   ins_encode %{
11424     __ string_compare($str2$$Register, $str1$$Register,
11425                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11426                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11427   %}
11428   ins_pipe( pipe_slow );
11429 %}
11430 
11431 // fast search of substring with known size.
11432 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11433                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11434 %{
11435   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11436   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11437   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11438 
11439   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11440   ins_encode %{
11441     int icnt2 = (int)$int_cnt2$$constant;
11442     if (icnt2 >= 16) {
11443       // IndexOf for constant substrings with size >= 16 elements
11444       // which don't need to be loaded through stack.
11445       __ string_indexofC8($str1$$Register, $str2$$Register,
11446                           $cnt1$$Register, $cnt2$$Register,
11447                           icnt2, $result$$Register,
11448                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11449     } else {
11450       // Small strings are loaded through stack if they cross page boundary.
11451       __ string_indexof($str1$$Register, $str2$$Register,
11452                         $cnt1$$Register, $cnt2$$Register,
11453                         icnt2, $result$$Register,
11454                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11455     }
11456   %}
11457   ins_pipe( pipe_slow );
11458 %}
11459 
11460 // fast search of substring with known size.
11461 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11462                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11463 %{
11464   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11465   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11466   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11467 
11468   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11469   ins_encode %{
11470     int icnt2 = (int)$int_cnt2$$constant;
11471     if (icnt2 >= 8) {
11472       // IndexOf for constant substrings with size >= 8 elements
11473       // which don't need to be loaded through stack.
11474       __ string_indexofC8($str1$$Register, $str2$$Register,
11475                           $cnt1$$Register, $cnt2$$Register,
11476                           icnt2, $result$$Register,
11477                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11478     } else {
11479       // Small strings are loaded through stack if they cross page boundary.
11480       __ string_indexof($str1$$Register, $str2$$Register,
11481                         $cnt1$$Register, $cnt2$$Register,
11482                         icnt2, $result$$Register,
11483                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11484     }
11485   %}
11486   ins_pipe( pipe_slow );
11487 %}
11488 
11489 // fast search of substring with known size.
11490 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11491                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11492 %{
11493   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11494   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11495   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11496 
11497   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11498   ins_encode %{
11499     int icnt2 = (int)$int_cnt2$$constant;
11500     if (icnt2 >= 8) {
11501       // IndexOf for constant substrings with size >= 8 elements
11502       // which don't need to be loaded through stack.
11503       __ string_indexofC8($str1$$Register, $str2$$Register,
11504                           $cnt1$$Register, $cnt2$$Register,
11505                           icnt2, $result$$Register,
11506                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11507     } else {
11508       // Small strings are loaded through stack if they cross page boundary.
11509       __ string_indexof($str1$$Register, $str2$$Register,
11510                         $cnt1$$Register, $cnt2$$Register,
11511                         icnt2, $result$$Register,
11512                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11513     }
11514   %}
11515   ins_pipe( pipe_slow );
11516 %}
11517 
11518 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11519                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11520 %{
11521   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11522   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11523   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11524 
11525   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11526   ins_encode %{
11527     __ string_indexof($str1$$Register, $str2$$Register,
11528                       $cnt1$$Register, $cnt2$$Register,
11529                       (-1), $result$$Register,
11530                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11531   %}
11532   ins_pipe( pipe_slow );
11533 %}
11534 
11535 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11536                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11537 %{
11538   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11539   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11540   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11541 
11542   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11543   ins_encode %{
11544     __ string_indexof($str1$$Register, $str2$$Register,
11545                       $cnt1$$Register, $cnt2$$Register,
11546                       (-1), $result$$Register,
11547                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11548   %}
11549   ins_pipe( pipe_slow );
11550 %}
11551 
11552 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11553                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11554 %{
11555   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11556   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11557   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11558 
11559   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11560   ins_encode %{
11561     __ string_indexof($str1$$Register, $str2$$Register,
11562                       $cnt1$$Register, $cnt2$$Register,
11563                       (-1), $result$$Register,
11564                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11565   %}
11566   ins_pipe( pipe_slow );
11567 %}
11568 
11569 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11570                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11571 %{
11572   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11573   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11574   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11575   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11576   ins_encode %{
11577     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11578                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11579   %}
11580   ins_pipe( pipe_slow );
11581 %}
11582 
11583 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11584                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11585 %{
11586   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11587   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11588   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11589   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11590   ins_encode %{
11591     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11592                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11593   %}
11594   ins_pipe( pipe_slow );
11595 %}
11596 
11597 // fast string equals
11598 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11599                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11600 %{
11601   predicate(!VM_Version::supports_avx512vlbw());
11602   match(Set result (StrEquals (Binary str1 str2) cnt));
11603   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11604 
11605   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11606   ins_encode %{
11607     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11608                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11609                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11610   %}
11611   ins_pipe( pipe_slow );
11612 %}
11613 
11614 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11615                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11616 %{
11617   predicate(VM_Version::supports_avx512vlbw());
11618   match(Set result (StrEquals (Binary str1 str2) cnt));
11619   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11620 
11621   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11622   ins_encode %{
11623     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11624                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11625                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11626   %}
11627   ins_pipe( pipe_slow );
11628 %}
11629 
11630 // fast array equals
11631 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11632                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11633 %{
11634   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11635   match(Set result (AryEq ary1 ary2));
11636   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11637 
11638   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11639   ins_encode %{
11640     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11641                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11642                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11643   %}
11644   ins_pipe( pipe_slow );
11645 %}
11646 
11647 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11648                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11649 %{
11650   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11651   match(Set result (AryEq ary1 ary2));
11652   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11653 
11654   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11655   ins_encode %{
11656     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11657                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11658                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11659   %}
11660   ins_pipe( pipe_slow );
11661 %}
11662 
11663 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11664                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11665 %{
11666   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11667   match(Set result (AryEq ary1 ary2));
11668   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11669 
11670   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11671   ins_encode %{
11672     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11673                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11674                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11675   %}
11676   ins_pipe( pipe_slow );
11677 %}
11678 
11679 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11680                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11681 %{
11682   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11683   match(Set result (AryEq ary1 ary2));
11684   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11685 
11686   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11687   ins_encode %{
11688     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11689                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11690                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11691   %}
11692   ins_pipe( pipe_slow );
11693 %}
11694 
11695 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11696                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11697 %{
11698   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11699   match(Set result (HasNegatives ary1 len));
11700   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11701 
11702   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11703   ins_encode %{
11704     __ has_negatives($ary1$$Register, $len$$Register,
11705                      $result$$Register, $tmp3$$Register,
11706                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11707   %}
11708   ins_pipe( pipe_slow );
11709 %}
11710 
11711 instruct has_negatives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11712                             legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11713 %{
11714   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11715   match(Set result (HasNegatives ary1 len));
11716   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11717 
11718   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11719   ins_encode %{
11720     __ has_negatives($ary1$$Register, $len$$Register,
11721                      $result$$Register, $tmp3$$Register,
11722                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11723   %}
11724   ins_pipe( pipe_slow );
11725 %}
11726 
11727 // fast char[] to byte[] compression
11728 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11729                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11730   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11731   match(Set result (StrCompressedCopy src (Binary dst len)));
11732   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11733          USE_KILL len, KILL tmp5, KILL cr);
11734 
11735   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11736   ins_encode %{
11737     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11738                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11739                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11740                            knoreg, knoreg);
11741   %}
11742   ins_pipe( pipe_slow );
11743 %}
11744 
11745 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11746                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11747   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11748   match(Set result (StrCompressedCopy src (Binary dst len)));
11749   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11750          USE_KILL len, KILL tmp5, KILL cr);
11751 
11752   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11753   ins_encode %{
11754     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11755                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11756                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11757                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11758   %}
11759   ins_pipe( pipe_slow );
11760 %}
11761 // fast byte[] to char[] inflation
11762 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11763                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11764   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11765   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11766   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11767 
11768   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11769   ins_encode %{
11770     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11771                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11772   %}
11773   ins_pipe( pipe_slow );
11774 %}
11775 
11776 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11777                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11778   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11779   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11780   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11781 
11782   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11783   ins_encode %{
11784     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11785                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11786   %}
11787   ins_pipe( pipe_slow );
11788 %}
11789 
11790 // encode char[] to byte[] in ISO_8859_1
11791 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11792                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11793                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11794   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11795   match(Set result (EncodeISOArray src (Binary dst len)));
11796   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11797 
11798   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11799   ins_encode %{
11800     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11801                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11802                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11803   %}
11804   ins_pipe( pipe_slow );
11805 %}
11806 
11807 // encode char[] to byte[] in ASCII
11808 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11809                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11810                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11811   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11812   match(Set result (EncodeISOArray src (Binary dst len)));
11813   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11814 
11815   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11816   ins_encode %{
11817     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11818                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11819                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11820   %}
11821   ins_pipe( pipe_slow );
11822 %}
11823 
11824 //----------Overflow Math Instructions-----------------------------------------
11825 
11826 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11827 %{
11828   match(Set cr (OverflowAddI op1 op2));
11829   effect(DEF cr, USE_KILL op1, USE op2);
11830 
11831   format %{ "addl    $op1, $op2\t# overflow check int" %}
11832 
11833   ins_encode %{
11834     __ addl($op1$$Register, $op2$$Register);
11835   %}
11836   ins_pipe(ialu_reg_reg);
11837 %}
11838 
11839 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11840 %{
11841   match(Set cr (OverflowAddI op1 op2));
11842   effect(DEF cr, USE_KILL op1, USE op2);
11843 
11844   format %{ "addl    $op1, $op2\t# overflow check int" %}
11845 
11846   ins_encode %{
11847     __ addl($op1$$Register, $op2$$constant);
11848   %}
11849   ins_pipe(ialu_reg_reg);
11850 %}
11851 
11852 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11853 %{
11854   match(Set cr (OverflowAddL op1 op2));
11855   effect(DEF cr, USE_KILL op1, USE op2);
11856 
11857   format %{ "addq    $op1, $op2\t# overflow check long" %}
11858   ins_encode %{
11859     __ addq($op1$$Register, $op2$$Register);
11860   %}
11861   ins_pipe(ialu_reg_reg);
11862 %}
11863 
11864 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11865 %{
11866   match(Set cr (OverflowAddL op1 op2));
11867   effect(DEF cr, USE_KILL op1, USE op2);
11868 
11869   format %{ "addq    $op1, $op2\t# overflow check long" %}
11870   ins_encode %{
11871     __ addq($op1$$Register, $op2$$constant);
11872   %}
11873   ins_pipe(ialu_reg_reg);
11874 %}
11875 
11876 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11877 %{
11878   match(Set cr (OverflowSubI op1 op2));
11879 
11880   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11881   ins_encode %{
11882     __ cmpl($op1$$Register, $op2$$Register);
11883   %}
11884   ins_pipe(ialu_reg_reg);
11885 %}
11886 
11887 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11888 %{
11889   match(Set cr (OverflowSubI op1 op2));
11890 
11891   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11892   ins_encode %{
11893     __ cmpl($op1$$Register, $op2$$constant);
11894   %}
11895   ins_pipe(ialu_reg_reg);
11896 %}
11897 
11898 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11899 %{
11900   match(Set cr (OverflowSubL op1 op2));
11901 
11902   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11903   ins_encode %{
11904     __ cmpq($op1$$Register, $op2$$Register);
11905   %}
11906   ins_pipe(ialu_reg_reg);
11907 %}
11908 
11909 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11910 %{
11911   match(Set cr (OverflowSubL op1 op2));
11912 
11913   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11914   ins_encode %{
11915     __ cmpq($op1$$Register, $op2$$constant);
11916   %}
11917   ins_pipe(ialu_reg_reg);
11918 %}
11919 
11920 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11921 %{
11922   match(Set cr (OverflowSubI zero op2));
11923   effect(DEF cr, USE_KILL op2);
11924 
11925   format %{ "negl    $op2\t# overflow check int" %}
11926   ins_encode %{
11927     __ negl($op2$$Register);
11928   %}
11929   ins_pipe(ialu_reg_reg);
11930 %}
11931 
11932 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11933 %{
11934   match(Set cr (OverflowSubL zero op2));
11935   effect(DEF cr, USE_KILL op2);
11936 
11937   format %{ "negq    $op2\t# overflow check long" %}
11938   ins_encode %{
11939     __ negq($op2$$Register);
11940   %}
11941   ins_pipe(ialu_reg_reg);
11942 %}
11943 
11944 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11945 %{
11946   match(Set cr (OverflowMulI op1 op2));
11947   effect(DEF cr, USE_KILL op1, USE op2);
11948 
11949   format %{ "imull    $op1, $op2\t# overflow check int" %}
11950   ins_encode %{
11951     __ imull($op1$$Register, $op2$$Register);
11952   %}
11953   ins_pipe(ialu_reg_reg_alu0);
11954 %}
11955 
11956 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11957 %{
11958   match(Set cr (OverflowMulI op1 op2));
11959   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11960 
11961   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11962   ins_encode %{
11963     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11964   %}
11965   ins_pipe(ialu_reg_reg_alu0);
11966 %}
11967 
11968 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11969 %{
11970   match(Set cr (OverflowMulL op1 op2));
11971   effect(DEF cr, USE_KILL op1, USE op2);
11972 
11973   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11974   ins_encode %{
11975     __ imulq($op1$$Register, $op2$$Register);
11976   %}
11977   ins_pipe(ialu_reg_reg_alu0);
11978 %}
11979 
11980 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11981 %{
11982   match(Set cr (OverflowMulL op1 op2));
11983   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11984 
11985   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11986   ins_encode %{
11987     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11988   %}
11989   ins_pipe(ialu_reg_reg_alu0);
11990 %}
11991 
11992 
11993 //----------Control Flow Instructions------------------------------------------
11994 // Signed compare Instructions
11995 
11996 // XXX more variants!!
11997 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11998 %{
11999   match(Set cr (CmpI op1 op2));
12000   effect(DEF cr, USE op1, USE op2);
12001 
12002   format %{ "cmpl    $op1, $op2" %}
12003   ins_encode %{
12004     __ cmpl($op1$$Register, $op2$$Register);
12005   %}
12006   ins_pipe(ialu_cr_reg_reg);
12007 %}
12008 
12009 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12010 %{
12011   match(Set cr (CmpI op1 op2));
12012 
12013   format %{ "cmpl    $op1, $op2" %}
12014   ins_encode %{
12015     __ cmpl($op1$$Register, $op2$$constant);
12016   %}
12017   ins_pipe(ialu_cr_reg_imm);
12018 %}
12019 
12020 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12021 %{
12022   match(Set cr (CmpI op1 (LoadI op2)));
12023 
12024   ins_cost(500); // XXX
12025   format %{ "cmpl    $op1, $op2" %}
12026   ins_encode %{
12027     __ cmpl($op1$$Register, $op2$$Address);
12028   %}
12029   ins_pipe(ialu_cr_reg_mem);
12030 %}
12031 
12032 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12033 %{
12034   match(Set cr (CmpI src zero));
12035 
12036   format %{ "testl   $src, $src" %}
12037   ins_encode %{
12038     __ testl($src$$Register, $src$$Register);
12039   %}
12040   ins_pipe(ialu_cr_reg_imm);
12041 %}
12042 
12043 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12044 %{
12045   match(Set cr (CmpI (AndI src con) zero));
12046 
12047   format %{ "testl   $src, $con" %}
12048   ins_encode %{
12049     __ testl($src$$Register, $con$$constant);
12050   %}
12051   ins_pipe(ialu_cr_reg_imm);
12052 %}
12053 
12054 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12055 %{
12056   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12057 
12058   format %{ "testl   $src, $mem" %}
12059   ins_encode %{
12060     __ testl($src$$Register, $mem$$Address);
12061   %}
12062   ins_pipe(ialu_cr_reg_mem);
12063 %}
12064 
12065 // Unsigned compare Instructions; really, same as signed except they
12066 // produce an rFlagsRegU instead of rFlagsReg.
12067 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12068 %{
12069   match(Set cr (CmpU op1 op2));
12070 
12071   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12072   ins_encode %{
12073     __ cmpl($op1$$Register, $op2$$Register);
12074   %}
12075   ins_pipe(ialu_cr_reg_reg);
12076 %}
12077 
12078 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12079 %{
12080   match(Set cr (CmpU op1 op2));
12081 
12082   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12083   ins_encode %{
12084     __ cmpl($op1$$Register, $op2$$constant);
12085   %}
12086   ins_pipe(ialu_cr_reg_imm);
12087 %}
12088 
12089 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12090 %{
12091   match(Set cr (CmpU op1 (LoadI op2)));
12092 
12093   ins_cost(500); // XXX
12094   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12095   ins_encode %{
12096     __ cmpl($op1$$Register, $op2$$Address);
12097   %}
12098   ins_pipe(ialu_cr_reg_mem);
12099 %}
12100 
12101 // // // Cisc-spilled version of cmpU_rReg
12102 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12103 // //%{
12104 // //  match(Set cr (CmpU (LoadI op1) op2));
12105 // //
12106 // //  format %{ "CMPu   $op1,$op2" %}
12107 // //  ins_cost(500);
12108 // //  opcode(0x39);  /* Opcode 39 /r */
12109 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12110 // //%}
12111 
12112 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12113 %{
12114   match(Set cr (CmpU src zero));
12115 
12116   format %{ "testl   $src, $src\t# unsigned" %}
12117   ins_encode %{
12118     __ testl($src$$Register, $src$$Register);
12119   %}
12120   ins_pipe(ialu_cr_reg_imm);
12121 %}
12122 
12123 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12124 %{
12125   match(Set cr (CmpP op1 op2));
12126 
12127   format %{ "cmpq    $op1, $op2\t# ptr" %}
12128   ins_encode %{
12129     __ cmpq($op1$$Register, $op2$$Register);
12130   %}
12131   ins_pipe(ialu_cr_reg_reg);
12132 %}
12133 
12134 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12135 %{
12136   match(Set cr (CmpP op1 (LoadP op2)));
12137   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12138 
12139   ins_cost(500); // XXX
12140   format %{ "cmpq    $op1, $op2\t# ptr" %}
12141   ins_encode %{
12142     __ cmpq($op1$$Register, $op2$$Address);
12143   %}
12144   ins_pipe(ialu_cr_reg_mem);
12145 %}
12146 
12147 // // // Cisc-spilled version of cmpP_rReg
12148 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12149 // //%{
12150 // //  match(Set cr (CmpP (LoadP op1) op2));
12151 // //
12152 // //  format %{ "CMPu   $op1,$op2" %}
12153 // //  ins_cost(500);
12154 // //  opcode(0x39);  /* Opcode 39 /r */
12155 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12156 // //%}
12157 
12158 // XXX this is generalized by compP_rReg_mem???
12159 // Compare raw pointer (used in out-of-heap check).
12160 // Only works because non-oop pointers must be raw pointers
12161 // and raw pointers have no anti-dependencies.
12162 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12163 %{
12164   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12165             n->in(2)->as_Load()->barrier_data() == 0);
12166   match(Set cr (CmpP op1 (LoadP op2)));
12167 
12168   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12169   ins_encode %{
12170     __ cmpq($op1$$Register, $op2$$Address);
12171   %}
12172   ins_pipe(ialu_cr_reg_mem);
12173 %}
12174 
12175 // This will generate a signed flags result. This should be OK since
12176 // any compare to a zero should be eq/neq.
12177 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12178 %{
12179   match(Set cr (CmpP src zero));
12180 
12181   format %{ "testq   $src, $src\t# ptr" %}
12182   ins_encode %{
12183     __ testq($src$$Register, $src$$Register);
12184   %}
12185   ins_pipe(ialu_cr_reg_imm);
12186 %}
12187 
12188 // This will generate a signed flags result. This should be OK since
12189 // any compare to a zero should be eq/neq.
12190 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12191 %{
12192   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12193             n->in(1)->as_Load()->barrier_data() == 0);
12194   match(Set cr (CmpP (LoadP op) zero));
12195 
12196   ins_cost(500); // XXX
12197   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12198   ins_encode %{
12199     __ testq($op$$Address, 0xFFFFFFFF);
12200   %}
12201   ins_pipe(ialu_cr_reg_imm);
12202 %}
12203 
12204 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12205 %{
12206   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12207             n->in(1)->as_Load()->barrier_data() == 0);
12208   match(Set cr (CmpP (LoadP mem) zero));
12209 
12210   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12211   ins_encode %{
12212     __ cmpq(r12, $mem$$Address);
12213   %}
12214   ins_pipe(ialu_cr_reg_mem);
12215 %}
12216 
12217 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12218 %{
12219   match(Set cr (CmpN op1 op2));
12220 
12221   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12222   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12223   ins_pipe(ialu_cr_reg_reg);
12224 %}
12225 
12226 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12227 %{
12228   match(Set cr (CmpN src (LoadN mem)));
12229 
12230   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12231   ins_encode %{
12232     __ cmpl($src$$Register, $mem$$Address);
12233   %}
12234   ins_pipe(ialu_cr_reg_mem);
12235 %}
12236 
12237 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12238   match(Set cr (CmpN op1 op2));
12239 
12240   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12241   ins_encode %{
12242     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12243   %}
12244   ins_pipe(ialu_cr_reg_imm);
12245 %}
12246 
12247 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12248 %{
12249   match(Set cr (CmpN src (LoadN mem)));
12250 
12251   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12252   ins_encode %{
12253     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12254   %}
12255   ins_pipe(ialu_cr_reg_mem);
12256 %}
12257 
12258 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12259   match(Set cr (CmpN op1 op2));
12260 
12261   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12262   ins_encode %{
12263     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12264   %}
12265   ins_pipe(ialu_cr_reg_imm);
12266 %}
12267 
12268 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12269 %{
12270   match(Set cr (CmpN src (LoadNKlass mem)));
12271 
12272   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12273   ins_encode %{
12274     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12275   %}
12276   ins_pipe(ialu_cr_reg_mem);
12277 %}
12278 
12279 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12280   match(Set cr (CmpN src zero));
12281 
12282   format %{ "testl   $src, $src\t# compressed ptr" %}
12283   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12284   ins_pipe(ialu_cr_reg_imm);
12285 %}
12286 
12287 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12288 %{
12289   predicate(CompressedOops::base() != NULL);
12290   match(Set cr (CmpN (LoadN mem) zero));
12291 
12292   ins_cost(500); // XXX
12293   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12294   ins_encode %{
12295     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12296   %}
12297   ins_pipe(ialu_cr_reg_mem);
12298 %}
12299 
12300 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12301 %{
12302   predicate(CompressedOops::base() == NULL);
12303   match(Set cr (CmpN (LoadN mem) zero));
12304 
12305   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12306   ins_encode %{
12307     __ cmpl(r12, $mem$$Address);
12308   %}
12309   ins_pipe(ialu_cr_reg_mem);
12310 %}
12311 
12312 // Yanked all unsigned pointer compare operations.
12313 // Pointer compares are done with CmpP which is already unsigned.
12314 
12315 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12316 %{
12317   match(Set cr (CmpL op1 op2));
12318 
12319   format %{ "cmpq    $op1, $op2" %}
12320   ins_encode %{
12321     __ cmpq($op1$$Register, $op2$$Register);
12322   %}
12323   ins_pipe(ialu_cr_reg_reg);
12324 %}
12325 
12326 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12327 %{
12328   match(Set cr (CmpL op1 op2));
12329 
12330   format %{ "cmpq    $op1, $op2" %}
12331   ins_encode %{
12332     __ cmpq($op1$$Register, $op2$$constant);
12333   %}
12334   ins_pipe(ialu_cr_reg_imm);
12335 %}
12336 
12337 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12338 %{
12339   match(Set cr (CmpL op1 (LoadL op2)));
12340 
12341   format %{ "cmpq    $op1, $op2" %}
12342   ins_encode %{
12343     __ cmpq($op1$$Register, $op2$$Address);
12344   %}
12345   ins_pipe(ialu_cr_reg_mem);
12346 %}
12347 
12348 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12349 %{
12350   match(Set cr (CmpL src zero));
12351 
12352   format %{ "testq   $src, $src" %}
12353   ins_encode %{
12354     __ testq($src$$Register, $src$$Register);
12355   %}
12356   ins_pipe(ialu_cr_reg_imm);
12357 %}
12358 
12359 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12360 %{
12361   match(Set cr (CmpL (AndL src con) zero));
12362 
12363   format %{ "testq   $src, $con\t# long" %}
12364   ins_encode %{
12365     __ testq($src$$Register, $con$$constant);
12366   %}
12367   ins_pipe(ialu_cr_reg_imm);
12368 %}
12369 
12370 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12371 %{
12372   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12373 
12374   format %{ "testq   $src, $mem" %}
12375   ins_encode %{
12376     __ testq($src$$Register, $mem$$Address);
12377   %}
12378   ins_pipe(ialu_cr_reg_mem);
12379 %}
12380 
12381 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12382 %{
12383   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12384 
12385   format %{ "testq   $src, $mem" %}
12386   ins_encode %{
12387     __ testq($src$$Register, $mem$$Address);
12388   %}
12389   ins_pipe(ialu_cr_reg_mem);
12390 %}
12391 
12392 // Manifest a CmpL result in an integer register.  Very painful.
12393 // This is the test to avoid.
12394 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12395 %{
12396   match(Set dst (CmpL3 src1 src2));
12397   effect(KILL flags);
12398 
12399   ins_cost(275); // XXX
12400   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12401             "movl    $dst, -1\n\t"
12402             "jl,s    done\n\t"
12403             "setne   $dst\n\t"
12404             "movzbl  $dst, $dst\n\t"
12405     "done:" %}
12406   ins_encode %{
12407     Label done;
12408     __ cmpq($src1$$Register, $src2$$Register);
12409     __ movl($dst$$Register, -1);
12410     __ jccb(Assembler::less, done);
12411     __ setne($dst$$Register);
12412     __ movzbl($dst$$Register, $dst$$Register);
12413     __ bind(done);
12414   %}
12415   ins_pipe(pipe_slow);
12416 %}
12417 
12418 // Unsigned long compare Instructions; really, same as signed long except they
12419 // produce an rFlagsRegU instead of rFlagsReg.
12420 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12421 %{
12422   match(Set cr (CmpUL op1 op2));
12423 
12424   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12425   ins_encode %{
12426     __ cmpq($op1$$Register, $op2$$Register);
12427   %}
12428   ins_pipe(ialu_cr_reg_reg);
12429 %}
12430 
12431 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12432 %{
12433   match(Set cr (CmpUL op1 op2));
12434 
12435   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12436   ins_encode %{
12437     __ cmpq($op1$$Register, $op2$$constant);
12438   %}
12439   ins_pipe(ialu_cr_reg_imm);
12440 %}
12441 
12442 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12443 %{
12444   match(Set cr (CmpUL op1 (LoadL op2)));
12445 
12446   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12447   ins_encode %{
12448     __ cmpq($op1$$Register, $op2$$Address);
12449   %}
12450   ins_pipe(ialu_cr_reg_mem);
12451 %}
12452 
12453 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12454 %{
12455   match(Set cr (CmpUL src zero));
12456 
12457   format %{ "testq   $src, $src\t# unsigned" %}
12458   ins_encode %{
12459     __ testq($src$$Register, $src$$Register);
12460   %}
12461   ins_pipe(ialu_cr_reg_imm);
12462 %}
12463 
12464 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12465 %{
12466   match(Set cr (CmpI (LoadB mem) imm));
12467 
12468   ins_cost(125);
12469   format %{ "cmpb    $mem, $imm" %}
12470   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12471   ins_pipe(ialu_cr_reg_mem);
12472 %}
12473 
12474 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12475 %{
12476   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12477 
12478   ins_cost(125);
12479   format %{ "testb   $mem, $imm\t# ubyte" %}
12480   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12481   ins_pipe(ialu_cr_reg_mem);
12482 %}
12483 
12484 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12485 %{
12486   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12487 
12488   ins_cost(125);
12489   format %{ "testb   $mem, $imm\t# byte" %}
12490   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12491   ins_pipe(ialu_cr_reg_mem);
12492 %}
12493 
12494 //----------Max and Min--------------------------------------------------------
12495 // Min Instructions
12496 
12497 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12498 %{
12499   effect(USE_DEF dst, USE src, USE cr);
12500 
12501   format %{ "cmovlgt $dst, $src\t# min" %}
12502   ins_encode %{
12503     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12504   %}
12505   ins_pipe(pipe_cmov_reg);
12506 %}
12507 
12508 
12509 instruct minI_rReg(rRegI dst, rRegI src)
12510 %{
12511   match(Set dst (MinI dst src));
12512 
12513   ins_cost(200);
12514   expand %{
12515     rFlagsReg cr;
12516     compI_rReg(cr, dst, src);
12517     cmovI_reg_g(dst, src, cr);
12518   %}
12519 %}
12520 
12521 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12522 %{
12523   effect(USE_DEF dst, USE src, USE cr);
12524 
12525   format %{ "cmovllt $dst, $src\t# max" %}
12526   ins_encode %{
12527     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12528   %}
12529   ins_pipe(pipe_cmov_reg);
12530 %}
12531 
12532 
12533 instruct maxI_rReg(rRegI dst, rRegI src)
12534 %{
12535   match(Set dst (MaxI dst src));
12536 
12537   ins_cost(200);
12538   expand %{
12539     rFlagsReg cr;
12540     compI_rReg(cr, dst, src);
12541     cmovI_reg_l(dst, src, cr);
12542   %}
12543 %}
12544 
12545 // ============================================================================
12546 // Branch Instructions
12547 
12548 // Jump Direct - Label defines a relative address from JMP+1
12549 instruct jmpDir(label labl)
12550 %{
12551   match(Goto);
12552   effect(USE labl);
12553 
12554   ins_cost(300);
12555   format %{ "jmp     $labl" %}
12556   size(5);
12557   ins_encode %{
12558     Label* L = $labl$$label;
12559     __ jmp(*L, false); // Always long jump
12560   %}
12561   ins_pipe(pipe_jmp);
12562 %}
12563 
12564 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12565 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12566 %{
12567   match(If cop cr);
12568   effect(USE labl);
12569 
12570   ins_cost(300);
12571   format %{ "j$cop     $labl" %}
12572   size(6);
12573   ins_encode %{
12574     Label* L = $labl$$label;
12575     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12576   %}
12577   ins_pipe(pipe_jcc);
12578 %}
12579 
12580 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12581 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12582 %{
12583   predicate(!n->has_vector_mask_set());
12584   match(CountedLoopEnd cop cr);
12585   effect(USE labl);
12586 
12587   ins_cost(300);
12588   format %{ "j$cop     $labl\t# loop end" %}
12589   size(6);
12590   ins_encode %{
12591     Label* L = $labl$$label;
12592     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12593   %}
12594   ins_pipe(pipe_jcc);
12595 %}
12596 
12597 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12598 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12599   predicate(!n->has_vector_mask_set());
12600   match(CountedLoopEnd cop cmp);
12601   effect(USE labl);
12602 
12603   ins_cost(300);
12604   format %{ "j$cop,u   $labl\t# loop end" %}
12605   size(6);
12606   ins_encode %{
12607     Label* L = $labl$$label;
12608     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12609   %}
12610   ins_pipe(pipe_jcc);
12611 %}
12612 
12613 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12614   predicate(!n->has_vector_mask_set());
12615   match(CountedLoopEnd cop cmp);
12616   effect(USE labl);
12617 
12618   ins_cost(200);
12619   format %{ "j$cop,u   $labl\t# loop end" %}
12620   size(6);
12621   ins_encode %{
12622     Label* L = $labl$$label;
12623     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12624   %}
12625   ins_pipe(pipe_jcc);
12626 %}
12627 
12628 // mask version
12629 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12630 // Bounded mask operand used in following patten is needed for
12631 // post-loop multiversioning.
12632 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, kReg_K1 ktmp, rFlagsReg cr, label labl)
12633 %{
12634   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12635   match(CountedLoopEnd cop cr);
12636   effect(USE labl, TEMP ktmp);
12637 
12638   ins_cost(400);
12639   format %{ "j$cop     $labl\t# loop end\n\t"
12640             "restorevectmask \t# vector mask restore for loops" %}
12641   size(10);
12642   ins_encode %{
12643     Label* L = $labl$$label;
12644     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12645     __ restorevectmask($ktmp$$KRegister);
12646   %}
12647   ins_pipe(pipe_jcc);
12648 %}
12649 
12650 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12651 // Bounded mask operand used in following patten is needed for
12652 // post-loop multiversioning.
12653 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, kReg_K1 ktmp, rFlagsRegU cmp, label labl) %{
12654   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12655   match(CountedLoopEnd cop cmp);
12656   effect(USE labl, TEMP ktmp);
12657 
12658   ins_cost(400);
12659   format %{ "j$cop,u   $labl\t# loop end\n\t"
12660             "restorevectmask \t# vector mask restore for loops" %}
12661   size(10);
12662   ins_encode %{
12663     Label* L = $labl$$label;
12664     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12665     __ restorevectmask($ktmp$$KRegister);
12666   %}
12667   ins_pipe(pipe_jcc);
12668 %}
12669 
12670 // Bounded mask operand used in following patten is needed for
12671 // post-loop multiversioning.
12672 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, kReg_K1 ktmp, rFlagsRegUCF cmp, label labl) %{
12673   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12674   match(CountedLoopEnd cop cmp);
12675   effect(USE labl, TEMP ktmp);
12676 
12677   ins_cost(300);
12678   format %{ "j$cop,u   $labl\t# loop end\n\t"
12679             "restorevectmask \t# vector mask restore for loops" %}
12680   size(10);
12681   ins_encode %{
12682     Label* L = $labl$$label;
12683     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12684     __ restorevectmask($ktmp$$KRegister);
12685   %}
12686   ins_pipe(pipe_jcc);
12687 %}
12688 
12689 // Jump Direct Conditional - using unsigned comparison
12690 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12691   match(If cop cmp);
12692   effect(USE labl);
12693 
12694   ins_cost(300);
12695   format %{ "j$cop,u   $labl" %}
12696   size(6);
12697   ins_encode %{
12698     Label* L = $labl$$label;
12699     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12700   %}
12701   ins_pipe(pipe_jcc);
12702 %}
12703 
12704 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12705   match(If cop cmp);
12706   effect(USE labl);
12707 
12708   ins_cost(200);
12709   format %{ "j$cop,u   $labl" %}
12710   size(6);
12711   ins_encode %{
12712     Label* L = $labl$$label;
12713     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12714   %}
12715   ins_pipe(pipe_jcc);
12716 %}
12717 
12718 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12719   match(If cop cmp);
12720   effect(USE labl);
12721 
12722   ins_cost(200);
12723   format %{ $$template
12724     if ($cop$$cmpcode == Assembler::notEqual) {
12725       $$emit$$"jp,u    $labl\n\t"
12726       $$emit$$"j$cop,u   $labl"
12727     } else {
12728       $$emit$$"jp,u    done\n\t"
12729       $$emit$$"j$cop,u   $labl\n\t"
12730       $$emit$$"done:"
12731     }
12732   %}
12733   ins_encode %{
12734     Label* l = $labl$$label;
12735     if ($cop$$cmpcode == Assembler::notEqual) {
12736       __ jcc(Assembler::parity, *l, false);
12737       __ jcc(Assembler::notEqual, *l, false);
12738     } else if ($cop$$cmpcode == Assembler::equal) {
12739       Label done;
12740       __ jccb(Assembler::parity, done);
12741       __ jcc(Assembler::equal, *l, false);
12742       __ bind(done);
12743     } else {
12744        ShouldNotReachHere();
12745     }
12746   %}
12747   ins_pipe(pipe_jcc);
12748 %}
12749 
12750 // ============================================================================
12751 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12752 // superklass array for an instance of the superklass.  Set a hidden
12753 // internal cache on a hit (cache is checked with exposed code in
12754 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12755 // encoding ALSO sets flags.
12756 
12757 instruct partialSubtypeCheck(rdi_RegP result,
12758                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12759                              rFlagsReg cr)
12760 %{
12761   match(Set result (PartialSubtypeCheck sub super));
12762   effect(KILL rcx, KILL cr);
12763 
12764   ins_cost(1100);  // slightly larger than the next version
12765   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12766             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12767             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12768             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12769             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12770             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12771             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12772     "miss:\t" %}
12773 
12774   opcode(0x1); // Force a XOR of RDI
12775   ins_encode(enc_PartialSubtypeCheck());
12776   ins_pipe(pipe_slow);
12777 %}
12778 
12779 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12780                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12781                                      immP0 zero,
12782                                      rdi_RegP result)
12783 %{
12784   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12785   effect(KILL rcx, KILL result);
12786 
12787   ins_cost(1000);
12788   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12789             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12790             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12791             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12792             "jne,s   miss\t\t# Missed: flags nz\n\t"
12793             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12794     "miss:\t" %}
12795 
12796   opcode(0x0); // No need to XOR RDI
12797   ins_encode(enc_PartialSubtypeCheck());
12798   ins_pipe(pipe_slow);
12799 %}
12800 
12801 // ============================================================================
12802 // Branch Instructions -- short offset versions
12803 //
12804 // These instructions are used to replace jumps of a long offset (the default
12805 // match) with jumps of a shorter offset.  These instructions are all tagged
12806 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12807 // match rules in general matching.  Instead, the ADLC generates a conversion
12808 // method in the MachNode which can be used to do in-place replacement of the
12809 // long variant with the shorter variant.  The compiler will determine if a
12810 // branch can be taken by the is_short_branch_offset() predicate in the machine
12811 // specific code section of the file.
12812 
12813 // Jump Direct - Label defines a relative address from JMP+1
12814 instruct jmpDir_short(label labl) %{
12815   match(Goto);
12816   effect(USE labl);
12817 
12818   ins_cost(300);
12819   format %{ "jmp,s   $labl" %}
12820   size(2);
12821   ins_encode %{
12822     Label* L = $labl$$label;
12823     __ jmpb(*L);
12824   %}
12825   ins_pipe(pipe_jmp);
12826   ins_short_branch(1);
12827 %}
12828 
12829 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12830 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12831   match(If cop cr);
12832   effect(USE labl);
12833 
12834   ins_cost(300);
12835   format %{ "j$cop,s   $labl" %}
12836   size(2);
12837   ins_encode %{
12838     Label* L = $labl$$label;
12839     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12840   %}
12841   ins_pipe(pipe_jcc);
12842   ins_short_branch(1);
12843 %}
12844 
12845 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12846 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12847   match(CountedLoopEnd cop cr);
12848   effect(USE labl);
12849 
12850   ins_cost(300);
12851   format %{ "j$cop,s   $labl\t# loop end" %}
12852   size(2);
12853   ins_encode %{
12854     Label* L = $labl$$label;
12855     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12856   %}
12857   ins_pipe(pipe_jcc);
12858   ins_short_branch(1);
12859 %}
12860 
12861 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12862 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12863   match(CountedLoopEnd cop cmp);
12864   effect(USE labl);
12865 
12866   ins_cost(300);
12867   format %{ "j$cop,us  $labl\t# loop end" %}
12868   size(2);
12869   ins_encode %{
12870     Label* L = $labl$$label;
12871     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12872   %}
12873   ins_pipe(pipe_jcc);
12874   ins_short_branch(1);
12875 %}
12876 
12877 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12878   match(CountedLoopEnd cop cmp);
12879   effect(USE labl);
12880 
12881   ins_cost(300);
12882   format %{ "j$cop,us  $labl\t# loop end" %}
12883   size(2);
12884   ins_encode %{
12885     Label* L = $labl$$label;
12886     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12887   %}
12888   ins_pipe(pipe_jcc);
12889   ins_short_branch(1);
12890 %}
12891 
12892 // Jump Direct Conditional - using unsigned comparison
12893 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12894   match(If cop cmp);
12895   effect(USE labl);
12896 
12897   ins_cost(300);
12898   format %{ "j$cop,us  $labl" %}
12899   size(2);
12900   ins_encode %{
12901     Label* L = $labl$$label;
12902     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12903   %}
12904   ins_pipe(pipe_jcc);
12905   ins_short_branch(1);
12906 %}
12907 
12908 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12909   match(If cop cmp);
12910   effect(USE labl);
12911 
12912   ins_cost(300);
12913   format %{ "j$cop,us  $labl" %}
12914   size(2);
12915   ins_encode %{
12916     Label* L = $labl$$label;
12917     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12918   %}
12919   ins_pipe(pipe_jcc);
12920   ins_short_branch(1);
12921 %}
12922 
12923 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12924   match(If cop cmp);
12925   effect(USE labl);
12926 
12927   ins_cost(300);
12928   format %{ $$template
12929     if ($cop$$cmpcode == Assembler::notEqual) {
12930       $$emit$$"jp,u,s  $labl\n\t"
12931       $$emit$$"j$cop,u,s  $labl"
12932     } else {
12933       $$emit$$"jp,u,s  done\n\t"
12934       $$emit$$"j$cop,u,s  $labl\n\t"
12935       $$emit$$"done:"
12936     }
12937   %}
12938   size(4);
12939   ins_encode %{
12940     Label* l = $labl$$label;
12941     if ($cop$$cmpcode == Assembler::notEqual) {
12942       __ jccb(Assembler::parity, *l);
12943       __ jccb(Assembler::notEqual, *l);
12944     } else if ($cop$$cmpcode == Assembler::equal) {
12945       Label done;
12946       __ jccb(Assembler::parity, done);
12947       __ jccb(Assembler::equal, *l);
12948       __ bind(done);
12949     } else {
12950        ShouldNotReachHere();
12951     }
12952   %}
12953   ins_pipe(pipe_jcc);
12954   ins_short_branch(1);
12955 %}
12956 
12957 // ============================================================================
12958 // inlined locking and unlocking
12959 
12960 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12961   predicate(Compile::current()->use_rtm());
12962   match(Set cr (FastLock object box));
12963   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12964   ins_cost(300);
12965   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12966   ins_encode %{
12967     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12968                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12969                  _rtm_counters, _stack_rtm_counters,
12970                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12971                  true, ra_->C->profile_rtm());
12972   %}
12973   ins_pipe(pipe_slow);
12974 %}
12975 
12976 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
12977   predicate(!Compile::current()->use_rtm());
12978   match(Set cr (FastLock object box));
12979   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
12980   ins_cost(300);
12981   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12982   ins_encode %{
12983     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12984                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
12985   %}
12986   ins_pipe(pipe_slow);
12987 %}
12988 
12989 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12990   match(Set cr (FastUnlock object box));
12991   effect(TEMP tmp, USE_KILL box);
12992   ins_cost(300);
12993   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12994   ins_encode %{
12995     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12996   %}
12997   ins_pipe(pipe_slow);
12998 %}
12999 
13000 
13001 // ============================================================================
13002 // Safepoint Instructions
13003 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13004 %{
13005   match(SafePoint poll);
13006   effect(KILL cr, USE poll);
13007 
13008   format %{ "testl   rax, [$poll]\t"
13009             "# Safepoint: poll for GC" %}
13010   ins_cost(125);
13011   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13012   ins_encode %{
13013     __ relocate(relocInfo::poll_type);
13014     address pre_pc = __ pc();
13015     __ testl(rax, Address($poll$$Register, 0));
13016     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13017   %}
13018   ins_pipe(ialu_reg_mem);
13019 %}
13020 
13021 // ============================================================================
13022 // Procedure Call/Return Instructions
13023 // Call Java Static Instruction
13024 // Note: If this code changes, the corresponding ret_addr_offset() and
13025 //       compute_padding() functions will have to be adjusted.
13026 instruct CallStaticJavaDirect(method meth) %{
13027   match(CallStaticJava);
13028   effect(USE meth);
13029 
13030   ins_cost(300);
13031   format %{ "call,static " %}
13032   opcode(0xE8); /* E8 cd */
13033   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13034   ins_pipe(pipe_slow);
13035   ins_alignment(4);
13036 %}
13037 
13038 // Call Java Dynamic Instruction
13039 // Note: If this code changes, the corresponding ret_addr_offset() and
13040 //       compute_padding() functions will have to be adjusted.
13041 instruct CallDynamicJavaDirect(method meth)
13042 %{
13043   match(CallDynamicJava);
13044   effect(USE meth);
13045 
13046   ins_cost(300);
13047   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13048             "call,dynamic " %}
13049   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13050   ins_pipe(pipe_slow);
13051   ins_alignment(4);
13052 %}
13053 
13054 // Call Runtime Instruction
13055 instruct CallRuntimeDirect(method meth)
13056 %{
13057   match(CallRuntime);
13058   effect(USE meth);
13059 
13060   ins_cost(300);
13061   format %{ "call,runtime " %}
13062   ins_encode(clear_avx, Java_To_Runtime(meth));
13063   ins_pipe(pipe_slow);
13064 %}
13065 
13066 // Call runtime without safepoint
13067 instruct CallLeafDirect(method meth)
13068 %{
13069   match(CallLeaf);
13070   effect(USE meth);
13071 
13072   ins_cost(300);
13073   format %{ "call_leaf,runtime " %}
13074   ins_encode(clear_avx, Java_To_Runtime(meth));
13075   ins_pipe(pipe_slow);
13076 %}
13077 
13078 // Call runtime without safepoint and with vector arguments
13079 instruct CallLeafDirectVector(method meth)
13080 %{
13081   match(CallLeafVector);
13082   effect(USE meth);
13083 
13084   ins_cost(300);
13085   format %{ "call_leaf,vector " %}
13086   ins_encode(Java_To_Runtime(meth));
13087   ins_pipe(pipe_slow);
13088 %}
13089 
13090 //
13091 instruct CallNativeDirect(method meth)
13092 %{
13093   match(CallNative);
13094   effect(USE meth);
13095 
13096   ins_cost(300);
13097   format %{ "call_native " %}
13098   ins_encode(clear_avx, Java_To_Runtime(meth));
13099   ins_pipe(pipe_slow);
13100 %}
13101 
13102 // Call runtime without safepoint
13103 instruct CallLeafNoFPDirect(method meth)
13104 %{
13105   match(CallLeafNoFP);
13106   effect(USE meth);
13107 
13108   ins_cost(300);
13109   format %{ "call_leaf_nofp,runtime " %}
13110   ins_encode(clear_avx, Java_To_Runtime(meth));
13111   ins_pipe(pipe_slow);
13112 %}
13113 
13114 // Return Instruction
13115 // Remove the return address & jump to it.
13116 // Notice: We always emit a nop after a ret to make sure there is room
13117 // for safepoint patching
13118 instruct Ret()
13119 %{
13120   match(Return);
13121 
13122   format %{ "ret" %}
13123   ins_encode %{
13124     __ ret(0);
13125   %}
13126   ins_pipe(pipe_jmp);
13127 %}
13128 
13129 // Tail Call; Jump from runtime stub to Java code.
13130 // Also known as an 'interprocedural jump'.
13131 // Target of jump will eventually return to caller.
13132 // TailJump below removes the return address.
13133 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13134 %{
13135   match(TailCall jump_target method_ptr);
13136 
13137   ins_cost(300);
13138   format %{ "jmp     $jump_target\t# rbx holds method" %}
13139   ins_encode %{
13140     __ jmp($jump_target$$Register);
13141   %}
13142   ins_pipe(pipe_jmp);
13143 %}
13144 
13145 // Tail Jump; remove the return address; jump to target.
13146 // TailCall above leaves the return address around.
13147 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13148 %{
13149   match(TailJump jump_target ex_oop);
13150 
13151   ins_cost(300);
13152   format %{ "popq    rdx\t# pop return address\n\t"
13153             "jmp     $jump_target" %}
13154   ins_encode %{
13155     __ popq(as_Register(RDX_enc));
13156     __ jmp($jump_target$$Register);
13157   %}
13158   ins_pipe(pipe_jmp);
13159 %}
13160 
13161 // Create exception oop: created by stack-crawling runtime code.
13162 // Created exception is now available to this handler, and is setup
13163 // just prior to jumping to this handler.  No code emitted.
13164 instruct CreateException(rax_RegP ex_oop)
13165 %{
13166   match(Set ex_oop (CreateEx));
13167 
13168   size(0);
13169   // use the following format syntax
13170   format %{ "# exception oop is in rax; no code emitted" %}
13171   ins_encode();
13172   ins_pipe(empty);
13173 %}
13174 
13175 // Rethrow exception:
13176 // The exception oop will come in the first argument position.
13177 // Then JUMP (not call) to the rethrow stub code.
13178 instruct RethrowException()
13179 %{
13180   match(Rethrow);
13181 
13182   // use the following format syntax
13183   format %{ "jmp     rethrow_stub" %}
13184   ins_encode(enc_rethrow);
13185   ins_pipe(pipe_jmp);
13186 %}
13187 
13188 // ============================================================================
13189 // This name is KNOWN by the ADLC and cannot be changed.
13190 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13191 // for this guy.
13192 instruct tlsLoadP(r15_RegP dst) %{
13193   match(Set dst (ThreadLocal));
13194   effect(DEF dst);
13195 
13196   size(0);
13197   format %{ "# TLS is in R15" %}
13198   ins_encode( /*empty encoding*/ );
13199   ins_pipe(ialu_reg_reg);
13200 %}
13201 
13202 
13203 //----------PEEPHOLE RULES-----------------------------------------------------
13204 // These must follow all instruction definitions as they use the names
13205 // defined in the instructions definitions.
13206 //
13207 // peepmatch ( root_instr_name [preceding_instruction]* );
13208 //
13209 // peepconstraint %{
13210 // (instruction_number.operand_name relational_op instruction_number.operand_name
13211 //  [, ...] );
13212 // // instruction numbers are zero-based using left to right order in peepmatch
13213 //
13214 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13215 // // provide an instruction_number.operand_name for each operand that appears
13216 // // in the replacement instruction's match rule
13217 //
13218 // ---------VM FLAGS---------------------------------------------------------
13219 //
13220 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13221 //
13222 // Each peephole rule is given an identifying number starting with zero and
13223 // increasing by one in the order seen by the parser.  An individual peephole
13224 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13225 // on the command-line.
13226 //
13227 // ---------CURRENT LIMITATIONS----------------------------------------------
13228 //
13229 // Only match adjacent instructions in same basic block
13230 // Only equality constraints
13231 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13232 // Only one replacement instruction
13233 //
13234 // ---------EXAMPLE----------------------------------------------------------
13235 //
13236 // // pertinent parts of existing instructions in architecture description
13237 // instruct movI(rRegI dst, rRegI src)
13238 // %{
13239 //   match(Set dst (CopyI src));
13240 // %}
13241 //
13242 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13243 // %{
13244 //   match(Set dst (AddI dst src));
13245 //   effect(KILL cr);
13246 // %}
13247 //
13248 // // Change (inc mov) to lea
13249 // peephole %{
13250 //   // increment preceeded by register-register move
13251 //   peepmatch ( incI_rReg movI );
13252 //   // require that the destination register of the increment
13253 //   // match the destination register of the move
13254 //   peepconstraint ( 0.dst == 1.dst );
13255 //   // construct a replacement instruction that sets
13256 //   // the destination to ( move's source register + one )
13257 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13258 // %}
13259 //
13260 
13261 // Implementation no longer uses movX instructions since
13262 // machine-independent system no longer uses CopyX nodes.
13263 //
13264 // peephole
13265 // %{
13266 //   peepmatch (incI_rReg movI);
13267 //   peepconstraint (0.dst == 1.dst);
13268 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13269 // %}
13270 
13271 // peephole
13272 // %{
13273 //   peepmatch (decI_rReg movI);
13274 //   peepconstraint (0.dst == 1.dst);
13275 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13276 // %}
13277 
13278 // peephole
13279 // %{
13280 //   peepmatch (addI_rReg_imm movI);
13281 //   peepconstraint (0.dst == 1.dst);
13282 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13283 // %}
13284 
13285 // peephole
13286 // %{
13287 //   peepmatch (incL_rReg movL);
13288 //   peepconstraint (0.dst == 1.dst);
13289 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13290 // %}
13291 
13292 // peephole
13293 // %{
13294 //   peepmatch (decL_rReg movL);
13295 //   peepconstraint (0.dst == 1.dst);
13296 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13297 // %}
13298 
13299 // peephole
13300 // %{
13301 //   peepmatch (addL_rReg_imm movL);
13302 //   peepconstraint (0.dst == 1.dst);
13303 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13304 // %}
13305 
13306 // peephole
13307 // %{
13308 //   peepmatch (addP_rReg_imm movP);
13309 //   peepconstraint (0.dst == 1.dst);
13310 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13311 // %}
13312 
13313 // // Change load of spilled value to only a spill
13314 // instruct storeI(memory mem, rRegI src)
13315 // %{
13316 //   match(Set mem (StoreI mem src));
13317 // %}
13318 //
13319 // instruct loadI(rRegI dst, memory mem)
13320 // %{
13321 //   match(Set dst (LoadI mem));
13322 // %}
13323 //
13324 
13325 peephole
13326 %{
13327   peepmatch (loadI storeI);
13328   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13329   peepreplace (storeI(1.mem 1.mem 1.src));
13330 %}
13331 
13332 peephole
13333 %{
13334   peepmatch (loadL storeL);
13335   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13336   peepreplace (storeL(1.mem 1.mem 1.src));
13337 %}
13338 
13339 //----------SMARTSPILL RULES---------------------------------------------------
13340 // These must follow all instruction definitions as they use the names
13341 // defined in the instructions definitions.
--- EOF ---