1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 }
  458 
  459 static bool generate_vzeroupper(Compile* C) {
  460   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  461 }
  462 
  463 static int clear_avx_size() {
  464   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  465 }
  466 
  467 // !!!!! Special hack to get all types of calls to specify the byte offset
  468 //       from the start of the call to the point where the return address
  469 //       will point.
  470 int MachCallStaticJavaNode::ret_addr_offset()
  471 {
  472   int offset = 5; // 5 bytes from start of call to where return address points
  473   offset += clear_avx_size();
  474   return offset;
  475 }
  476 
  477 int MachCallDynamicJavaNode::ret_addr_offset()
  478 {
  479   int offset = 15; // 15 bytes from start of call to where return address points
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 
  484 int MachCallRuntimeNode::ret_addr_offset() {
  485   int offset = 13; // movq r10,#addr; callq (r10)
  486   if (this->ideal_Opcode() != Op_CallLeafVector) {
  487     offset += clear_avx_size();
  488   }
  489   return offset;
  490 }
  491 //
  492 // Compute padding required for nodes which need alignment
  493 //
  494 
  495 // The address of the call instruction needs to be 4-byte aligned to
  496 // ensure that it does not span a cache line so that it can be patched.
  497 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  498 {
  499   current_offset += clear_avx_size(); // skip vzeroupper
  500   current_offset += 1; // skip call opcode byte
  501   return align_up(current_offset, alignment_required()) - current_offset;
  502 }
  503 
  504 // The address of the call instruction needs to be 4-byte aligned to
  505 // ensure that it does not span a cache line so that it can be patched.
  506 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  507 {
  508   current_offset += clear_avx_size(); // skip vzeroupper
  509   current_offset += 11; // skip movq instruction + call opcode byte
  510   return align_up(current_offset, alignment_required()) - current_offset;
  511 }
  512 
  513 // EMIT_RM()
  514 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  515   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  516   cbuf.insts()->emit_int8(c);
  517 }
  518 
  519 // EMIT_CC()
  520 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  521   unsigned char c = (unsigned char) (f1 | f2);
  522   cbuf.insts()->emit_int8(c);
  523 }
  524 
  525 // EMIT_OPCODE()
  526 void emit_opcode(CodeBuffer &cbuf, int code) {
  527   cbuf.insts()->emit_int8((unsigned char) code);
  528 }
  529 
  530 // EMIT_OPCODE() w/ relocation information
  531 void emit_opcode(CodeBuffer &cbuf,
  532                  int code, relocInfo::relocType reloc, int offset, int format)
  533 {
  534   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  535   emit_opcode(cbuf, code);
  536 }
  537 
  538 // EMIT_D8()
  539 void emit_d8(CodeBuffer &cbuf, int d8) {
  540   cbuf.insts()->emit_int8((unsigned char) d8);
  541 }
  542 
  543 // EMIT_D16()
  544 void emit_d16(CodeBuffer &cbuf, int d16) {
  545   cbuf.insts()->emit_int16(d16);
  546 }
  547 
  548 // EMIT_D32()
  549 void emit_d32(CodeBuffer &cbuf, int d32) {
  550   cbuf.insts()->emit_int32(d32);
  551 }
  552 
  553 // EMIT_D64()
  554 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  555   cbuf.insts()->emit_int64(d64);
  556 }
  557 
  558 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  559 void emit_d32_reloc(CodeBuffer& cbuf,
  560                     int d32,
  561                     relocInfo::relocType reloc,
  562                     int format)
  563 {
  564   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  565   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  566   cbuf.insts()->emit_int32(d32);
  567 }
  568 
  569 // emit 32 bit value and construct relocation entry from RelocationHolder
  570 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  571 #ifdef ASSERT
  572   if (rspec.reloc()->type() == relocInfo::oop_type &&
  573       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  574     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  575     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  576   }
  577 #endif
  578   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  579   cbuf.insts()->emit_int32(d32);
  580 }
  581 
  582 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  583   address next_ip = cbuf.insts_end() + 4;
  584   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  585                  external_word_Relocation::spec(addr),
  586                  RELOC_DISP32);
  587 }
  588 
  589 
  590 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  591 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  592   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  593   cbuf.insts()->emit_int64(d64);
  594 }
  595 
  596 // emit 64 bit value and construct relocation entry from RelocationHolder
  597 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  598 #ifdef ASSERT
  599   if (rspec.reloc()->type() == relocInfo::oop_type &&
  600       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  601     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  602     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  603   }
  604 #endif
  605   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  606   cbuf.insts()->emit_int64(d64);
  607 }
  608 
  609 // Access stack slot for load or store
  610 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  611 {
  612   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  613   if (-0x80 <= disp && disp < 0x80) {
  614     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  615     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  616     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  617   } else {
  618     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  619     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  620     emit_d32(cbuf, disp);     // Displacement // R/M byte
  621   }
  622 }
  623 
  624    // rRegI ereg, memory mem) %{    // emit_reg_mem
  625 void encode_RegMem(CodeBuffer &cbuf,
  626                    int reg,
  627                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  628 {
  629   assert(disp_reloc == relocInfo::none, "cannot have disp");
  630   int regenc = reg & 7;
  631   int baseenc = base & 7;
  632   int indexenc = index & 7;
  633 
  634   // There is no index & no scale, use form without SIB byte
  635   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  636     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  637     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  638       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  639     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  640       // If 8-bit displacement, mode 0x1
  641       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  642       emit_d8(cbuf, disp);
  643     } else {
  644       // If 32-bit displacement
  645       if (base == -1) { // Special flag for absolute address
  646         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       } else {
  653         // Normal base + offset
  654         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       }
  661     }
  662   } else {
  663     // Else, encode with the SIB byte
  664     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  665     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  666       // If no displacement
  667       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  668       emit_rm(cbuf, scale, indexenc, baseenc);
  669     } else {
  670       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  671         // If 8-bit displacement, mode 0x1
  672         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  673         emit_rm(cbuf, scale, indexenc, baseenc);
  674         emit_d8(cbuf, disp);
  675       } else {
  676         // If 32-bit displacement
  677         if (base == 0x04 ) {
  678           emit_rm(cbuf, 0x2, regenc, 0x4);
  679           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  680         } else {
  681           emit_rm(cbuf, 0x2, regenc, 0x4);
  682           emit_rm(cbuf, scale, indexenc, baseenc); // *
  683         }
  684         if (disp_reloc != relocInfo::none) {
  685           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  686         } else {
  687           emit_d32(cbuf, disp);
  688         }
  689       }
  690     }
  691   }
  692 }
  693 
  694 // This could be in MacroAssembler but it's fairly C2 specific
  695 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  696   Label exit;
  697   __ jccb(Assembler::noParity, exit);
  698   __ pushf();
  699   //
  700   // comiss/ucomiss instructions set ZF,PF,CF flags and
  701   // zero OF,AF,SF for NaN values.
  702   // Fixup flags by zeroing ZF,PF so that compare of NaN
  703   // values returns 'less than' result (CF is set).
  704   // Leave the rest of flags unchanged.
  705   //
  706   //    7 6 5 4 3 2 1 0
  707   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  708   //    0 0 1 0 1 0 1 1   (0x2B)
  709   //
  710   __ andq(Address(rsp, 0), 0xffffff2b);
  711   __ popf();
  712   __ bind(exit);
  713 }
  714 
  715 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  716   Label done;
  717   __ movl(dst, -1);
  718   __ jcc(Assembler::parity, done);
  719   __ jcc(Assembler::below, done);
  720   __ setb(Assembler::notEqual, dst);
  721   __ movzbl(dst, dst);
  722   __ bind(done);
  723 }
  724 
  725 // Math.min()    # Math.max()
  726 // --------------------------
  727 // ucomis[s/d]   #
  728 // ja   -> b     # a
  729 // jp   -> NaN   # NaN
  730 // jb   -> a     # b
  731 // je            #
  732 // |-jz -> a | b # a & b
  733 // |    -> a     #
  734 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  735                      XMMRegister a, XMMRegister b,
  736                      XMMRegister xmmt, Register rt,
  737                      bool min, bool single) {
  738 
  739   Label nan, zero, below, above, done;
  740 
  741   if (single)
  742     __ ucomiss(a, b);
  743   else
  744     __ ucomisd(a, b);
  745 
  746   if (dst->encoding() != (min ? b : a)->encoding())
  747     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  748   else
  749     __ jccb(Assembler::above, done);
  750 
  751   __ jccb(Assembler::parity, nan);  // PF=1
  752   __ jccb(Assembler::below, below); // CF=1
  753 
  754   // equal
  755   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  756   if (single) {
  757     __ ucomiss(a, xmmt);
  758     __ jccb(Assembler::equal, zero);
  759 
  760     __ movflt(dst, a);
  761     __ jmp(done);
  762   }
  763   else {
  764     __ ucomisd(a, xmmt);
  765     __ jccb(Assembler::equal, zero);
  766 
  767     __ movdbl(dst, a);
  768     __ jmp(done);
  769   }
  770 
  771   __ bind(zero);
  772   if (min)
  773     __ vpor(dst, a, b, Assembler::AVX_128bit);
  774   else
  775     __ vpand(dst, a, b, Assembler::AVX_128bit);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(above);
  780   if (single)
  781     __ movflt(dst, min ? b : a);
  782   else
  783     __ movdbl(dst, min ? b : a);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(nan);
  788   if (single) {
  789     __ movl(rt, 0x7fc00000); // Float.NaN
  790     __ movdl(dst, rt);
  791   }
  792   else {
  793     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  794     __ movdq(dst, rt);
  795   }
  796   __ jmp(done);
  797 
  798   __ bind(below);
  799   if (single)
  800     __ movflt(dst, min ? a : b);
  801   else
  802     __ movdbl(dst, min ? a : b);
  803 
  804   __ bind(done);
  805 }
  806 
  807 //=============================================================================
  808 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  809 
  810 int ConstantTable::calculate_table_base_offset() const {
  811   return 0;  // absolute addressing, no offset
  812 }
  813 
  814 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  815 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  816   ShouldNotReachHere();
  817 }
  818 
  819 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  820   // Empty encoding
  821 }
  822 
  823 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  824   return 0;
  825 }
  826 
  827 #ifndef PRODUCT
  828 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   st->print("# MachConstantBaseNode (empty encoding)");
  830 }
  831 #endif
  832 
  833 
  834 //=============================================================================
  835 #ifndef PRODUCT
  836 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   Compile* C = ra_->C;
  838 
  839   int framesize = C->output()->frame_size_in_bytes();
  840   int bangsize = C->output()->bang_size_in_bytes();
  841   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  842   // Remove wordSize for return addr which is already pushed.
  843   framesize -= wordSize;
  844 
  845   if (C->output()->need_stack_bang(bangsize)) {
  846     framesize -= wordSize;
  847     st->print("# stack bang (%d bytes)", bangsize);
  848     st->print("\n\t");
  849     st->print("pushq   rbp\t# Save rbp");
  850     if (PreserveFramePointer) {
  851         st->print("\n\t");
  852         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  853     }
  854     if (framesize) {
  855       st->print("\n\t");
  856       st->print("subq    rsp, #%d\t# Create frame",framesize);
  857     }
  858   } else {
  859     st->print("subq    rsp, #%d\t# Create frame",framesize);
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  863     if (PreserveFramePointer) {
  864       st->print("\n\t");
  865       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  866       if (framesize > 0) {
  867         st->print("\n\t");
  868         st->print("addq    rbp, #%d", framesize);
  869       }
  870     }
  871   }
  872 
  873   if (VerifyStackAtCalls) {
  874     st->print("\n\t");
  875     framesize -= wordSize;
  876     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  877 #ifdef ASSERT
  878     st->print("\n\t");
  879     st->print("# stack alignment check");
  880 #endif
  881   }
  882   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  883     st->print("\n\t");
  884     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  885     st->print("\n\t");
  886     st->print("je      fast_entry\t");
  887     st->print("\n\t");
  888     st->print("call    #nmethod_entry_barrier_stub\t");
  889     st->print("\n\tfast_entry:");
  890   }
  891   st->cr();
  892 }
  893 #endif
  894 
  895 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  896   Compile* C = ra_->C;
  897   C2_MacroAssembler _masm(&cbuf);
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   int bangsize = C->output()->bang_size_in_bytes();
  901 
  902   if (C->clinit_barrier_on_entry()) {
  903     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  904     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  905 
  906     Label L_skip_barrier;
  907     Register klass = rscratch1;
  908 
  909     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  910     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  911 
  912     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  913 
  914     __ bind(L_skip_barrier);
  915   }
  916 
  917   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  918 
  919   C->output()->set_frame_complete(cbuf.insts_size());
  920 
  921   if (C->has_mach_constant_base_node()) {
  922     // NOTE: We set the table base offset here because users might be
  923     // emitted before MachConstantBaseNode.
  924     ConstantTable& constant_table = C->output()->constant_table();
  925     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  926   }
  927 }
  928 
  929 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  930 {
  931   return MachNode::size(ra_); // too many variables; just compute it
  932                               // the hard way
  933 }
  934 
  935 int MachPrologNode::reloc() const
  936 {
  937   return 0; // a large enough number
  938 }
  939 
  940 //=============================================================================
  941 #ifndef PRODUCT
  942 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  943 {
  944   Compile* C = ra_->C;
  945   if (generate_vzeroupper(C)) {
  946     st->print("vzeroupper");
  947     st->cr(); st->print("\t");
  948   }
  949 
  950   int framesize = C->output()->frame_size_in_bytes();
  951   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  952   // Remove word for return adr already pushed
  953   // and RBP
  954   framesize -= 2*wordSize;
  955 
  956   if (framesize) {
  957     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  958     st->print("\t");
  959   }
  960 
  961   st->print_cr("popq    rbp");
  962   if (do_polling() && C->is_method_compilation()) {
  963     st->print("\t");
  964     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  965                  "ja       #safepoint_stub\t"
  966                  "# Safepoint: poll for GC");
  967   }
  968 }
  969 #endif
  970 
  971 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  972 {
  973   Compile* C = ra_->C;
  974   MacroAssembler _masm(&cbuf);
  975 
  976   if (generate_vzeroupper(C)) {
  977     // Clear upper bits of YMM registers when current compiled code uses
  978     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  979     __ vzeroupper();
  980   }
  981 
  982   int framesize = C->output()->frame_size_in_bytes();
  983   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  984   // Remove word for return adr already pushed
  985   // and RBP
  986   framesize -= 2*wordSize;
  987 
  988   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  989 
  990   if (framesize) {
  991     emit_opcode(cbuf, Assembler::REX_W);
  992     if (framesize < 0x80) {
  993       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  995       emit_d8(cbuf, framesize);
  996     } else {
  997       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  998       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  999       emit_d32(cbuf, framesize);
 1000     }
 1001   }
 1002 
 1003   // popq rbp
 1004   emit_opcode(cbuf, 0x58 | RBP_enc);
 1005 
 1006   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1007     __ reserved_stack_check();
 1008   }
 1009 
 1010   if (do_polling() && C->is_method_compilation()) {
 1011     MacroAssembler _masm(&cbuf);
 1012     Label dummy_label;
 1013     Label* code_stub = &dummy_label;
 1014     if (!C->output()->in_scratch_emit_size()) {
 1015       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1016     }
 1017     __ relocate(relocInfo::poll_return_type);
 1018     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1019   }
 1020 }
 1021 
 1022 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1023 {
 1024   return MachNode::size(ra_); // too many variables; just compute it
 1025                               // the hard way
 1026 }
 1027 
 1028 int MachEpilogNode::reloc() const
 1029 {
 1030   return 2; // a large enough number
 1031 }
 1032 
 1033 const Pipeline* MachEpilogNode::pipeline() const
 1034 {
 1035   return MachNode::pipeline_class();
 1036 }
 1037 
 1038 //=============================================================================
 1039 
 1040 enum RC {
 1041   rc_bad,
 1042   rc_int,
 1043   rc_kreg,
 1044   rc_float,
 1045   rc_stack
 1046 };
 1047 
 1048 static enum RC rc_class(OptoReg::Name reg)
 1049 {
 1050   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1051 
 1052   if (OptoReg::is_stack(reg)) return rc_stack;
 1053 
 1054   VMReg r = OptoReg::as_VMReg(reg);
 1055 
 1056   if (r->is_Register()) return rc_int;
 1057 
 1058   if (r->is_KRegister()) return rc_kreg;
 1059 
 1060   assert(r->is_XMMRegister(), "must be");
 1061   return rc_float;
 1062 }
 1063 
 1064 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1065 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1066                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1067 
 1068 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1069                      int stack_offset, int reg, uint ireg, outputStream* st);
 1070 
 1071 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1072                                       int dst_offset, uint ireg, outputStream* st) {
 1073   if (cbuf) {
 1074     MacroAssembler _masm(cbuf);
 1075     switch (ireg) {
 1076     case Op_VecS:
 1077       __ movq(Address(rsp, -8), rax);
 1078       __ movl(rax, Address(rsp, src_offset));
 1079       __ movl(Address(rsp, dst_offset), rax);
 1080       __ movq(rax, Address(rsp, -8));
 1081       break;
 1082     case Op_VecD:
 1083       __ pushq(Address(rsp, src_offset));
 1084       __ popq (Address(rsp, dst_offset));
 1085       break;
 1086     case Op_VecX:
 1087       __ pushq(Address(rsp, src_offset));
 1088       __ popq (Address(rsp, dst_offset));
 1089       __ pushq(Address(rsp, src_offset+8));
 1090       __ popq (Address(rsp, dst_offset+8));
 1091       break;
 1092     case Op_VecY:
 1093       __ vmovdqu(Address(rsp, -32), xmm0);
 1094       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1095       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1096       __ vmovdqu(xmm0, Address(rsp, -32));
 1097       break;
 1098     case Op_VecZ:
 1099       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1100       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1101       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1102       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1103       break;
 1104     default:
 1105       ShouldNotReachHere();
 1106     }
 1107 #ifndef PRODUCT
 1108   } else {
 1109     switch (ireg) {
 1110     case Op_VecS:
 1111       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1112                 "movl    rax, [rsp + #%d]\n\t"
 1113                 "movl    [rsp + #%d], rax\n\t"
 1114                 "movq    rax, [rsp - #8]",
 1115                 src_offset, dst_offset);
 1116       break;
 1117     case Op_VecD:
 1118       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1119                 "popq    [rsp + #%d]",
 1120                 src_offset, dst_offset);
 1121       break;
 1122      case Op_VecX:
 1123       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1124                 "popq    [rsp + #%d]\n\t"
 1125                 "pushq   [rsp + #%d]\n\t"
 1126                 "popq    [rsp + #%d]",
 1127                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1128       break;
 1129     case Op_VecY:
 1130       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1133                 "vmovdqu xmm0, [rsp - #32]",
 1134                 src_offset, dst_offset);
 1135       break;
 1136     case Op_VecZ:
 1137       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1138                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1139                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1140                 "vmovdqu xmm0, [rsp - #64]",
 1141                 src_offset, dst_offset);
 1142       break;
 1143     default:
 1144       ShouldNotReachHere();
 1145     }
 1146 #endif
 1147   }
 1148 }
 1149 
 1150 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1151                                        PhaseRegAlloc* ra_,
 1152                                        bool do_size,
 1153                                        outputStream* st) const {
 1154   assert(cbuf != NULL || st  != NULL, "sanity");
 1155   // Get registers to move
 1156   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1157   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1158   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1159   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1160 
 1161   enum RC src_second_rc = rc_class(src_second);
 1162   enum RC src_first_rc = rc_class(src_first);
 1163   enum RC dst_second_rc = rc_class(dst_second);
 1164   enum RC dst_first_rc = rc_class(dst_first);
 1165 
 1166   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1167          "must move at least 1 register" );
 1168 
 1169   if (src_first == dst_first && src_second == dst_second) {
 1170     // Self copy, no move
 1171     return 0;
 1172   }
 1173   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1174     uint ireg = ideal_reg();
 1175     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1176     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1177     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1178       // mem -> mem
 1179       int src_offset = ra_->reg2offset(src_first);
 1180       int dst_offset = ra_->reg2offset(dst_first);
 1181       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1182     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1183       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1184     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1185       int stack_offset = ra_->reg2offset(dst_first);
 1186       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1187     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1188       int stack_offset = ra_->reg2offset(src_first);
 1189       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1190     } else {
 1191       ShouldNotReachHere();
 1192     }
 1193     return 0;
 1194   }
 1195   if (src_first_rc == rc_stack) {
 1196     // mem ->
 1197     if (dst_first_rc == rc_stack) {
 1198       // mem -> mem
 1199       assert(src_second != dst_first, "overlap");
 1200       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1201           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1202         // 64-bit
 1203         int src_offset = ra_->reg2offset(src_first);
 1204         int dst_offset = ra_->reg2offset(dst_first);
 1205         if (cbuf) {
 1206           MacroAssembler _masm(cbuf);
 1207           __ pushq(Address(rsp, src_offset));
 1208           __ popq (Address(rsp, dst_offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1212                     "popq    [rsp + #%d]",
 1213                      src_offset, dst_offset);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         // No pushl/popl, so:
 1221         int src_offset = ra_->reg2offset(src_first);
 1222         int dst_offset = ra_->reg2offset(dst_first);
 1223         if (cbuf) {
 1224           MacroAssembler _masm(cbuf);
 1225           __ movq(Address(rsp, -8), rax);
 1226           __ movl(rax, Address(rsp, src_offset));
 1227           __ movl(Address(rsp, dst_offset), rax);
 1228           __ movq(rax, Address(rsp, -8));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1232                     "movl    rax, [rsp + #%d]\n\t"
 1233                     "movl    [rsp + #%d], rax\n\t"
 1234                     "movq    rax, [rsp - #8]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_int) {
 1241       // mem -> gpr
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (cbuf) {
 1247           MacroAssembler _masm(cbuf);
 1248           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1249 #ifndef PRODUCT
 1250         } else {
 1251           st->print("movq    %s, [rsp + #%d]\t# spill",
 1252                      Matcher::regName[dst_first],
 1253                      offset);
 1254 #endif
 1255         }
 1256       } else {
 1257         // 32-bit
 1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1260         int offset = ra_->reg2offset(src_first);
 1261         if (cbuf) {
 1262           MacroAssembler _masm(cbuf);
 1263           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1264 #ifndef PRODUCT
 1265         } else {
 1266           st->print("movl    %s, [rsp + #%d]\t# spill",
 1267                      Matcher::regName[dst_first],
 1268                      offset);
 1269 #endif
 1270         }
 1271       }
 1272       return 0;
 1273     } else if (dst_first_rc == rc_float) {
 1274       // mem-> xmm
 1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1277         // 64-bit
 1278         int offset = ra_->reg2offset(src_first);
 1279         if (cbuf) {
 1280           MacroAssembler _masm(cbuf);
 1281           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("%s  %s, [rsp + #%d]\t# spill",
 1285                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1286                      Matcher::regName[dst_first],
 1287                      offset);
 1288 #endif
 1289         }
 1290       } else {
 1291         // 32-bit
 1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1294         int offset = ra_->reg2offset(src_first);
 1295         if (cbuf) {
 1296           MacroAssembler _masm(cbuf);
 1297           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movss   %s, [rsp + #%d]\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      offset);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_kreg) {
 1308       // mem -> kreg
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         int offset = ra_->reg2offset(src_first);
 1313         if (cbuf) {
 1314           MacroAssembler _masm(cbuf);
 1315           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1316 #ifndef PRODUCT
 1317         } else {
 1318           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1319                      Matcher::regName[dst_first],
 1320                      offset);
 1321 #endif
 1322         }
 1323       }
 1324       return 0;
 1325     }
 1326   } else if (src_first_rc == rc_int) {
 1327     // gpr ->
 1328     if (dst_first_rc == rc_stack) {
 1329       // gpr -> mem
 1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1332         // 64-bit
 1333         int offset = ra_->reg2offset(dst_first);
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("movq    [rsp + #%d], %s\t# spill",
 1340                      offset,
 1341                      Matcher::regName[src_first]);
 1342 #endif
 1343         }
 1344       } else {
 1345         // 32-bit
 1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("movl    [rsp + #%d], %s\t# spill",
 1355                      offset,
 1356                      Matcher::regName[src_first]);
 1357 #endif
 1358         }
 1359       }
 1360       return 0;
 1361     } else if (dst_first_rc == rc_int) {
 1362       // gpr -> gpr
 1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1365         // 64-bit
 1366         if (cbuf) {
 1367           MacroAssembler _masm(cbuf);
 1368           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1369                   as_Register(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movq    %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377         return 0;
 1378       } else {
 1379         // 32-bit
 1380         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1381         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1382         if (cbuf) {
 1383           MacroAssembler _masm(cbuf);
 1384           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1385                   as_Register(Matcher::_regEncode[src_first]));
 1386 #ifndef PRODUCT
 1387         } else {
 1388           st->print("movl    %s, %s\t# spill",
 1389                      Matcher::regName[dst_first],
 1390                      Matcher::regName[src_first]);
 1391 #endif
 1392         }
 1393         return 0;
 1394       }
 1395     } else if (dst_first_rc == rc_float) {
 1396       // gpr -> xmm
 1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1399         // 64-bit
 1400         if (cbuf) {
 1401           MacroAssembler _masm(cbuf);
 1402           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1403 #ifndef PRODUCT
 1404         } else {
 1405           st->print("movdq   %s, %s\t# spill",
 1406                      Matcher::regName[dst_first],
 1407                      Matcher::regName[src_first]);
 1408 #endif
 1409         }
 1410       } else {
 1411         // 32-bit
 1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1414         if (cbuf) {
 1415           MacroAssembler _masm(cbuf);
 1416           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1417 #ifndef PRODUCT
 1418         } else {
 1419           st->print("movdl   %s, %s\t# spill",
 1420                      Matcher::regName[dst_first],
 1421                      Matcher::regName[src_first]);
 1422 #endif
 1423         }
 1424       }
 1425       return 0;
 1426     } else if (dst_first_rc == rc_kreg) {
 1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1429         // 64-bit
 1430         if (cbuf) {
 1431           MacroAssembler _masm(cbuf);
 1432           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1433   #ifndef PRODUCT
 1434         } else {
 1435            st->print("kmovq   %s, %s\t# spill",
 1436                        Matcher::regName[dst_first],
 1437                        Matcher::regName[src_first]);
 1438   #endif
 1439         }
 1440       }
 1441       Unimplemented();
 1442       return 0;
 1443     }
 1444   } else if (src_first_rc == rc_float) {
 1445     // xmm ->
 1446     if (dst_first_rc == rc_stack) {
 1447       // xmm -> mem
 1448       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1449           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1450         // 64-bit
 1451         int offset = ra_->reg2offset(dst_first);
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1455 #ifndef PRODUCT
 1456         } else {
 1457           st->print("movsd   [rsp + #%d], %s\t# spill",
 1458                      offset,
 1459                      Matcher::regName[src_first]);
 1460 #endif
 1461         }
 1462       } else {
 1463         // 32-bit
 1464         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1465         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1466         int offset = ra_->reg2offset(dst_first);
 1467         if (cbuf) {
 1468           MacroAssembler _masm(cbuf);
 1469           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1470 #ifndef PRODUCT
 1471         } else {
 1472           st->print("movss   [rsp + #%d], %s\t# spill",
 1473                      offset,
 1474                      Matcher::regName[src_first]);
 1475 #endif
 1476         }
 1477       }
 1478       return 0;
 1479     } else if (dst_first_rc == rc_int) {
 1480       // xmm -> gpr
 1481       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1482           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1483         // 64-bit
 1484         if (cbuf) {
 1485           MacroAssembler _masm(cbuf);
 1486           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1487 #ifndef PRODUCT
 1488         } else {
 1489           st->print("movdq   %s, %s\t# spill",
 1490                      Matcher::regName[dst_first],
 1491                      Matcher::regName[src_first]);
 1492 #endif
 1493         }
 1494       } else {
 1495         // 32-bit
 1496         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1497         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1498         if (cbuf) {
 1499           MacroAssembler _masm(cbuf);
 1500           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1501 #ifndef PRODUCT
 1502         } else {
 1503           st->print("movdl   %s, %s\t# spill",
 1504                      Matcher::regName[dst_first],
 1505                      Matcher::regName[src_first]);
 1506 #endif
 1507         }
 1508       }
 1509       return 0;
 1510     } else if (dst_first_rc == rc_float) {
 1511       // xmm -> xmm
 1512       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1513           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1514         // 64-bit
 1515         if (cbuf) {
 1516           MacroAssembler _masm(cbuf);
 1517           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1518 #ifndef PRODUCT
 1519         } else {
 1520           st->print("%s  %s, %s\t# spill",
 1521                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1522                      Matcher::regName[dst_first],
 1523                      Matcher::regName[src_first]);
 1524 #endif
 1525         }
 1526       } else {
 1527         // 32-bit
 1528         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1529         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1530         if (cbuf) {
 1531           MacroAssembler _masm(cbuf);
 1532           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1533 #ifndef PRODUCT
 1534         } else {
 1535           st->print("%s  %s, %s\t# spill",
 1536                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1537                      Matcher::regName[dst_first],
 1538                      Matcher::regName[src_first]);
 1539 #endif
 1540         }
 1541       }
 1542       return 0;
 1543     } else if (dst_first_rc == rc_kreg) {
 1544       assert(false, "Illegal spilling");
 1545       return 0;
 1546     }
 1547   } else if (src_first_rc == rc_kreg) {
 1548     if (dst_first_rc == rc_stack) {
 1549       // mem -> kreg
 1550       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1551           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1552         // 64-bit
 1553         int offset = ra_->reg2offset(dst_first);
 1554         if (cbuf) {
 1555           MacroAssembler _masm(cbuf);
 1556           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1557 #ifndef PRODUCT
 1558         } else {
 1559           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1560                      offset,
 1561                      Matcher::regName[src_first]);
 1562 #endif
 1563         }
 1564       }
 1565       return 0;
 1566     } else if (dst_first_rc == rc_int) {
 1567       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1568           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1569         // 64-bit
 1570         if (cbuf) {
 1571           MacroAssembler _masm(cbuf);
 1572           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1573 #ifndef PRODUCT
 1574         } else {
 1575          st->print("kmovq   %s, %s\t# spill",
 1576                      Matcher::regName[dst_first],
 1577                      Matcher::regName[src_first]);
 1578 #endif
 1579         }
 1580       }
 1581       Unimplemented();
 1582       return 0;
 1583     } else if (dst_first_rc == rc_kreg) {
 1584       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1585           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1586         // 64-bit
 1587         if (cbuf) {
 1588           MacroAssembler _masm(cbuf);
 1589           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1590 #ifndef PRODUCT
 1591         } else {
 1592          st->print("kmovq   %s, %s\t# spill",
 1593                      Matcher::regName[dst_first],
 1594                      Matcher::regName[src_first]);
 1595 #endif
 1596         }
 1597       }
 1598       return 0;
 1599     } else if (dst_first_rc == rc_float) {
 1600       assert(false, "Illegal spill");
 1601       return 0;
 1602     }
 1603   }
 1604 
 1605   assert(0," foo ");
 1606   Unimplemented();
 1607   return 0;
 1608 }
 1609 
 1610 #ifndef PRODUCT
 1611 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1612   implementation(NULL, ra_, false, st);
 1613 }
 1614 #endif
 1615 
 1616 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1617   implementation(&cbuf, ra_, false, NULL);
 1618 }
 1619 
 1620 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1621   return MachNode::size(ra_);
 1622 }
 1623 
 1624 //=============================================================================
 1625 #ifndef PRODUCT
 1626 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1627 {
 1628   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1629   int reg = ra_->get_reg_first(this);
 1630   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1631             Matcher::regName[reg], offset);
 1632 }
 1633 #endif
 1634 
 1635 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1636 {
 1637   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1638   int reg = ra_->get_encode(this);
 1639   if (offset >= 0x80) {
 1640     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1641     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1642     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1643     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1644     emit_d32(cbuf, offset);
 1645   } else {
 1646     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1647     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1648     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1649     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1650     emit_d8(cbuf, offset);
 1651   }
 1652 }
 1653 
 1654 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1655 {
 1656   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1657   return (offset < 0x80) ? 5 : 8; // REX
 1658 }
 1659 
 1660 //=============================================================================
 1661 #ifndef PRODUCT
 1662 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1663 {
 1664   if (UseCompressedClassPointers) {
 1665     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1666     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1667     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1668   } else {
 1669     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1670                  "# Inline cache check");
 1671   }
 1672   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1673   st->print_cr("\tnop\t# nops to align entry point");
 1674 }
 1675 #endif
 1676 
 1677 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1678 {
 1679   MacroAssembler masm(&cbuf);
 1680   uint insts_size = cbuf.insts_size();
 1681   if (UseCompressedClassPointers) {
 1682     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1683     masm.cmpptr(rax, rscratch1);
 1684   } else {
 1685     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1686   }
 1687 
 1688   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1689 
 1690   /* WARNING these NOPs are critical so that verified entry point is properly
 1691      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1692   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1693   if (OptoBreakpoint) {
 1694     // Leave space for int3
 1695     nops_cnt -= 1;
 1696   }
 1697   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1698   if (nops_cnt > 0)
 1699     masm.nop(nops_cnt);
 1700 }
 1701 
 1702 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1703 {
 1704   return MachNode::size(ra_); // too many variables; just compute it
 1705                               // the hard way
 1706 }
 1707 
 1708 
 1709 //=============================================================================
 1710 
 1711 const bool Matcher::supports_vector_calling_convention(void) {
 1712   if (EnableVectorSupport && UseVectorStubs) {
 1713     return true;
 1714   }
 1715   return false;
 1716 }
 1717 
 1718 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1719   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1720   int lo = XMM0_num;
 1721   int hi = XMM0b_num;
 1722   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1723   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1724   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1725   return OptoRegPair(hi, lo);
 1726 }
 1727 
 1728 // Is this branch offset short enough that a short branch can be used?
 1729 //
 1730 // NOTE: If the platform does not provide any short branch variants, then
 1731 //       this method should return false for offset 0.
 1732 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1733   // The passed offset is relative to address of the branch.
 1734   // On 86 a branch displacement is calculated relative to address
 1735   // of a next instruction.
 1736   offset -= br_size;
 1737 
 1738   // the short version of jmpConUCF2 contains multiple branches,
 1739   // making the reach slightly less
 1740   if (rule == jmpConUCF2_rule)
 1741     return (-126 <= offset && offset <= 125);
 1742   return (-128 <= offset && offset <= 127);
 1743 }
 1744 
 1745 // Return whether or not this register is ever used as an argument.
 1746 // This function is used on startup to build the trampoline stubs in
 1747 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1748 // call in the trampoline, and arguments in those registers not be
 1749 // available to the callee.
 1750 bool Matcher::can_be_java_arg(int reg)
 1751 {
 1752   return
 1753     reg ==  RDI_num || reg == RDI_H_num ||
 1754     reg ==  RSI_num || reg == RSI_H_num ||
 1755     reg ==  RDX_num || reg == RDX_H_num ||
 1756     reg ==  RCX_num || reg == RCX_H_num ||
 1757     reg ==   R8_num || reg ==  R8_H_num ||
 1758     reg ==   R9_num || reg ==  R9_H_num ||
 1759     reg ==  R12_num || reg == R12_H_num ||
 1760     reg == XMM0_num || reg == XMM0b_num ||
 1761     reg == XMM1_num || reg == XMM1b_num ||
 1762     reg == XMM2_num || reg == XMM2b_num ||
 1763     reg == XMM3_num || reg == XMM3b_num ||
 1764     reg == XMM4_num || reg == XMM4b_num ||
 1765     reg == XMM5_num || reg == XMM5b_num ||
 1766     reg == XMM6_num || reg == XMM6b_num ||
 1767     reg == XMM7_num || reg == XMM7b_num;
 1768 }
 1769 
 1770 bool Matcher::is_spillable_arg(int reg)
 1771 {
 1772   return can_be_java_arg(reg);
 1773 }
 1774 
 1775 uint Matcher::int_pressure_limit()
 1776 {
 1777   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1778 }
 1779 
 1780 uint Matcher::float_pressure_limit()
 1781 {
 1782   // After experiment around with different values, the following default threshold
 1783   // works best for LCM's register pressure scheduling on x64.
 1784   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1785   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1786   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1787 }
 1788 
 1789 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1790   // In 64 bit mode a code which use multiply when
 1791   // devisor is constant is faster than hardware
 1792   // DIV instruction (it uses MulHiL).
 1793   return false;
 1794 }
 1795 
 1796 // Register for DIVI projection of divmodI
 1797 RegMask Matcher::divI_proj_mask() {
 1798   return INT_RAX_REG_mask();
 1799 }
 1800 
 1801 // Register for MODI projection of divmodI
 1802 RegMask Matcher::modI_proj_mask() {
 1803   return INT_RDX_REG_mask();
 1804 }
 1805 
 1806 // Register for DIVL projection of divmodL
 1807 RegMask Matcher::divL_proj_mask() {
 1808   return LONG_RAX_REG_mask();
 1809 }
 1810 
 1811 // Register for MODL projection of divmodL
 1812 RegMask Matcher::modL_proj_mask() {
 1813   return LONG_RDX_REG_mask();
 1814 }
 1815 
 1816 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1817 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1818     return NO_REG_mask();
 1819 }
 1820 
 1821 %}
 1822 
 1823 //----------ENCODING BLOCK-----------------------------------------------------
 1824 // This block specifies the encoding classes used by the compiler to
 1825 // output byte streams.  Encoding classes are parameterized macros
 1826 // used by Machine Instruction Nodes in order to generate the bit
 1827 // encoding of the instruction.  Operands specify their base encoding
 1828 // interface with the interface keyword.  There are currently
 1829 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1830 // COND_INTER.  REG_INTER causes an operand to generate a function
 1831 // which returns its register number when queried.  CONST_INTER causes
 1832 // an operand to generate a function which returns the value of the
 1833 // constant when queried.  MEMORY_INTER causes an operand to generate
 1834 // four functions which return the Base Register, the Index Register,
 1835 // the Scale Value, and the Offset Value of the operand when queried.
 1836 // COND_INTER causes an operand to generate six functions which return
 1837 // the encoding code (ie - encoding bits for the instruction)
 1838 // associated with each basic boolean condition for a conditional
 1839 // instruction.
 1840 //
 1841 // Instructions specify two basic values for encoding.  Again, a
 1842 // function is available to check if the constant displacement is an
 1843 // oop. They use the ins_encode keyword to specify their encoding
 1844 // classes (which must be a sequence of enc_class names, and their
 1845 // parameters, specified in the encoding block), and they use the
 1846 // opcode keyword to specify, in order, their primary, secondary, and
 1847 // tertiary opcode.  Only the opcode sections which a particular
 1848 // instruction needs for encoding need to be specified.
 1849 encode %{
 1850   // Build emit functions for each basic byte or larger field in the
 1851   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1852   // from C++ code in the enc_class source block.  Emit functions will
 1853   // live in the main source block for now.  In future, we can
 1854   // generalize this by adding a syntax that specifies the sizes of
 1855   // fields in an order, so that the adlc can build the emit functions
 1856   // automagically
 1857 
 1858   // Emit primary opcode
 1859   enc_class OpcP
 1860   %{
 1861     emit_opcode(cbuf, $primary);
 1862   %}
 1863 
 1864   // Emit secondary opcode
 1865   enc_class OpcS
 1866   %{
 1867     emit_opcode(cbuf, $secondary);
 1868   %}
 1869 
 1870   // Emit tertiary opcode
 1871   enc_class OpcT
 1872   %{
 1873     emit_opcode(cbuf, $tertiary);
 1874   %}
 1875 
 1876   // Emit opcode directly
 1877   enc_class Opcode(immI d8)
 1878   %{
 1879     emit_opcode(cbuf, $d8$$constant);
 1880   %}
 1881 
 1882   // Emit size prefix
 1883   enc_class SizePrefix
 1884   %{
 1885     emit_opcode(cbuf, 0x66);
 1886   %}
 1887 
 1888   enc_class reg(rRegI reg)
 1889   %{
 1890     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1891   %}
 1892 
 1893   enc_class reg_reg(rRegI dst, rRegI src)
 1894   %{
 1895     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1896   %}
 1897 
 1898   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1899   %{
 1900     emit_opcode(cbuf, $opcode$$constant);
 1901     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1902   %}
 1903 
 1904   enc_class cdql_enc(no_rax_rdx_RegI div)
 1905   %{
 1906     // Full implementation of Java idiv and irem; checks for
 1907     // special case as described in JVM spec., p.243 & p.271.
 1908     //
 1909     //         normal case                           special case
 1910     //
 1911     // input : rax: dividend                         min_int
 1912     //         reg: divisor                          -1
 1913     //
 1914     // output: rax: quotient  (= rax idiv reg)       min_int
 1915     //         rdx: remainder (= rax irem reg)       0
 1916     //
 1917     //  Code sequnce:
 1918     //
 1919     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1920     //    5:   75 07/08                jne    e <normal>
 1921     //    7:   33 d2                   xor    %edx,%edx
 1922     //  [div >= 8 -> offset + 1]
 1923     //  [REX_B]
 1924     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1925     //    c:   74 03/04                je     11 <done>
 1926     // 000000000000000e <normal>:
 1927     //    e:   99                      cltd
 1928     //  [div >= 8 -> offset + 1]
 1929     //  [REX_B]
 1930     //    f:   f7 f9                   idiv   $div
 1931     // 0000000000000011 <done>:
 1932     MacroAssembler _masm(&cbuf);
 1933     Label normal;
 1934     Label done;
 1935 
 1936     // cmp    $0x80000000,%eax
 1937     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1938 
 1939     // jne    e <normal>
 1940     __ jccb(Assembler::notEqual, normal);
 1941 
 1942     // xor    %edx,%edx
 1943     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1944 
 1945     // cmp    $0xffffffffffffffff,%ecx
 1946     __ cmpl($div$$Register, -1);
 1947 
 1948     // je     11 <done>
 1949     __ jccb(Assembler::equal, done);
 1950 
 1951     // <normal>
 1952     // cltd
 1953     __ bind(normal);
 1954     __ cdql();
 1955 
 1956     // idivl
 1957     // <done>
 1958     __ idivl($div$$Register);
 1959     __ bind(done);
 1960   %}
 1961 
 1962   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1963   %{
 1964     // Full implementation of Java ldiv and lrem; checks for
 1965     // special case as described in JVM spec., p.243 & p.271.
 1966     //
 1967     //         normal case                           special case
 1968     //
 1969     // input : rax: dividend                         min_long
 1970     //         reg: divisor                          -1
 1971     //
 1972     // output: rax: quotient  (= rax idiv reg)       min_long
 1973     //         rdx: remainder (= rax irem reg)       0
 1974     //
 1975     //  Code sequnce:
 1976     //
 1977     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1978     //    7:   00 00 80
 1979     //    a:   48 39 d0                cmp    %rdx,%rax
 1980     //    d:   75 08                   jne    17 <normal>
 1981     //    f:   33 d2                   xor    %edx,%edx
 1982     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1983     //   15:   74 05                   je     1c <done>
 1984     // 0000000000000017 <normal>:
 1985     //   17:   48 99                   cqto
 1986     //   19:   48 f7 f9                idiv   $div
 1987     // 000000000000001c <done>:
 1988     MacroAssembler _masm(&cbuf);
 1989     Label normal;
 1990     Label done;
 1991 
 1992     // mov    $0x8000000000000000,%rdx
 1993     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1994 
 1995     // cmp    %rdx,%rax
 1996     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1997 
 1998     // jne    17 <normal>
 1999     __ jccb(Assembler::notEqual, normal);
 2000 
 2001     // xor    %edx,%edx
 2002     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2003 
 2004     // cmp    $0xffffffffffffffff,$div
 2005     __ cmpq($div$$Register, -1);
 2006 
 2007     // je     1e <done>
 2008     __ jccb(Assembler::equal, done);
 2009 
 2010     // <normal>
 2011     // cqto
 2012     __ bind(normal);
 2013     __ cdqq();
 2014 
 2015     // idivq (note: must be emitted by the user of this rule)
 2016     // <done>
 2017     __ idivq($div$$Register);
 2018     __ bind(done);
 2019   %}
 2020 
 2021   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2022   enc_class OpcSE(immI imm)
 2023   %{
 2024     // Emit primary opcode and set sign-extend bit
 2025     // Check for 8-bit immediate, and set sign extend bit in opcode
 2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2027       emit_opcode(cbuf, $primary | 0x02);
 2028     } else {
 2029       // 32-bit immediate
 2030       emit_opcode(cbuf, $primary);
 2031     }
 2032   %}
 2033 
 2034   enc_class OpcSErm(rRegI dst, immI imm)
 2035   %{
 2036     // OpcSEr/m
 2037     int dstenc = $dst$$reg;
 2038     if (dstenc >= 8) {
 2039       emit_opcode(cbuf, Assembler::REX_B);
 2040       dstenc -= 8;
 2041     }
 2042     // Emit primary opcode and set sign-extend bit
 2043     // Check for 8-bit immediate, and set sign extend bit in opcode
 2044     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2045       emit_opcode(cbuf, $primary | 0x02);
 2046     } else {
 2047       // 32-bit immediate
 2048       emit_opcode(cbuf, $primary);
 2049     }
 2050     // Emit r/m byte with secondary opcode, after primary opcode.
 2051     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2052   %}
 2053 
 2054   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2055   %{
 2056     // OpcSEr/m
 2057     int dstenc = $dst$$reg;
 2058     if (dstenc < 8) {
 2059       emit_opcode(cbuf, Assembler::REX_W);
 2060     } else {
 2061       emit_opcode(cbuf, Assembler::REX_WB);
 2062       dstenc -= 8;
 2063     }
 2064     // Emit primary opcode and set sign-extend bit
 2065     // Check for 8-bit immediate, and set sign extend bit in opcode
 2066     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2067       emit_opcode(cbuf, $primary | 0x02);
 2068     } else {
 2069       // 32-bit immediate
 2070       emit_opcode(cbuf, $primary);
 2071     }
 2072     // Emit r/m byte with secondary opcode, after primary opcode.
 2073     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2074   %}
 2075 
 2076   enc_class Con8or32(immI imm)
 2077   %{
 2078     // Check for 8-bit immediate, and set sign extend bit in opcode
 2079     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2080       $$$emit8$imm$$constant;
 2081     } else {
 2082       // 32-bit immediate
 2083       $$$emit32$imm$$constant;
 2084     }
 2085   %}
 2086 
 2087   enc_class opc2_reg(rRegI dst)
 2088   %{
 2089     // BSWAP
 2090     emit_cc(cbuf, $secondary, $dst$$reg);
 2091   %}
 2092 
 2093   enc_class opc3_reg(rRegI dst)
 2094   %{
 2095     // BSWAP
 2096     emit_cc(cbuf, $tertiary, $dst$$reg);
 2097   %}
 2098 
 2099   enc_class reg_opc(rRegI div)
 2100   %{
 2101     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2102     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2103   %}
 2104 
 2105   enc_class enc_cmov(cmpOp cop)
 2106   %{
 2107     // CMOV
 2108     $$$emit8$primary;
 2109     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2110   %}
 2111 
 2112   enc_class enc_PartialSubtypeCheck()
 2113   %{
 2114     Register Rrdi = as_Register(RDI_enc); // result register
 2115     Register Rrax = as_Register(RAX_enc); // super class
 2116     Register Rrcx = as_Register(RCX_enc); // killed
 2117     Register Rrsi = as_Register(RSI_enc); // sub class
 2118     Label miss;
 2119     const bool set_cond_codes = true;
 2120 
 2121     MacroAssembler _masm(&cbuf);
 2122     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2123                                      NULL, &miss,
 2124                                      /*set_cond_codes:*/ true);
 2125     if ($primary) {
 2126       __ xorptr(Rrdi, Rrdi);
 2127     }
 2128     __ bind(miss);
 2129   %}
 2130 
 2131   enc_class clear_avx %{
 2132     debug_only(int off0 = cbuf.insts_size());
 2133     if (generate_vzeroupper(Compile::current())) {
 2134       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2135       // Clear upper bits of YMM registers when current compiled code uses
 2136       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2137       MacroAssembler _masm(&cbuf);
 2138       __ vzeroupper();
 2139     }
 2140     debug_only(int off1 = cbuf.insts_size());
 2141     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2142   %}
 2143 
 2144   enc_class Java_To_Runtime(method meth) %{
 2145     // No relocation needed
 2146     MacroAssembler _masm(&cbuf);
 2147     __ mov64(r10, (int64_t) $meth$$method);
 2148     __ call(r10);
 2149     __ post_call_nop();
 2150   %}
 2151 
 2152   enc_class Java_Static_Call(method meth)
 2153   %{
 2154     // JAVA STATIC CALL
 2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2156     // determine who we intended to call.
 2157     MacroAssembler _masm(&cbuf);
 2158     cbuf.set_insts_mark();
 2159     $$$emit8$primary;
 2160 
 2161     if (!_method) {
 2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2163                      runtime_call_Relocation::spec(),
 2164                      RELOC_DISP32);
 2165     } else {
 2166       int method_index = resolved_method_index(cbuf);
 2167       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2168                                                   : static_call_Relocation::spec(method_index);
 2169       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2170                      rspec, RELOC_DISP32);
 2171       address mark = cbuf.insts_mark();
 2172       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2173         // Calls of the same statically bound method can share
 2174         // a stub to the interpreter.
 2175         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2176       } else {
 2177         // Emit stubs for static call.
 2178         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2179         if (stub == NULL) {
 2180           ciEnv::current()->record_failure("CodeCache is full");
 2181           return;
 2182         }
 2183       }
 2184     }
 2185     _masm.clear_inst_mark();
 2186     __ post_call_nop();
 2187   %}
 2188 
 2189   enc_class Java_Dynamic_Call(method meth) %{
 2190     MacroAssembler _masm(&cbuf);
 2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2192     __ post_call_nop();
 2193   %}
 2194 
 2195   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2196   %{
 2197     // SAL, SAR, SHR
 2198     int dstenc = $dst$$reg;
 2199     if (dstenc >= 8) {
 2200       emit_opcode(cbuf, Assembler::REX_B);
 2201       dstenc -= 8;
 2202     }
 2203     $$$emit8$primary;
 2204     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2205     $$$emit8$shift$$constant;
 2206   %}
 2207 
 2208   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2209   %{
 2210     // SAL, SAR, SHR
 2211     int dstenc = $dst$$reg;
 2212     if (dstenc < 8) {
 2213       emit_opcode(cbuf, Assembler::REX_W);
 2214     } else {
 2215       emit_opcode(cbuf, Assembler::REX_WB);
 2216       dstenc -= 8;
 2217     }
 2218     $$$emit8$primary;
 2219     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2220     $$$emit8$shift$$constant;
 2221   %}
 2222 
 2223   enc_class load_immI(rRegI dst, immI src)
 2224   %{
 2225     int dstenc = $dst$$reg;
 2226     if (dstenc >= 8) {
 2227       emit_opcode(cbuf, Assembler::REX_B);
 2228       dstenc -= 8;
 2229     }
 2230     emit_opcode(cbuf, 0xB8 | dstenc);
 2231     $$$emit32$src$$constant;
 2232   %}
 2233 
 2234   enc_class load_immL(rRegL dst, immL src)
 2235   %{
 2236     int dstenc = $dst$$reg;
 2237     if (dstenc < 8) {
 2238       emit_opcode(cbuf, Assembler::REX_W);
 2239     } else {
 2240       emit_opcode(cbuf, Assembler::REX_WB);
 2241       dstenc -= 8;
 2242     }
 2243     emit_opcode(cbuf, 0xB8 | dstenc);
 2244     emit_d64(cbuf, $src$$constant);
 2245   %}
 2246 
 2247   enc_class load_immUL32(rRegL dst, immUL32 src)
 2248   %{
 2249     // same as load_immI, but this time we care about zeroes in the high word
 2250     int dstenc = $dst$$reg;
 2251     if (dstenc >= 8) {
 2252       emit_opcode(cbuf, Assembler::REX_B);
 2253       dstenc -= 8;
 2254     }
 2255     emit_opcode(cbuf, 0xB8 | dstenc);
 2256     $$$emit32$src$$constant;
 2257   %}
 2258 
 2259   enc_class load_immL32(rRegL dst, immL32 src)
 2260   %{
 2261     int dstenc = $dst$$reg;
 2262     if (dstenc < 8) {
 2263       emit_opcode(cbuf, Assembler::REX_W);
 2264     } else {
 2265       emit_opcode(cbuf, Assembler::REX_WB);
 2266       dstenc -= 8;
 2267     }
 2268     emit_opcode(cbuf, 0xC7);
 2269     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2270     $$$emit32$src$$constant;
 2271   %}
 2272 
 2273   enc_class load_immP31(rRegP dst, immP32 src)
 2274   %{
 2275     // same as load_immI, but this time we care about zeroes in the high word
 2276     int dstenc = $dst$$reg;
 2277     if (dstenc >= 8) {
 2278       emit_opcode(cbuf, Assembler::REX_B);
 2279       dstenc -= 8;
 2280     }
 2281     emit_opcode(cbuf, 0xB8 | dstenc);
 2282     $$$emit32$src$$constant;
 2283   %}
 2284 
 2285   enc_class load_immP(rRegP dst, immP src)
 2286   %{
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc < 8) {
 2289       emit_opcode(cbuf, Assembler::REX_W);
 2290     } else {
 2291       emit_opcode(cbuf, Assembler::REX_WB);
 2292       dstenc -= 8;
 2293     }
 2294     emit_opcode(cbuf, 0xB8 | dstenc);
 2295     // This next line should be generated from ADLC
 2296     if ($src->constant_reloc() != relocInfo::none) {
 2297       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2298     } else {
 2299       emit_d64(cbuf, $src$$constant);
 2300     }
 2301   %}
 2302 
 2303   enc_class Con32(immI src)
 2304   %{
 2305     // Output immediate
 2306     $$$emit32$src$$constant;
 2307   %}
 2308 
 2309   enc_class Con32F_as_bits(immF src)
 2310   %{
 2311     // Output Float immediate bits
 2312     jfloat jf = $src$$constant;
 2313     jint jf_as_bits = jint_cast(jf);
 2314     emit_d32(cbuf, jf_as_bits);
 2315   %}
 2316 
 2317   enc_class Con16(immI src)
 2318   %{
 2319     // Output immediate
 2320     $$$emit16$src$$constant;
 2321   %}
 2322 
 2323   // How is this different from Con32??? XXX
 2324   enc_class Con_d32(immI src)
 2325   %{
 2326     emit_d32(cbuf,$src$$constant);
 2327   %}
 2328 
 2329   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2330     // Output immediate memory reference
 2331     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2332     emit_d32(cbuf, 0x00);
 2333   %}
 2334 
 2335   enc_class lock_prefix()
 2336   %{
 2337     emit_opcode(cbuf, 0xF0); // lock
 2338   %}
 2339 
 2340   enc_class REX_mem(memory mem)
 2341   %{
 2342     if ($mem$$base >= 8) {
 2343       if ($mem$$index < 8) {
 2344         emit_opcode(cbuf, Assembler::REX_B);
 2345       } else {
 2346         emit_opcode(cbuf, Assembler::REX_XB);
 2347       }
 2348     } else {
 2349       if ($mem$$index >= 8) {
 2350         emit_opcode(cbuf, Assembler::REX_X);
 2351       }
 2352     }
 2353   %}
 2354 
 2355   enc_class REX_mem_wide(memory mem)
 2356   %{
 2357     if ($mem$$base >= 8) {
 2358       if ($mem$$index < 8) {
 2359         emit_opcode(cbuf, Assembler::REX_WB);
 2360       } else {
 2361         emit_opcode(cbuf, Assembler::REX_WXB);
 2362       }
 2363     } else {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_W);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_WX);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   // for byte regs
 2373   enc_class REX_breg(rRegI reg)
 2374   %{
 2375     if ($reg$$reg >= 4) {
 2376       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2382   %{
 2383     if ($dst$$reg < 8) {
 2384       if ($src$$reg >= 4) {
 2385         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2386       }
 2387     } else {
 2388       if ($src$$reg < 8) {
 2389         emit_opcode(cbuf, Assembler::REX_R);
 2390       } else {
 2391         emit_opcode(cbuf, Assembler::REX_RB);
 2392       }
 2393     }
 2394   %}
 2395 
 2396   // for byte regs
 2397   enc_class REX_breg_mem(rRegI reg, memory mem)
 2398   %{
 2399     if ($reg$$reg < 8) {
 2400       if ($mem$$base < 8) {
 2401         if ($mem$$index >= 8) {
 2402           emit_opcode(cbuf, Assembler::REX_X);
 2403         } else if ($reg$$reg >= 4) {
 2404           emit_opcode(cbuf, Assembler::REX);
 2405         }
 2406       } else {
 2407         if ($mem$$index < 8) {
 2408           emit_opcode(cbuf, Assembler::REX_B);
 2409         } else {
 2410           emit_opcode(cbuf, Assembler::REX_XB);
 2411         }
 2412       }
 2413     } else {
 2414       if ($mem$$base < 8) {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_R);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_RX);
 2419         }
 2420       } else {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_RB);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RXB);
 2425         }
 2426       }
 2427     }
 2428   %}
 2429 
 2430   enc_class REX_reg(rRegI reg)
 2431   %{
 2432     if ($reg$$reg >= 8) {
 2433       emit_opcode(cbuf, Assembler::REX_B);
 2434     }
 2435   %}
 2436 
 2437   enc_class REX_reg_wide(rRegI reg)
 2438   %{
 2439     if ($reg$$reg < 8) {
 2440       emit_opcode(cbuf, Assembler::REX_W);
 2441     } else {
 2442       emit_opcode(cbuf, Assembler::REX_WB);
 2443     }
 2444   %}
 2445 
 2446   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2447   %{
 2448     if ($dst$$reg < 8) {
 2449       if ($src$$reg >= 8) {
 2450         emit_opcode(cbuf, Assembler::REX_B);
 2451       }
 2452     } else {
 2453       if ($src$$reg < 8) {
 2454         emit_opcode(cbuf, Assembler::REX_R);
 2455       } else {
 2456         emit_opcode(cbuf, Assembler::REX_RB);
 2457       }
 2458     }
 2459   %}
 2460 
 2461   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2462   %{
 2463     if ($dst$$reg < 8) {
 2464       if ($src$$reg < 8) {
 2465         emit_opcode(cbuf, Assembler::REX_W);
 2466       } else {
 2467         emit_opcode(cbuf, Assembler::REX_WB);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_WR);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_WRB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_mem(rRegI reg, memory mem)
 2479   %{
 2480     if ($reg$$reg < 8) {
 2481       if ($mem$$base < 8) {
 2482         if ($mem$$index >= 8) {
 2483           emit_opcode(cbuf, Assembler::REX_X);
 2484         }
 2485       } else {
 2486         if ($mem$$index < 8) {
 2487           emit_opcode(cbuf, Assembler::REX_B);
 2488         } else {
 2489           emit_opcode(cbuf, Assembler::REX_XB);
 2490         }
 2491       }
 2492     } else {
 2493       if ($mem$$base < 8) {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_R);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_RX);
 2498         }
 2499       } else {
 2500         if ($mem$$index < 8) {
 2501           emit_opcode(cbuf, Assembler::REX_RB);
 2502         } else {
 2503           emit_opcode(cbuf, Assembler::REX_RXB);
 2504         }
 2505       }
 2506     }
 2507   %}
 2508 
 2509   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2510   %{
 2511     if ($reg$$reg < 8) {
 2512       if ($mem$$base < 8) {
 2513         if ($mem$$index < 8) {
 2514           emit_opcode(cbuf, Assembler::REX_W);
 2515         } else {
 2516           emit_opcode(cbuf, Assembler::REX_WX);
 2517         }
 2518       } else {
 2519         if ($mem$$index < 8) {
 2520           emit_opcode(cbuf, Assembler::REX_WB);
 2521         } else {
 2522           emit_opcode(cbuf, Assembler::REX_WXB);
 2523         }
 2524       }
 2525     } else {
 2526       if ($mem$$base < 8) {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WR);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WRX);
 2531         }
 2532       } else {
 2533         if ($mem$$index < 8) {
 2534           emit_opcode(cbuf, Assembler::REX_WRB);
 2535         } else {
 2536           emit_opcode(cbuf, Assembler::REX_WRXB);
 2537         }
 2538       }
 2539     }
 2540   %}
 2541 
 2542   enc_class reg_mem(rRegI ereg, memory mem)
 2543   %{
 2544     // High registers handle in encode_RegMem
 2545     int reg = $ereg$$reg;
 2546     int base = $mem$$base;
 2547     int index = $mem$$index;
 2548     int scale = $mem$$scale;
 2549     int disp = $mem$$disp;
 2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2551 
 2552     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2553   %}
 2554 
 2555   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2556   %{
 2557     int rm_byte_opcode = $rm_opcode$$constant;
 2558 
 2559     // High registers handle in encode_RegMem
 2560     int base = $mem$$base;
 2561     int index = $mem$$index;
 2562     int scale = $mem$$scale;
 2563     int displace = $mem$$disp;
 2564 
 2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2566                                             // working with static
 2567                                             // globals
 2568     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2569                   disp_reloc);
 2570   %}
 2571 
 2572   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2573   %{
 2574     int reg_encoding = $dst$$reg;
 2575     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2576     int index        = 0x04;            // 0x04 indicates no index
 2577     int scale        = 0x00;            // 0x00 indicates no scale
 2578     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2579     relocInfo::relocType disp_reloc = relocInfo::none;
 2580     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2581                   disp_reloc);
 2582   %}
 2583 
 2584   enc_class neg_reg(rRegI dst)
 2585   %{
 2586     int dstenc = $dst$$reg;
 2587     if (dstenc >= 8) {
 2588       emit_opcode(cbuf, Assembler::REX_B);
 2589       dstenc -= 8;
 2590     }
 2591     // NEG $dst
 2592     emit_opcode(cbuf, 0xF7);
 2593     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2594   %}
 2595 
 2596   enc_class neg_reg_wide(rRegI dst)
 2597   %{
 2598     int dstenc = $dst$$reg;
 2599     if (dstenc < 8) {
 2600       emit_opcode(cbuf, Assembler::REX_W);
 2601     } else {
 2602       emit_opcode(cbuf, Assembler::REX_WB);
 2603       dstenc -= 8;
 2604     }
 2605     // NEG $dst
 2606     emit_opcode(cbuf, 0xF7);
 2607     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2608   %}
 2609 
 2610   enc_class setLT_reg(rRegI dst)
 2611   %{
 2612     int dstenc = $dst$$reg;
 2613     if (dstenc >= 8) {
 2614       emit_opcode(cbuf, Assembler::REX_B);
 2615       dstenc -= 8;
 2616     } else if (dstenc >= 4) {
 2617       emit_opcode(cbuf, Assembler::REX);
 2618     }
 2619     // SETLT $dst
 2620     emit_opcode(cbuf, 0x0F);
 2621     emit_opcode(cbuf, 0x9C);
 2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2623   %}
 2624 
 2625   enc_class setNZ_reg(rRegI dst)
 2626   %{
 2627     int dstenc = $dst$$reg;
 2628     if (dstenc >= 8) {
 2629       emit_opcode(cbuf, Assembler::REX_B);
 2630       dstenc -= 8;
 2631     } else if (dstenc >= 4) {
 2632       emit_opcode(cbuf, Assembler::REX);
 2633     }
 2634     // SETNZ $dst
 2635     emit_opcode(cbuf, 0x0F);
 2636     emit_opcode(cbuf, 0x95);
 2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2638   %}
 2639 
 2640 
 2641   // Compare the lonogs and set -1, 0, or 1 into dst
 2642   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2643   %{
 2644     int src1enc = $src1$$reg;
 2645     int src2enc = $src2$$reg;
 2646     int dstenc = $dst$$reg;
 2647 
 2648     // cmpq $src1, $src2
 2649     if (src1enc < 8) {
 2650       if (src2enc < 8) {
 2651         emit_opcode(cbuf, Assembler::REX_W);
 2652       } else {
 2653         emit_opcode(cbuf, Assembler::REX_WB);
 2654       }
 2655     } else {
 2656       if (src2enc < 8) {
 2657         emit_opcode(cbuf, Assembler::REX_WR);
 2658       } else {
 2659         emit_opcode(cbuf, Assembler::REX_WRB);
 2660       }
 2661     }
 2662     emit_opcode(cbuf, 0x3B);
 2663     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2664 
 2665     // movl $dst, -1
 2666     if (dstenc >= 8) {
 2667       emit_opcode(cbuf, Assembler::REX_B);
 2668     }
 2669     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2670     emit_d32(cbuf, -1);
 2671 
 2672     // jl,s done
 2673     emit_opcode(cbuf, 0x7C);
 2674     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2675 
 2676     // setne $dst
 2677     if (dstenc >= 4) {
 2678       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2679     }
 2680     emit_opcode(cbuf, 0x0F);
 2681     emit_opcode(cbuf, 0x95);
 2682     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2683 
 2684     // movzbl $dst, $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0xB6);
 2690     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2691   %}
 2692 
 2693   enc_class Push_ResultXD(regD dst) %{
 2694     MacroAssembler _masm(&cbuf);
 2695     __ fstp_d(Address(rsp, 0));
 2696     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2697     __ addptr(rsp, 8);
 2698   %}
 2699 
 2700   enc_class Push_SrcXD(regD src) %{
 2701     MacroAssembler _masm(&cbuf);
 2702     __ subptr(rsp, 8);
 2703     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2704     __ fld_d(Address(rsp, 0));
 2705   %}
 2706 
 2707 
 2708   enc_class enc_rethrow()
 2709   %{
 2710     cbuf.set_insts_mark();
 2711     emit_opcode(cbuf, 0xE9); // jmp entry
 2712     emit_d32_reloc(cbuf,
 2713                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2714                    runtime_call_Relocation::spec(),
 2715                    RELOC_DISP32);
 2716   %}
 2717 
 2718 %}
 2719 
 2720 
 2721 
 2722 //----------FRAME--------------------------------------------------------------
 2723 // Definition of frame structure and management information.
 2724 //
 2725 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2726 //                             |   (to get allocators register number
 2727 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2728 //  r   CALLER     |        |
 2729 //  o     |        +--------+      pad to even-align allocators stack-slot
 2730 //  w     V        |  pad0  |        numbers; owned by CALLER
 2731 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2732 //  h     ^        |   in   |  5
 2733 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2734 //  |     |        |        |  3
 2735 //  |     |        +--------+
 2736 //  V     |        | old out|      Empty on Intel, window on Sparc
 2737 //        |    old |preserve|      Must be even aligned.
 2738 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2739 //        |        |   in   |  3   area for Intel ret address
 2740 //     Owned by    |preserve|      Empty on Sparc.
 2741 //       SELF      +--------+
 2742 //        |        |  pad2  |  2   pad to align old SP
 2743 //        |        +--------+  1
 2744 //        |        | locks  |  0
 2745 //        |        +--------+----> OptoReg::stack0(), even aligned
 2746 //        |        |  pad1  | 11   pad to align new SP
 2747 //        |        +--------+
 2748 //        |        |        | 10
 2749 //        |        | spills |  9   spills
 2750 //        V        |        |  8   (pad0 slot for callee)
 2751 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2752 //        ^        |  out   |  7
 2753 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2754 //     Owned by    +--------+
 2755 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2756 //        |    new |preserve|      Must be even-aligned.
 2757 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2758 //        |        |        |
 2759 //
 2760 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2761 //         known from SELF's arguments and the Java calling convention.
 2762 //         Region 6-7 is determined per call site.
 2763 // Note 2: If the calling convention leaves holes in the incoming argument
 2764 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2765 //         are owned by the CALLEE.  Holes should not be necessary in the
 2766 //         incoming area, as the Java calling convention is completely under
 2767 //         the control of the AD file.  Doubles can be sorted and packed to
 2768 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2769 //         varargs C calling conventions.
 2770 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2771 //         even aligned with pad0 as needed.
 2772 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2773 //         region 6-11 is even aligned; it may be padded out more so that
 2774 //         the region from SP to FP meets the minimum stack alignment.
 2775 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2776 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2777 //         SP meets the minimum alignment.
 2778 
 2779 frame
 2780 %{
 2781   // These three registers define part of the calling convention
 2782   // between compiled code and the interpreter.
 2783   inline_cache_reg(RAX);                // Inline Cache Register
 2784 
 2785   // Optional: name the operand used by cisc-spilling to access
 2786   // [stack_pointer + offset]
 2787   cisc_spilling_operand_name(indOffset32);
 2788 
 2789   // Number of stack slots consumed by locking an object
 2790   sync_stack_slots(2);
 2791 
 2792   // Compiled code's Frame Pointer
 2793   frame_pointer(RSP);
 2794 
 2795   // Interpreter stores its frame pointer in a register which is
 2796   // stored to the stack by I2CAdaptors.
 2797   // I2CAdaptors convert from interpreted java to compiled java.
 2798   interpreter_frame_pointer(RBP);
 2799 
 2800   // Stack alignment requirement
 2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2802 
 2803   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2804   // for calls to C.  Supports the var-args backing area for register parms.
 2805   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2806 
 2807   // The after-PROLOG location of the return address.  Location of
 2808   // return address specifies a type (REG or STACK) and a number
 2809   // representing the register number (i.e. - use a register name) or
 2810   // stack slot.
 2811   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2812   // Otherwise, it is above the locks and verification slot and alignment word
 2813   return_addr(STACK - 2 +
 2814               align_up((Compile::current()->in_preserve_stack_slots() +
 2815                         Compile::current()->fixed_slots()),
 2816                        stack_alignment_in_slots()));
 2817 
 2818   // Location of compiled Java return values.  Same as C for now.
 2819   return_value
 2820   %{
 2821     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2822            "only return normal values");
 2823 
 2824     static const int lo[Op_RegL + 1] = {
 2825       0,
 2826       0,
 2827       RAX_num,  // Op_RegN
 2828       RAX_num,  // Op_RegI
 2829       RAX_num,  // Op_RegP
 2830       XMM0_num, // Op_RegF
 2831       XMM0_num, // Op_RegD
 2832       RAX_num   // Op_RegL
 2833     };
 2834     static const int hi[Op_RegL + 1] = {
 2835       0,
 2836       0,
 2837       OptoReg::Bad, // Op_RegN
 2838       OptoReg::Bad, // Op_RegI
 2839       RAX_H_num,    // Op_RegP
 2840       OptoReg::Bad, // Op_RegF
 2841       XMM0b_num,    // Op_RegD
 2842       RAX_H_num     // Op_RegL
 2843     };
 2844     // Excluded flags and vector registers.
 2845     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2846     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2847   %}
 2848 %}
 2849 
 2850 //----------ATTRIBUTES---------------------------------------------------------
 2851 //----------Operand Attributes-------------------------------------------------
 2852 op_attrib op_cost(0);        // Required cost attribute
 2853 
 2854 //----------Instruction Attributes---------------------------------------------
 2855 ins_attrib ins_cost(100);       // Required cost attribute
 2856 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2857 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2858                                 // a non-matching short branch variant
 2859                                 // of some long branch?
 2860 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2861                                 // be a power of 2) specifies the
 2862                                 // alignment that some part of the
 2863                                 // instruction (not necessarily the
 2864                                 // start) requires.  If > 1, a
 2865                                 // compute_padding() function must be
 2866                                 // provided for the instruction
 2867 
 2868 //----------OPERANDS-----------------------------------------------------------
 2869 // Operand definitions must precede instruction definitions for correct parsing
 2870 // in the ADLC because operands constitute user defined types which are used in
 2871 // instruction definitions.
 2872 
 2873 //----------Simple Operands----------------------------------------------------
 2874 // Immediate Operands
 2875 // Integer Immediate
 2876 operand immI()
 2877 %{
 2878   match(ConI);
 2879 
 2880   op_cost(10);
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 // Constant for test vs zero
 2886 operand immI_0()
 2887 %{
 2888   predicate(n->get_int() == 0);
 2889   match(ConI);
 2890 
 2891   op_cost(0);
 2892   format %{ %}
 2893   interface(CONST_INTER);
 2894 %}
 2895 
 2896 // Constant for increment
 2897 operand immI_1()
 2898 %{
 2899   predicate(n->get_int() == 1);
 2900   match(ConI);
 2901 
 2902   op_cost(0);
 2903   format %{ %}
 2904   interface(CONST_INTER);
 2905 %}
 2906 
 2907 // Constant for decrement
 2908 operand immI_M1()
 2909 %{
 2910   predicate(n->get_int() == -1);
 2911   match(ConI);
 2912 
 2913   op_cost(0);
 2914   format %{ %}
 2915   interface(CONST_INTER);
 2916 %}
 2917 
 2918 operand immI_2()
 2919 %{
 2920   predicate(n->get_int() == 2);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 operand immI_4()
 2929 %{
 2930   predicate(n->get_int() == 4);
 2931   match(ConI);
 2932 
 2933   op_cost(0);
 2934   format %{ %}
 2935   interface(CONST_INTER);
 2936 %}
 2937 
 2938 operand immI_8()
 2939 %{
 2940   predicate(n->get_int() == 8);
 2941   match(ConI);
 2942 
 2943   op_cost(0);
 2944   format %{ %}
 2945   interface(CONST_INTER);
 2946 %}
 2947 
 2948 // Valid scale values for addressing modes
 2949 operand immI2()
 2950 %{
 2951   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2952   match(ConI);
 2953 
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 operand immU7()
 2959 %{
 2960   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2961   match(ConI);
 2962 
 2963   op_cost(5);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 operand immI8()
 2969 %{
 2970   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2971   match(ConI);
 2972 
 2973   op_cost(5);
 2974   format %{ %}
 2975   interface(CONST_INTER);
 2976 %}
 2977 
 2978 operand immU8()
 2979 %{
 2980   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2981   match(ConI);
 2982 
 2983   op_cost(5);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immI16()
 2989 %{
 2990   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2991   match(ConI);
 2992 
 2993   op_cost(10);
 2994   format %{ %}
 2995   interface(CONST_INTER);
 2996 %}
 2997 
 2998 // Int Immediate non-negative
 2999 operand immU31()
 3000 %{
 3001   predicate(n->get_int() >= 0);
 3002   match(ConI);
 3003 
 3004   op_cost(0);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 // Constant for long shifts
 3010 operand immI_32()
 3011 %{
 3012   predicate( n->get_int() == 32 );
 3013   match(ConI);
 3014 
 3015   op_cost(0);
 3016   format %{ %}
 3017   interface(CONST_INTER);
 3018 %}
 3019 
 3020 // Constant for long shifts
 3021 operand immI_64()
 3022 %{
 3023   predicate( n->get_int() == 64 );
 3024   match(ConI);
 3025 
 3026   op_cost(0);
 3027   format %{ %}
 3028   interface(CONST_INTER);
 3029 %}
 3030 
 3031 // Pointer Immediate
 3032 operand immP()
 3033 %{
 3034   match(ConP);
 3035 
 3036   op_cost(10);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // NULL Pointer Immediate
 3042 operand immP0()
 3043 %{
 3044   predicate(n->get_ptr() == 0);
 3045   match(ConP);
 3046 
 3047   op_cost(5);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immN() %{
 3054   match(ConN);
 3055 
 3056   op_cost(10);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immNKlass() %{
 3062   match(ConNKlass);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // NULL Pointer Immediate
 3070 operand immN0() %{
 3071   predicate(n->get_narrowcon() == 0);
 3072   match(ConN);
 3073 
 3074   op_cost(5);
 3075   format %{ %}
 3076   interface(CONST_INTER);
 3077 %}
 3078 
 3079 operand immP31()
 3080 %{
 3081   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3082             && (n->get_ptr() >> 31) == 0);
 3083   match(ConP);
 3084 
 3085   op_cost(5);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 
 3091 // Long Immediate
 3092 operand immL()
 3093 %{
 3094   match(ConL);
 3095 
 3096   op_cost(20);
 3097   format %{ %}
 3098   interface(CONST_INTER);
 3099 %}
 3100 
 3101 // Long Immediate 8-bit
 3102 operand immL8()
 3103 %{
 3104   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3105   match(ConL);
 3106 
 3107   op_cost(5);
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate 32-bit unsigned
 3113 operand immUL32()
 3114 %{
 3115   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3116   match(ConL);
 3117 
 3118   op_cost(10);
 3119   format %{ %}
 3120   interface(CONST_INTER);
 3121 %}
 3122 
 3123 // Long Immediate 32-bit signed
 3124 operand immL32()
 3125 %{
 3126   predicate(n->get_long() == (int) (n->get_long()));
 3127   match(ConL);
 3128 
 3129   op_cost(15);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 operand immL_Pow2()
 3135 %{
 3136   predicate(is_power_of_2((julong)n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(15);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 operand immL_NotPow2()
 3145 %{
 3146   predicate(is_power_of_2((julong)~n->get_long()));
 3147   match(ConL);
 3148 
 3149   op_cost(15);
 3150   format %{ %}
 3151   interface(CONST_INTER);
 3152 %}
 3153 
 3154 // Long Immediate zero
 3155 operand immL0()
 3156 %{
 3157   predicate(n->get_long() == 0L);
 3158   match(ConL);
 3159 
 3160   op_cost(10);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 // Constant for increment
 3166 operand immL1()
 3167 %{
 3168   predicate(n->get_long() == 1);
 3169   match(ConL);
 3170 
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Constant for decrement
 3176 operand immL_M1()
 3177 %{
 3178   predicate(n->get_long() == -1);
 3179   match(ConL);
 3180 
 3181   format %{ %}
 3182   interface(CONST_INTER);
 3183 %}
 3184 
 3185 // Long Immediate: the value 10
 3186 operand immL10()
 3187 %{
 3188   predicate(n->get_long() == 10);
 3189   match(ConL);
 3190 
 3191   format %{ %}
 3192   interface(CONST_INTER);
 3193 %}
 3194 
 3195 // Long immediate from 0 to 127.
 3196 // Used for a shorter form of long mul by 10.
 3197 operand immL_127()
 3198 %{
 3199   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3200   match(ConL);
 3201 
 3202   op_cost(10);
 3203   format %{ %}
 3204   interface(CONST_INTER);
 3205 %}
 3206 
 3207 // Long Immediate: low 32-bit mask
 3208 operand immL_32bits()
 3209 %{
 3210   predicate(n->get_long() == 0xFFFFFFFFL);
 3211   match(ConL);
 3212   op_cost(20);
 3213 
 3214   format %{ %}
 3215   interface(CONST_INTER);
 3216 %}
 3217 
 3218 // Int Immediate: 2^n-1, positive
 3219 operand immI_Pow2M1()
 3220 %{
 3221   predicate((n->get_int() > 0)
 3222             && is_power_of_2(n->get_int() + 1));
 3223   match(ConI);
 3224 
 3225   op_cost(20);
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Float Immediate zero
 3231 operand immF0()
 3232 %{
 3233   predicate(jint_cast(n->getf()) == 0);
 3234   match(ConF);
 3235 
 3236   op_cost(5);
 3237   format %{ %}
 3238   interface(CONST_INTER);
 3239 %}
 3240 
 3241 // Float Immediate
 3242 operand immF()
 3243 %{
 3244   match(ConF);
 3245 
 3246   op_cost(15);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Double Immediate zero
 3252 operand immD0()
 3253 %{
 3254   predicate(jlong_cast(n->getd()) == 0);
 3255   match(ConD);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Double Immediate
 3263 operand immD()
 3264 %{
 3265   match(ConD);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Immediates for special shifts (sign extend)
 3273 
 3274 // Constants for increment
 3275 operand immI_16()
 3276 %{
 3277   predicate(n->get_int() == 16);
 3278   match(ConI);
 3279 
 3280   format %{ %}
 3281   interface(CONST_INTER);
 3282 %}
 3283 
 3284 operand immI_24()
 3285 %{
 3286   predicate(n->get_int() == 24);
 3287   match(ConI);
 3288 
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Constant for byte-wide masking
 3294 operand immI_255()
 3295 %{
 3296   predicate(n->get_int() == 255);
 3297   match(ConI);
 3298 
 3299   format %{ %}
 3300   interface(CONST_INTER);
 3301 %}
 3302 
 3303 // Constant for short-wide masking
 3304 operand immI_65535()
 3305 %{
 3306   predicate(n->get_int() == 65535);
 3307   match(ConI);
 3308 
 3309   format %{ %}
 3310   interface(CONST_INTER);
 3311 %}
 3312 
 3313 // Constant for byte-wide masking
 3314 operand immL_255()
 3315 %{
 3316   predicate(n->get_long() == 255);
 3317   match(ConL);
 3318 
 3319   format %{ %}
 3320   interface(CONST_INTER);
 3321 %}
 3322 
 3323 // Constant for short-wide masking
 3324 operand immL_65535()
 3325 %{
 3326   predicate(n->get_long() == 65535);
 3327   match(ConL);
 3328 
 3329   format %{ %}
 3330   interface(CONST_INTER);
 3331 %}
 3332 
 3333 operand kReg()
 3334 %{
 3335   constraint(ALLOC_IN_RC(vectmask_reg));
 3336   match(RegVectMask);
 3337   format %{%}
 3338   interface(REG_INTER);
 3339 %}
 3340 
 3341 operand kReg_K1()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K2()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 // Special Registers
 3358 operand kReg_K3()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K4()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K5()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K6()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 // Special Registers
 3391 operand kReg_K7()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 // Register Operands
 3400 // Integer Register
 3401 operand rRegI()
 3402 %{
 3403   constraint(ALLOC_IN_RC(int_reg));
 3404   match(RegI);
 3405 
 3406   match(rax_RegI);
 3407   match(rbx_RegI);
 3408   match(rcx_RegI);
 3409   match(rdx_RegI);
 3410   match(rdi_RegI);
 3411 
 3412   format %{ %}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Special Registers
 3417 operand rax_RegI()
 3418 %{
 3419   constraint(ALLOC_IN_RC(int_rax_reg));
 3420   match(RegI);
 3421   match(rRegI);
 3422 
 3423   format %{ "RAX" %}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand rbx_RegI()
 3429 %{
 3430   constraint(ALLOC_IN_RC(int_rbx_reg));
 3431   match(RegI);
 3432   match(rRegI);
 3433 
 3434   format %{ "RBX" %}
 3435   interface(REG_INTER);
 3436 %}
 3437 
 3438 operand rcx_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rcx_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RCX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 operand rdx_RegI()
 3449 %{
 3450   constraint(ALLOC_IN_RC(int_rdx_reg));
 3451   match(RegI);
 3452   match(rRegI);
 3453 
 3454   format %{ "RDX" %}
 3455   interface(REG_INTER);
 3456 %}
 3457 
 3458 operand rdi_RegI()
 3459 %{
 3460   constraint(ALLOC_IN_RC(int_rdi_reg));
 3461   match(RegI);
 3462   match(rRegI);
 3463 
 3464   format %{ "RDI" %}
 3465   interface(REG_INTER);
 3466 %}
 3467 
 3468 operand no_rax_rdx_RegI()
 3469 %{
 3470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3471   match(RegI);
 3472   match(rbx_RegI);
 3473   match(rcx_RegI);
 3474   match(rdi_RegI);
 3475 
 3476   format %{ %}
 3477   interface(REG_INTER);
 3478 %}
 3479 
 3480 operand no_rbp_r13_RegI()
 3481 %{
 3482   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3483   match(RegI);
 3484   match(rRegI);
 3485   match(rax_RegI);
 3486   match(rbx_RegI);
 3487   match(rcx_RegI);
 3488   match(rdx_RegI);
 3489   match(rdi_RegI);
 3490 
 3491   format %{ %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 // Pointer Register
 3496 operand any_RegP()
 3497 %{
 3498   constraint(ALLOC_IN_RC(any_reg));
 3499   match(RegP);
 3500   match(rax_RegP);
 3501   match(rbx_RegP);
 3502   match(rdi_RegP);
 3503   match(rsi_RegP);
 3504   match(rbp_RegP);
 3505   match(r15_RegP);
 3506   match(rRegP);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 operand rRegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_reg));
 3515   match(RegP);
 3516   match(rax_RegP);
 3517   match(rbx_RegP);
 3518   match(rdi_RegP);
 3519   match(rsi_RegP);
 3520   match(rbp_RegP);  // See Q&A below about
 3521   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3522 
 3523   format %{ %}
 3524   interface(REG_INTER);
 3525 %}
 3526 
 3527 operand rRegN() %{
 3528   constraint(ALLOC_IN_RC(int_reg));
 3529   match(RegN);
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3536 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3537 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3538 // The output of an instruction is controlled by the allocator, which respects
 3539 // register class masks, not match rules.  Unless an instruction mentions
 3540 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3541 // by the allocator as an input.
 3542 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3543 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3544 // result, RBP is not included in the output of the instruction either.
 3545 
 3546 operand no_rax_RegP()
 3547 %{
 3548   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3549   match(RegP);
 3550   match(rbx_RegP);
 3551   match(rsi_RegP);
 3552   match(rdi_RegP);
 3553 
 3554   format %{ %}
 3555   interface(REG_INTER);
 3556 %}
 3557 
 3558 // This operand is not allowed to use RBP even if
 3559 // RBP is not used to hold the frame pointer.
 3560 operand no_rbp_RegP()
 3561 %{
 3562   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3563   match(RegP);
 3564   match(rbx_RegP);
 3565   match(rsi_RegP);
 3566   match(rdi_RegP);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 operand no_rax_rbx_RegP()
 3573 %{
 3574   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3575   match(RegP);
 3576   match(rsi_RegP);
 3577   match(rdi_RegP);
 3578 
 3579   format %{ %}
 3580   interface(REG_INTER);
 3581 %}
 3582 
 3583 // Special Registers
 3584 // Return a pointer value
 3585 operand rax_RegP()
 3586 %{
 3587   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3588   match(RegP);
 3589   match(rRegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // Special Registers
 3596 // Return a compressed pointer value
 3597 operand rax_RegN()
 3598 %{
 3599   constraint(ALLOC_IN_RC(int_rax_reg));
 3600   match(RegN);
 3601   match(rRegN);
 3602 
 3603   format %{ %}
 3604   interface(REG_INTER);
 3605 %}
 3606 
 3607 // Used in AtomicAdd
 3608 operand rbx_RegP()
 3609 %{
 3610   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3611   match(RegP);
 3612   match(rRegP);
 3613 
 3614   format %{ %}
 3615   interface(REG_INTER);
 3616 %}
 3617 
 3618 operand rsi_RegP()
 3619 %{
 3620   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3621   match(RegP);
 3622   match(rRegP);
 3623 
 3624   format %{ %}
 3625   interface(REG_INTER);
 3626 %}
 3627 
 3628 operand rbp_RegP()
 3629 %{
 3630   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3631   match(RegP);
 3632   match(rRegP);
 3633 
 3634   format %{ %}
 3635   interface(REG_INTER);
 3636 %}
 3637 
 3638 // Used in rep stosq
 3639 operand rdi_RegP()
 3640 %{
 3641   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3642   match(RegP);
 3643   match(rRegP);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand r15_RegP()
 3650 %{
 3651   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3652   match(RegP);
 3653   match(rRegP);
 3654 
 3655   format %{ %}
 3656   interface(REG_INTER);
 3657 %}
 3658 
 3659 operand rRegL()
 3660 %{
 3661   constraint(ALLOC_IN_RC(long_reg));
 3662   match(RegL);
 3663   match(rax_RegL);
 3664   match(rdx_RegL);
 3665 
 3666   format %{ %}
 3667   interface(REG_INTER);
 3668 %}
 3669 
 3670 // Special Registers
 3671 operand no_rax_rdx_RegL()
 3672 %{
 3673   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3674   match(RegL);
 3675   match(rRegL);
 3676 
 3677   format %{ %}
 3678   interface(REG_INTER);
 3679 %}
 3680 
 3681 operand rax_RegL()
 3682 %{
 3683   constraint(ALLOC_IN_RC(long_rax_reg));
 3684   match(RegL);
 3685   match(rRegL);
 3686 
 3687   format %{ "RAX" %}
 3688   interface(REG_INTER);
 3689 %}
 3690 
 3691 operand rcx_RegL()
 3692 %{
 3693   constraint(ALLOC_IN_RC(long_rcx_reg));
 3694   match(RegL);
 3695   match(rRegL);
 3696 
 3697   format %{ %}
 3698   interface(REG_INTER);
 3699 %}
 3700 
 3701 operand rdx_RegL()
 3702 %{
 3703   constraint(ALLOC_IN_RC(long_rdx_reg));
 3704   match(RegL);
 3705   match(rRegL);
 3706 
 3707   format %{ %}
 3708   interface(REG_INTER);
 3709 %}
 3710 
 3711 operand no_rbp_r13_RegL()
 3712 %{
 3713   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3714   match(RegL);
 3715   match(rRegL);
 3716   match(rax_RegL);
 3717   match(rcx_RegL);
 3718   match(rdx_RegL);
 3719 
 3720   format %{ %}
 3721   interface(REG_INTER);
 3722 %}
 3723 
 3724 // Flags register, used as output of compare instructions
 3725 operand rFlagsReg()
 3726 %{
 3727   constraint(ALLOC_IN_RC(int_flags));
 3728   match(RegFlags);
 3729 
 3730   format %{ "RFLAGS" %}
 3731   interface(REG_INTER);
 3732 %}
 3733 
 3734 // Flags register, used as output of FLOATING POINT compare instructions
 3735 operand rFlagsRegU()
 3736 %{
 3737   constraint(ALLOC_IN_RC(int_flags));
 3738   match(RegFlags);
 3739 
 3740   format %{ "RFLAGS_U" %}
 3741   interface(REG_INTER);
 3742 %}
 3743 
 3744 operand rFlagsRegUCF() %{
 3745   constraint(ALLOC_IN_RC(int_flags));
 3746   match(RegFlags);
 3747   predicate(false);
 3748 
 3749   format %{ "RFLAGS_U_CF" %}
 3750   interface(REG_INTER);
 3751 %}
 3752 
 3753 // Float register operands
 3754 operand regF() %{
 3755    constraint(ALLOC_IN_RC(float_reg));
 3756    match(RegF);
 3757 
 3758    format %{ %}
 3759    interface(REG_INTER);
 3760 %}
 3761 
 3762 // Float register operands
 3763 operand legRegF() %{
 3764    constraint(ALLOC_IN_RC(float_reg_legacy));
 3765    match(RegF);
 3766 
 3767    format %{ %}
 3768    interface(REG_INTER);
 3769 %}
 3770 
 3771 // Float register operands
 3772 operand vlRegF() %{
 3773    constraint(ALLOC_IN_RC(float_reg_vl));
 3774    match(RegF);
 3775 
 3776    format %{ %}
 3777    interface(REG_INTER);
 3778 %}
 3779 
 3780 // Double register operands
 3781 operand regD() %{
 3782    constraint(ALLOC_IN_RC(double_reg));
 3783    match(RegD);
 3784 
 3785    format %{ %}
 3786    interface(REG_INTER);
 3787 %}
 3788 
 3789 // Double register operands
 3790 operand legRegD() %{
 3791    constraint(ALLOC_IN_RC(double_reg_legacy));
 3792    match(RegD);
 3793 
 3794    format %{ %}
 3795    interface(REG_INTER);
 3796 %}
 3797 
 3798 // Double register operands
 3799 operand vlRegD() %{
 3800    constraint(ALLOC_IN_RC(double_reg_vl));
 3801    match(RegD);
 3802 
 3803    format %{ %}
 3804    interface(REG_INTER);
 3805 %}
 3806 
 3807 //----------Memory Operands----------------------------------------------------
 3808 // Direct Memory Operand
 3809 // operand direct(immP addr)
 3810 // %{
 3811 //   match(addr);
 3812 
 3813 //   format %{ "[$addr]" %}
 3814 //   interface(MEMORY_INTER) %{
 3815 //     base(0xFFFFFFFF);
 3816 //     index(0x4);
 3817 //     scale(0x0);
 3818 //     disp($addr);
 3819 //   %}
 3820 // %}
 3821 
 3822 // Indirect Memory Operand
 3823 operand indirect(any_RegP reg)
 3824 %{
 3825   constraint(ALLOC_IN_RC(ptr_reg));
 3826   match(reg);
 3827 
 3828   format %{ "[$reg]" %}
 3829   interface(MEMORY_INTER) %{
 3830     base($reg);
 3831     index(0x4);
 3832     scale(0x0);
 3833     disp(0x0);
 3834   %}
 3835 %}
 3836 
 3837 // Indirect Memory Plus Short Offset Operand
 3838 operand indOffset8(any_RegP reg, immL8 off)
 3839 %{
 3840   constraint(ALLOC_IN_RC(ptr_reg));
 3841   match(AddP reg off);
 3842 
 3843   format %{ "[$reg + $off (8-bit)]" %}
 3844   interface(MEMORY_INTER) %{
 3845     base($reg);
 3846     index(0x4);
 3847     scale(0x0);
 3848     disp($off);
 3849   %}
 3850 %}
 3851 
 3852 // Indirect Memory Plus Long Offset Operand
 3853 operand indOffset32(any_RegP reg, immL32 off)
 3854 %{
 3855   constraint(ALLOC_IN_RC(ptr_reg));
 3856   match(AddP reg off);
 3857 
 3858   format %{ "[$reg + $off (32-bit)]" %}
 3859   interface(MEMORY_INTER) %{
 3860     base($reg);
 3861     index(0x4);
 3862     scale(0x0);
 3863     disp($off);
 3864   %}
 3865 %}
 3866 
 3867 // Indirect Memory Plus Index Register Plus Offset Operand
 3868 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3869 %{
 3870   constraint(ALLOC_IN_RC(ptr_reg));
 3871   match(AddP (AddP reg lreg) off);
 3872 
 3873   op_cost(10);
 3874   format %{"[$reg + $off + $lreg]" %}
 3875   interface(MEMORY_INTER) %{
 3876     base($reg);
 3877     index($lreg);
 3878     scale(0x0);
 3879     disp($off);
 3880   %}
 3881 %}
 3882 
 3883 // Indirect Memory Plus Index Register Plus Offset Operand
 3884 operand indIndex(any_RegP reg, rRegL lreg)
 3885 %{
 3886   constraint(ALLOC_IN_RC(ptr_reg));
 3887   match(AddP reg lreg);
 3888 
 3889   op_cost(10);
 3890   format %{"[$reg + $lreg]" %}
 3891   interface(MEMORY_INTER) %{
 3892     base($reg);
 3893     index($lreg);
 3894     scale(0x0);
 3895     disp(0x0);
 3896   %}
 3897 %}
 3898 
 3899 // Indirect Memory Times Scale Plus Index Register
 3900 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3901 %{
 3902   constraint(ALLOC_IN_RC(ptr_reg));
 3903   match(AddP reg (LShiftL lreg scale));
 3904 
 3905   op_cost(10);
 3906   format %{"[$reg + $lreg << $scale]" %}
 3907   interface(MEMORY_INTER) %{
 3908     base($reg);
 3909     index($lreg);
 3910     scale($scale);
 3911     disp(0x0);
 3912   %}
 3913 %}
 3914 
 3915 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3916 %{
 3917   constraint(ALLOC_IN_RC(ptr_reg));
 3918   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3919   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3920 
 3921   op_cost(10);
 3922   format %{"[$reg + pos $idx << $scale]" %}
 3923   interface(MEMORY_INTER) %{
 3924     base($reg);
 3925     index($idx);
 3926     scale($scale);
 3927     disp(0x0);
 3928   %}
 3929 %}
 3930 
 3931 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3932 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3933 %{
 3934   constraint(ALLOC_IN_RC(ptr_reg));
 3935   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3936 
 3937   op_cost(10);
 3938   format %{"[$reg + $off + $lreg << $scale]" %}
 3939   interface(MEMORY_INTER) %{
 3940     base($reg);
 3941     index($lreg);
 3942     scale($scale);
 3943     disp($off);
 3944   %}
 3945 %}
 3946 
 3947 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3948 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3949 %{
 3950   constraint(ALLOC_IN_RC(ptr_reg));
 3951   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3952   match(AddP (AddP reg (ConvI2L idx)) off);
 3953 
 3954   op_cost(10);
 3955   format %{"[$reg + $off + $idx]" %}
 3956   interface(MEMORY_INTER) %{
 3957     base($reg);
 3958     index($idx);
 3959     scale(0x0);
 3960     disp($off);
 3961   %}
 3962 %}
 3963 
 3964 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3965 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3966 %{
 3967   constraint(ALLOC_IN_RC(ptr_reg));
 3968   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3969   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3970 
 3971   op_cost(10);
 3972   format %{"[$reg + $off + $idx << $scale]" %}
 3973   interface(MEMORY_INTER) %{
 3974     base($reg);
 3975     index($idx);
 3976     scale($scale);
 3977     disp($off);
 3978   %}
 3979 %}
 3980 
 3981 // Indirect Narrow Oop Plus Offset Operand
 3982 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3983 // we can't free r12 even with CompressedOops::base() == NULL.
 3984 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3985   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3986   constraint(ALLOC_IN_RC(ptr_reg));
 3987   match(AddP (DecodeN reg) off);
 3988 
 3989   op_cost(10);
 3990   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3991   interface(MEMORY_INTER) %{
 3992     base(0xc); // R12
 3993     index($reg);
 3994     scale(0x3);
 3995     disp($off);
 3996   %}
 3997 %}
 3998 
 3999 // Indirect Memory Operand
 4000 operand indirectNarrow(rRegN reg)
 4001 %{
 4002   predicate(CompressedOops::shift() == 0);
 4003   constraint(ALLOC_IN_RC(ptr_reg));
 4004   match(DecodeN reg);
 4005 
 4006   format %{ "[$reg]" %}
 4007   interface(MEMORY_INTER) %{
 4008     base($reg);
 4009     index(0x4);
 4010     scale(0x0);
 4011     disp(0x0);
 4012   %}
 4013 %}
 4014 
 4015 // Indirect Memory Plus Short Offset Operand
 4016 operand indOffset8Narrow(rRegN reg, immL8 off)
 4017 %{
 4018   predicate(CompressedOops::shift() == 0);
 4019   constraint(ALLOC_IN_RC(ptr_reg));
 4020   match(AddP (DecodeN reg) off);
 4021 
 4022   format %{ "[$reg + $off (8-bit)]" %}
 4023   interface(MEMORY_INTER) %{
 4024     base($reg);
 4025     index(0x4);
 4026     scale(0x0);
 4027     disp($off);
 4028   %}
 4029 %}
 4030 
 4031 // Indirect Memory Plus Long Offset Operand
 4032 operand indOffset32Narrow(rRegN reg, immL32 off)
 4033 %{
 4034   predicate(CompressedOops::shift() == 0);
 4035   constraint(ALLOC_IN_RC(ptr_reg));
 4036   match(AddP (DecodeN reg) off);
 4037 
 4038   format %{ "[$reg + $off (32-bit)]" %}
 4039   interface(MEMORY_INTER) %{
 4040     base($reg);
 4041     index(0x4);
 4042     scale(0x0);
 4043     disp($off);
 4044   %}
 4045 %}
 4046 
 4047 // Indirect Memory Plus Index Register Plus Offset Operand
 4048 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4049 %{
 4050   predicate(CompressedOops::shift() == 0);
 4051   constraint(ALLOC_IN_RC(ptr_reg));
 4052   match(AddP (AddP (DecodeN reg) lreg) off);
 4053 
 4054   op_cost(10);
 4055   format %{"[$reg + $off + $lreg]" %}
 4056   interface(MEMORY_INTER) %{
 4057     base($reg);
 4058     index($lreg);
 4059     scale(0x0);
 4060     disp($off);
 4061   %}
 4062 %}
 4063 
 4064 // Indirect Memory Plus Index Register Plus Offset Operand
 4065 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4066 %{
 4067   predicate(CompressedOops::shift() == 0);
 4068   constraint(ALLOC_IN_RC(ptr_reg));
 4069   match(AddP (DecodeN reg) lreg);
 4070 
 4071   op_cost(10);
 4072   format %{"[$reg + $lreg]" %}
 4073   interface(MEMORY_INTER) %{
 4074     base($reg);
 4075     index($lreg);
 4076     scale(0x0);
 4077     disp(0x0);
 4078   %}
 4079 %}
 4080 
 4081 // Indirect Memory Times Scale Plus Index Register
 4082 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4083 %{
 4084   predicate(CompressedOops::shift() == 0);
 4085   constraint(ALLOC_IN_RC(ptr_reg));
 4086   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4087 
 4088   op_cost(10);
 4089   format %{"[$reg + $lreg << $scale]" %}
 4090   interface(MEMORY_INTER) %{
 4091     base($reg);
 4092     index($lreg);
 4093     scale($scale);
 4094     disp(0x0);
 4095   %}
 4096 %}
 4097 
 4098 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4099 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4100 %{
 4101   predicate(CompressedOops::shift() == 0);
 4102   constraint(ALLOC_IN_RC(ptr_reg));
 4103   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4104 
 4105   op_cost(10);
 4106   format %{"[$reg + $off + $lreg << $scale]" %}
 4107   interface(MEMORY_INTER) %{
 4108     base($reg);
 4109     index($lreg);
 4110     scale($scale);
 4111     disp($off);
 4112   %}
 4113 %}
 4114 
 4115 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4116 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4117 %{
 4118   constraint(ALLOC_IN_RC(ptr_reg));
 4119   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4120   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4121 
 4122   op_cost(10);
 4123   format %{"[$reg + $off + $idx]" %}
 4124   interface(MEMORY_INTER) %{
 4125     base($reg);
 4126     index($idx);
 4127     scale(0x0);
 4128     disp($off);
 4129   %}
 4130 %}
 4131 
 4132 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4133 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4134 %{
 4135   constraint(ALLOC_IN_RC(ptr_reg));
 4136   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4137   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4138 
 4139   op_cost(10);
 4140   format %{"[$reg + $off + $idx << $scale]" %}
 4141   interface(MEMORY_INTER) %{
 4142     base($reg);
 4143     index($idx);
 4144     scale($scale);
 4145     disp($off);
 4146   %}
 4147 %}
 4148 
 4149 //----------Special Memory Operands--------------------------------------------
 4150 // Stack Slot Operand - This operand is used for loading and storing temporary
 4151 //                      values on the stack where a match requires a value to
 4152 //                      flow through memory.
 4153 operand stackSlotP(sRegP reg)
 4154 %{
 4155   constraint(ALLOC_IN_RC(stack_slots));
 4156   // No match rule because this operand is only generated in matching
 4157 
 4158   format %{ "[$reg]" %}
 4159   interface(MEMORY_INTER) %{
 4160     base(0x4);   // RSP
 4161     index(0x4);  // No Index
 4162     scale(0x0);  // No Scale
 4163     disp($reg);  // Stack Offset
 4164   %}
 4165 %}
 4166 
 4167 operand stackSlotI(sRegI reg)
 4168 %{
 4169   constraint(ALLOC_IN_RC(stack_slots));
 4170   // No match rule because this operand is only generated in matching
 4171 
 4172   format %{ "[$reg]" %}
 4173   interface(MEMORY_INTER) %{
 4174     base(0x4);   // RSP
 4175     index(0x4);  // No Index
 4176     scale(0x0);  // No Scale
 4177     disp($reg);  // Stack Offset
 4178   %}
 4179 %}
 4180 
 4181 operand stackSlotF(sRegF reg)
 4182 %{
 4183   constraint(ALLOC_IN_RC(stack_slots));
 4184   // No match rule because this operand is only generated in matching
 4185 
 4186   format %{ "[$reg]" %}
 4187   interface(MEMORY_INTER) %{
 4188     base(0x4);   // RSP
 4189     index(0x4);  // No Index
 4190     scale(0x0);  // No Scale
 4191     disp($reg);  // Stack Offset
 4192   %}
 4193 %}
 4194 
 4195 operand stackSlotD(sRegD reg)
 4196 %{
 4197   constraint(ALLOC_IN_RC(stack_slots));
 4198   // No match rule because this operand is only generated in matching
 4199 
 4200   format %{ "[$reg]" %}
 4201   interface(MEMORY_INTER) %{
 4202     base(0x4);   // RSP
 4203     index(0x4);  // No Index
 4204     scale(0x0);  // No Scale
 4205     disp($reg);  // Stack Offset
 4206   %}
 4207 %}
 4208 operand stackSlotL(sRegL reg)
 4209 %{
 4210   constraint(ALLOC_IN_RC(stack_slots));
 4211   // No match rule because this operand is only generated in matching
 4212 
 4213   format %{ "[$reg]" %}
 4214   interface(MEMORY_INTER) %{
 4215     base(0x4);   // RSP
 4216     index(0x4);  // No Index
 4217     scale(0x0);  // No Scale
 4218     disp($reg);  // Stack Offset
 4219   %}
 4220 %}
 4221 
 4222 //----------Conditional Branch Operands----------------------------------------
 4223 // Comparison Op  - This is the operation of the comparison, and is limited to
 4224 //                  the following set of codes:
 4225 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4226 //
 4227 // Other attributes of the comparison, such as unsignedness, are specified
 4228 // by the comparison instruction that sets a condition code flags register.
 4229 // That result is represented by a flags operand whose subtype is appropriate
 4230 // to the unsignedness (etc.) of the comparison.
 4231 //
 4232 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4233 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4234 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4235 
 4236 // Comparison Code
 4237 operand cmpOp()
 4238 %{
 4239   match(Bool);
 4240 
 4241   format %{ "" %}
 4242   interface(COND_INTER) %{
 4243     equal(0x4, "e");
 4244     not_equal(0x5, "ne");
 4245     less(0xC, "l");
 4246     greater_equal(0xD, "ge");
 4247     less_equal(0xE, "le");
 4248     greater(0xF, "g");
 4249     overflow(0x0, "o");
 4250     no_overflow(0x1, "no");
 4251   %}
 4252 %}
 4253 
 4254 // Comparison Code, unsigned compare.  Used by FP also, with
 4255 // C2 (unordered) turned into GT or LT already.  The other bits
 4256 // C0 and C3 are turned into Carry & Zero flags.
 4257 operand cmpOpU()
 4258 %{
 4259   match(Bool);
 4260 
 4261   format %{ "" %}
 4262   interface(COND_INTER) %{
 4263     equal(0x4, "e");
 4264     not_equal(0x5, "ne");
 4265     less(0x2, "b");
 4266     greater_equal(0x3, "ae");
 4267     less_equal(0x6, "be");
 4268     greater(0x7, "a");
 4269     overflow(0x0, "o");
 4270     no_overflow(0x1, "no");
 4271   %}
 4272 %}
 4273 
 4274 
 4275 // Floating comparisons that don't require any fixup for the unordered case,
 4276 // If both inputs of the comparison are the same, ZF is always set so we
 4277 // don't need to use cmpOpUCF2 for eq/ne
 4278 operand cmpOpUCF() %{
 4279   match(Bool);
 4280   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4281             n->as_Bool()->_test._test == BoolTest::ge ||
 4282             n->as_Bool()->_test._test == BoolTest::le ||
 4283             n->as_Bool()->_test._test == BoolTest::gt ||
 4284             n->in(1)->in(1) == n->in(1)->in(2));
 4285   format %{ "" %}
 4286   interface(COND_INTER) %{
 4287     equal(0xb, "np");
 4288     not_equal(0xa, "p");
 4289     less(0x2, "b");
 4290     greater_equal(0x3, "ae");
 4291     less_equal(0x6, "be");
 4292     greater(0x7, "a");
 4293     overflow(0x0, "o");
 4294     no_overflow(0x1, "no");
 4295   %}
 4296 %}
 4297 
 4298 
 4299 // Floating comparisons that can be fixed up with extra conditional jumps
 4300 operand cmpOpUCF2() %{
 4301   match(Bool);
 4302   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4303              n->as_Bool()->_test._test == BoolTest::eq) &&
 4304             n->in(1)->in(1) != n->in(1)->in(2));
 4305   format %{ "" %}
 4306   interface(COND_INTER) %{
 4307     equal(0x4, "e");
 4308     not_equal(0x5, "ne");
 4309     less(0x2, "b");
 4310     greater_equal(0x3, "ae");
 4311     less_equal(0x6, "be");
 4312     greater(0x7, "a");
 4313     overflow(0x0, "o");
 4314     no_overflow(0x1, "no");
 4315   %}
 4316 %}
 4317 
 4318 //----------OPERAND CLASSES----------------------------------------------------
 4319 // Operand Classes are groups of operands that are used as to simplify
 4320 // instruction definitions by not requiring the AD writer to specify separate
 4321 // instructions for every form of operand when the instruction accepts
 4322 // multiple operand types with the same basic encoding and format.  The classic
 4323 // case of this is memory operands.
 4324 
 4325 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4326                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4327                indCompressedOopOffset,
 4328                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4329                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4330                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4331 
 4332 //----------PIPELINE-----------------------------------------------------------
 4333 // Rules which define the behavior of the target architectures pipeline.
 4334 pipeline %{
 4335 
 4336 //----------ATTRIBUTES---------------------------------------------------------
 4337 attributes %{
 4338   variable_size_instructions;        // Fixed size instructions
 4339   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4340   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4341   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4342   instruction_fetch_units = 1;       // of 16 bytes
 4343 
 4344   // List of nop instructions
 4345   nops( MachNop );
 4346 %}
 4347 
 4348 //----------RESOURCES----------------------------------------------------------
 4349 // Resources are the functional units available to the machine
 4350 
 4351 // Generic P2/P3 pipeline
 4352 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4353 // 3 instructions decoded per cycle.
 4354 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4355 // 3 ALU op, only ALU0 handles mul instructions.
 4356 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4357            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4358            BR, FPU,
 4359            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4360 
 4361 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4362 // Pipeline Description specifies the stages in the machine's pipeline
 4363 
 4364 // Generic P2/P3 pipeline
 4365 pipe_desc(S0, S1, S2, S3, S4, S5);
 4366 
 4367 //----------PIPELINE CLASSES---------------------------------------------------
 4368 // Pipeline Classes describe the stages in which input and output are
 4369 // referenced by the hardware pipeline.
 4370 
 4371 // Naming convention: ialu or fpu
 4372 // Then: _reg
 4373 // Then: _reg if there is a 2nd register
 4374 // Then: _long if it's a pair of instructions implementing a long
 4375 // Then: _fat if it requires the big decoder
 4376 //   Or: _mem if it requires the big decoder and a memory unit.
 4377 
 4378 // Integer ALU reg operation
 4379 pipe_class ialu_reg(rRegI dst)
 4380 %{
 4381     single_instruction;
 4382     dst    : S4(write);
 4383     dst    : S3(read);
 4384     DECODE : S0;        // any decoder
 4385     ALU    : S3;        // any alu
 4386 %}
 4387 
 4388 // Long ALU reg operation
 4389 pipe_class ialu_reg_long(rRegL dst)
 4390 %{
 4391     instruction_count(2);
 4392     dst    : S4(write);
 4393     dst    : S3(read);
 4394     DECODE : S0(2);     // any 2 decoders
 4395     ALU    : S3(2);     // both alus
 4396 %}
 4397 
 4398 // Integer ALU reg operation using big decoder
 4399 pipe_class ialu_reg_fat(rRegI dst)
 4400 %{
 4401     single_instruction;
 4402     dst    : S4(write);
 4403     dst    : S3(read);
 4404     D0     : S0;        // big decoder only
 4405     ALU    : S3;        // any alu
 4406 %}
 4407 
 4408 // Integer ALU reg-reg operation
 4409 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4410 %{
 4411     single_instruction;
 4412     dst    : S4(write);
 4413     src    : S3(read);
 4414     DECODE : S0;        // any decoder
 4415     ALU    : S3;        // any alu
 4416 %}
 4417 
 4418 // Integer ALU reg-reg operation
 4419 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4420 %{
 4421     single_instruction;
 4422     dst    : S4(write);
 4423     src    : S3(read);
 4424     D0     : S0;        // big decoder only
 4425     ALU    : S3;        // any alu
 4426 %}
 4427 
 4428 // Integer ALU reg-mem operation
 4429 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4430 %{
 4431     single_instruction;
 4432     dst    : S5(write);
 4433     mem    : S3(read);
 4434     D0     : S0;        // big decoder only
 4435     ALU    : S4;        // any alu
 4436     MEM    : S3;        // any mem
 4437 %}
 4438 
 4439 // Integer mem operation (prefetch)
 4440 pipe_class ialu_mem(memory mem)
 4441 %{
 4442     single_instruction;
 4443     mem    : S3(read);
 4444     D0     : S0;        // big decoder only
 4445     MEM    : S3;        // any mem
 4446 %}
 4447 
 4448 // Integer Store to Memory
 4449 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4450 %{
 4451     single_instruction;
 4452     mem    : S3(read);
 4453     src    : S5(read);
 4454     D0     : S0;        // big decoder only
 4455     ALU    : S4;        // any alu
 4456     MEM    : S3;
 4457 %}
 4458 
 4459 // // Long Store to Memory
 4460 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4461 // %{
 4462 //     instruction_count(2);
 4463 //     mem    : S3(read);
 4464 //     src    : S5(read);
 4465 //     D0     : S0(2);          // big decoder only; twice
 4466 //     ALU    : S4(2);     // any 2 alus
 4467 //     MEM    : S3(2);  // Both mems
 4468 // %}
 4469 
 4470 // Integer Store to Memory
 4471 pipe_class ialu_mem_imm(memory mem)
 4472 %{
 4473     single_instruction;
 4474     mem    : S3(read);
 4475     D0     : S0;        // big decoder only
 4476     ALU    : S4;        // any alu
 4477     MEM    : S3;
 4478 %}
 4479 
 4480 // Integer ALU0 reg-reg operation
 4481 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4482 %{
 4483     single_instruction;
 4484     dst    : S4(write);
 4485     src    : S3(read);
 4486     D0     : S0;        // Big decoder only
 4487     ALU0   : S3;        // only alu0
 4488 %}
 4489 
 4490 // Integer ALU0 reg-mem operation
 4491 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4492 %{
 4493     single_instruction;
 4494     dst    : S5(write);
 4495     mem    : S3(read);
 4496     D0     : S0;        // big decoder only
 4497     ALU0   : S4;        // ALU0 only
 4498     MEM    : S3;        // any mem
 4499 %}
 4500 
 4501 // Integer ALU reg-reg operation
 4502 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4503 %{
 4504     single_instruction;
 4505     cr     : S4(write);
 4506     src1   : S3(read);
 4507     src2   : S3(read);
 4508     DECODE : S0;        // any decoder
 4509     ALU    : S3;        // any alu
 4510 %}
 4511 
 4512 // Integer ALU reg-imm operation
 4513 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     DECODE : S0;        // any decoder
 4519     ALU    : S3;        // any alu
 4520 %}
 4521 
 4522 // Integer ALU reg-mem operation
 4523 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4524 %{
 4525     single_instruction;
 4526     cr     : S4(write);
 4527     src1   : S3(read);
 4528     src2   : S3(read);
 4529     D0     : S0;        // big decoder only
 4530     ALU    : S4;        // any alu
 4531     MEM    : S3;
 4532 %}
 4533 
 4534 // Conditional move reg-reg
 4535 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4536 %{
 4537     instruction_count(4);
 4538     y      : S4(read);
 4539     q      : S3(read);
 4540     p      : S3(read);
 4541     DECODE : S0(4);     // any decoder
 4542 %}
 4543 
 4544 // Conditional move reg-reg
 4545 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4546 %{
 4547     single_instruction;
 4548     dst    : S4(write);
 4549     src    : S3(read);
 4550     cr     : S3(read);
 4551     DECODE : S0;        // any decoder
 4552 %}
 4553 
 4554 // Conditional move reg-mem
 4555 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4556 %{
 4557     single_instruction;
 4558     dst    : S4(write);
 4559     src    : S3(read);
 4560     cr     : S3(read);
 4561     DECODE : S0;        // any decoder
 4562     MEM    : S3;
 4563 %}
 4564 
 4565 // Conditional move reg-reg long
 4566 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0(2);     // any 2 decoders
 4573 %}
 4574 
 4575 // XXX
 4576 // // Conditional move double reg-reg
 4577 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4578 // %{
 4579 //     single_instruction;
 4580 //     dst    : S4(write);
 4581 //     src    : S3(read);
 4582 //     cr     : S3(read);
 4583 //     DECODE : S0;     // any decoder
 4584 // %}
 4585 
 4586 // Float reg-reg operation
 4587 pipe_class fpu_reg(regD dst)
 4588 %{
 4589     instruction_count(2);
 4590     dst    : S3(read);
 4591     DECODE : S0(2);     // any 2 decoders
 4592     FPU    : S3;
 4593 %}
 4594 
 4595 // Float reg-reg operation
 4596 pipe_class fpu_reg_reg(regD dst, regD src)
 4597 %{
 4598     instruction_count(2);
 4599     dst    : S4(write);
 4600     src    : S3(read);
 4601     DECODE : S0(2);     // any 2 decoders
 4602     FPU    : S3;
 4603 %}
 4604 
 4605 // Float reg-reg operation
 4606 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4607 %{
 4608     instruction_count(3);
 4609     dst    : S4(write);
 4610     src1   : S3(read);
 4611     src2   : S3(read);
 4612     DECODE : S0(3);     // any 3 decoders
 4613     FPU    : S3(2);
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4618 %{
 4619     instruction_count(4);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     src3   : S3(read);
 4624     DECODE : S0(4);     // any 3 decoders
 4625     FPU    : S3(2);
 4626 %}
 4627 
 4628 // Float reg-reg operation
 4629 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4630 %{
 4631     instruction_count(4);
 4632     dst    : S4(write);
 4633     src1   : S3(read);
 4634     src2   : S3(read);
 4635     src3   : S3(read);
 4636     DECODE : S1(3);     // any 3 decoders
 4637     D0     : S0;        // Big decoder only
 4638     FPU    : S3(2);
 4639     MEM    : S3;
 4640 %}
 4641 
 4642 // Float reg-mem operation
 4643 pipe_class fpu_reg_mem(regD dst, memory mem)
 4644 %{
 4645     instruction_count(2);
 4646     dst    : S5(write);
 4647     mem    : S3(read);
 4648     D0     : S0;        // big decoder only
 4649     DECODE : S1;        // any decoder for FPU POP
 4650     FPU    : S4;
 4651     MEM    : S3;        // any mem
 4652 %}
 4653 
 4654 // Float reg-mem operation
 4655 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4656 %{
 4657     instruction_count(3);
 4658     dst    : S5(write);
 4659     src1   : S3(read);
 4660     mem    : S3(read);
 4661     D0     : S0;        // big decoder only
 4662     DECODE : S1(2);     // any decoder for FPU POP
 4663     FPU    : S4;
 4664     MEM    : S3;        // any mem
 4665 %}
 4666 
 4667 // Float mem-reg operation
 4668 pipe_class fpu_mem_reg(memory mem, regD src)
 4669 %{
 4670     instruction_count(2);
 4671     src    : S5(read);
 4672     mem    : S3(read);
 4673     DECODE : S0;        // any decoder for FPU PUSH
 4674     D0     : S1;        // big decoder only
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4680 %{
 4681     instruction_count(3);
 4682     src1   : S3(read);
 4683     src2   : S3(read);
 4684     mem    : S3(read);
 4685     DECODE : S0(2);     // any decoder for FPU PUSH
 4686     D0     : S1;        // big decoder only
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4692 %{
 4693     instruction_count(3);
 4694     src1   : S3(read);
 4695     src2   : S3(read);
 4696     mem    : S4(read);
 4697     DECODE : S0;        // any decoder for FPU PUSH
 4698     D0     : S0(2);     // big decoder only
 4699     FPU    : S4;
 4700     MEM    : S3(2);     // any mem
 4701 %}
 4702 
 4703 pipe_class fpu_mem_mem(memory dst, memory src1)
 4704 %{
 4705     instruction_count(2);
 4706     src1   : S3(read);
 4707     dst    : S4(read);
 4708     D0     : S0(2);     // big decoder only
 4709     MEM    : S3(2);     // any mem
 4710 %}
 4711 
 4712 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4713 %{
 4714     instruction_count(3);
 4715     src1   : S3(read);
 4716     src2   : S3(read);
 4717     dst    : S4(read);
 4718     D0     : S0(3);     // big decoder only
 4719     FPU    : S4;
 4720     MEM    : S3(3);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S4(read);
 4727     mem    : S4(read);
 4728     DECODE : S0;        // any decoder for FPU PUSH
 4729     D0     : S0(2);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(2);     // any mem
 4732 %}
 4733 
 4734 // Float load constant
 4735 pipe_class fpu_reg_con(regD dst)
 4736 %{
 4737     instruction_count(2);
 4738     dst    : S5(write);
 4739     D0     : S0;        // big decoder only for the load
 4740     DECODE : S1;        // any decoder for FPU POP
 4741     FPU    : S4;
 4742     MEM    : S3;        // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4747 %{
 4748     instruction_count(3);
 4749     dst    : S5(write);
 4750     src    : S3(read);
 4751     D0     : S0;        // big decoder only for the load
 4752     DECODE : S1(2);     // any decoder for FPU POP
 4753     FPU    : S4;
 4754     MEM    : S3;        // any mem
 4755 %}
 4756 
 4757 // UnConditional branch
 4758 pipe_class pipe_jmp(label labl)
 4759 %{
 4760     single_instruction;
 4761     BR   : S3;
 4762 %}
 4763 
 4764 // Conditional branch
 4765 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4766 %{
 4767     single_instruction;
 4768     cr    : S1(read);
 4769     BR    : S3;
 4770 %}
 4771 
 4772 // Allocation idiom
 4773 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4774 %{
 4775     instruction_count(1); force_serialization;
 4776     fixed_latency(6);
 4777     heap_ptr : S3(read);
 4778     DECODE   : S0(3);
 4779     D0       : S2;
 4780     MEM      : S3;
 4781     ALU      : S3(2);
 4782     dst      : S5(write);
 4783     BR       : S5;
 4784 %}
 4785 
 4786 // Generic big/slow expanded idiom
 4787 pipe_class pipe_slow()
 4788 %{
 4789     instruction_count(10); multiple_bundles; force_serialization;
 4790     fixed_latency(100);
 4791     D0  : S0(2);
 4792     MEM : S3(2);
 4793 %}
 4794 
 4795 // The real do-nothing guy
 4796 pipe_class empty()
 4797 %{
 4798     instruction_count(0);
 4799 %}
 4800 
 4801 // Define the class for the Nop node
 4802 define
 4803 %{
 4804    MachNop = empty;
 4805 %}
 4806 
 4807 %}
 4808 
 4809 //----------INSTRUCTIONS-------------------------------------------------------
 4810 //
 4811 // match      -- States which machine-independent subtree may be replaced
 4812 //               by this instruction.
 4813 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4814 //               selection to identify a minimum cost tree of machine
 4815 //               instructions that matches a tree of machine-independent
 4816 //               instructions.
 4817 // format     -- A string providing the disassembly for this instruction.
 4818 //               The value of an instruction's operand may be inserted
 4819 //               by referring to it with a '$' prefix.
 4820 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4821 //               to within an encode class as $primary, $secondary, and $tertiary
 4822 //               rrspectively.  The primary opcode is commonly used to
 4823 //               indicate the type of machine instruction, while secondary
 4824 //               and tertiary are often used for prefix options or addressing
 4825 //               modes.
 4826 // ins_encode -- A list of encode classes with parameters. The encode class
 4827 //               name must have been defined in an 'enc_class' specification
 4828 //               in the encode section of the architecture description.
 4829 
 4830 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4831 // Load Float
 4832 instruct MoveF2VL(vlRegF dst, regF src) %{
 4833   match(Set dst src);
 4834   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4835   ins_encode %{
 4836     ShouldNotReachHere();
 4837   %}
 4838   ins_pipe( fpu_reg_reg );
 4839 %}
 4840 
 4841 // Load Float
 4842 instruct MoveF2LEG(legRegF dst, regF src) %{
 4843   match(Set dst src);
 4844   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4845   ins_encode %{
 4846     ShouldNotReachHere();
 4847   %}
 4848   ins_pipe( fpu_reg_reg );
 4849 %}
 4850 
 4851 // Load Float
 4852 instruct MoveVL2F(regF dst, vlRegF src) %{
 4853   match(Set dst src);
 4854   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4855   ins_encode %{
 4856     ShouldNotReachHere();
 4857   %}
 4858   ins_pipe( fpu_reg_reg );
 4859 %}
 4860 
 4861 // Load Float
 4862 instruct MoveLEG2F(regF dst, legRegF src) %{
 4863   match(Set dst src);
 4864   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4865   ins_encode %{
 4866     ShouldNotReachHere();
 4867   %}
 4868   ins_pipe( fpu_reg_reg );
 4869 %}
 4870 
 4871 // Load Double
 4872 instruct MoveD2VL(vlRegD dst, regD src) %{
 4873   match(Set dst src);
 4874   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4875   ins_encode %{
 4876     ShouldNotReachHere();
 4877   %}
 4878   ins_pipe( fpu_reg_reg );
 4879 %}
 4880 
 4881 // Load Double
 4882 instruct MoveD2LEG(legRegD dst, regD src) %{
 4883   match(Set dst src);
 4884   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4885   ins_encode %{
 4886     ShouldNotReachHere();
 4887   %}
 4888   ins_pipe( fpu_reg_reg );
 4889 %}
 4890 
 4891 // Load Double
 4892 instruct MoveVL2D(regD dst, vlRegD src) %{
 4893   match(Set dst src);
 4894   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4895   ins_encode %{
 4896     ShouldNotReachHere();
 4897   %}
 4898   ins_pipe( fpu_reg_reg );
 4899 %}
 4900 
 4901 // Load Double
 4902 instruct MoveLEG2D(regD dst, legRegD src) %{
 4903   match(Set dst src);
 4904   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4905   ins_encode %{
 4906     ShouldNotReachHere();
 4907   %}
 4908   ins_pipe( fpu_reg_reg );
 4909 %}
 4910 
 4911 //----------Load/Store/Move Instructions---------------------------------------
 4912 //----------Load Instructions--------------------------------------------------
 4913 
 4914 // Load Byte (8 bit signed)
 4915 instruct loadB(rRegI dst, memory mem)
 4916 %{
 4917   match(Set dst (LoadB mem));
 4918 
 4919   ins_cost(125);
 4920   format %{ "movsbl  $dst, $mem\t# byte" %}
 4921 
 4922   ins_encode %{
 4923     __ movsbl($dst$$Register, $mem$$Address);
 4924   %}
 4925 
 4926   ins_pipe(ialu_reg_mem);
 4927 %}
 4928 
 4929 // Load Byte (8 bit signed) into Long Register
 4930 instruct loadB2L(rRegL dst, memory mem)
 4931 %{
 4932   match(Set dst (ConvI2L (LoadB mem)));
 4933 
 4934   ins_cost(125);
 4935   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4936 
 4937   ins_encode %{
 4938     __ movsbq($dst$$Register, $mem$$Address);
 4939   %}
 4940 
 4941   ins_pipe(ialu_reg_mem);
 4942 %}
 4943 
 4944 // Load Unsigned Byte (8 bit UNsigned)
 4945 instruct loadUB(rRegI dst, memory mem)
 4946 %{
 4947   match(Set dst (LoadUB mem));
 4948 
 4949   ins_cost(125);
 4950   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4951 
 4952   ins_encode %{
 4953     __ movzbl($dst$$Register, $mem$$Address);
 4954   %}
 4955 
 4956   ins_pipe(ialu_reg_mem);
 4957 %}
 4958 
 4959 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4960 instruct loadUB2L(rRegL dst, memory mem)
 4961 %{
 4962   match(Set dst (ConvI2L (LoadUB mem)));
 4963 
 4964   ins_cost(125);
 4965   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4966 
 4967   ins_encode %{
 4968     __ movzbq($dst$$Register, $mem$$Address);
 4969   %}
 4970 
 4971   ins_pipe(ialu_reg_mem);
 4972 %}
 4973 
 4974 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4975 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4976   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4977   effect(KILL cr);
 4978 
 4979   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4980             "andl    $dst, right_n_bits($mask, 8)" %}
 4981   ins_encode %{
 4982     Register Rdst = $dst$$Register;
 4983     __ movzbq(Rdst, $mem$$Address);
 4984     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4985   %}
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 // Load Short (16 bit signed)
 4990 instruct loadS(rRegI dst, memory mem)
 4991 %{
 4992   match(Set dst (LoadS mem));
 4993 
 4994   ins_cost(125);
 4995   format %{ "movswl $dst, $mem\t# short" %}
 4996 
 4997   ins_encode %{
 4998     __ movswl($dst$$Register, $mem$$Address);
 4999   %}
 5000 
 5001   ins_pipe(ialu_reg_mem);
 5002 %}
 5003 
 5004 // Load Short (16 bit signed) to Byte (8 bit signed)
 5005 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5006   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5007 
 5008   ins_cost(125);
 5009   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5010   ins_encode %{
 5011     __ movsbl($dst$$Register, $mem$$Address);
 5012   %}
 5013   ins_pipe(ialu_reg_mem);
 5014 %}
 5015 
 5016 // Load Short (16 bit signed) into Long Register
 5017 instruct loadS2L(rRegL dst, memory mem)
 5018 %{
 5019   match(Set dst (ConvI2L (LoadS mem)));
 5020 
 5021   ins_cost(125);
 5022   format %{ "movswq $dst, $mem\t# short -> long" %}
 5023 
 5024   ins_encode %{
 5025     __ movswq($dst$$Register, $mem$$Address);
 5026   %}
 5027 
 5028   ins_pipe(ialu_reg_mem);
 5029 %}
 5030 
 5031 // Load Unsigned Short/Char (16 bit UNsigned)
 5032 instruct loadUS(rRegI dst, memory mem)
 5033 %{
 5034   match(Set dst (LoadUS mem));
 5035 
 5036   ins_cost(125);
 5037   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5038 
 5039   ins_encode %{
 5040     __ movzwl($dst$$Register, $mem$$Address);
 5041   %}
 5042 
 5043   ins_pipe(ialu_reg_mem);
 5044 %}
 5045 
 5046 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5047 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5048   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5049 
 5050   ins_cost(125);
 5051   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5052   ins_encode %{
 5053     __ movsbl($dst$$Register, $mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_reg_mem);
 5056 %}
 5057 
 5058 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5059 instruct loadUS2L(rRegL dst, memory mem)
 5060 %{
 5061   match(Set dst (ConvI2L (LoadUS mem)));
 5062 
 5063   ins_cost(125);
 5064   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5065 
 5066   ins_encode %{
 5067     __ movzwq($dst$$Register, $mem$$Address);
 5068   %}
 5069 
 5070   ins_pipe(ialu_reg_mem);
 5071 %}
 5072 
 5073 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5074 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5075   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5076 
 5077   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5078   ins_encode %{
 5079     __ movzbq($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5085 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087   effect(KILL cr);
 5088 
 5089   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5090             "andl    $dst, right_n_bits($mask, 16)" %}
 5091   ins_encode %{
 5092     Register Rdst = $dst$$Register;
 5093     __ movzwq(Rdst, $mem$$Address);
 5094     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5095   %}
 5096   ins_pipe(ialu_reg_mem);
 5097 %}
 5098 
 5099 // Load Integer
 5100 instruct loadI(rRegI dst, memory mem)
 5101 %{
 5102   match(Set dst (LoadI mem));
 5103 
 5104   ins_cost(125);
 5105   format %{ "movl    $dst, $mem\t# int" %}
 5106 
 5107   ins_encode %{
 5108     __ movl($dst$$Register, $mem$$Address);
 5109   %}
 5110 
 5111   ins_pipe(ialu_reg_mem);
 5112 %}
 5113 
 5114 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5115 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5116   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5117 
 5118   ins_cost(125);
 5119   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5120   ins_encode %{
 5121     __ movsbl($dst$$Register, $mem$$Address);
 5122   %}
 5123   ins_pipe(ialu_reg_mem);
 5124 %}
 5125 
 5126 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5127 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5128   match(Set dst (AndI (LoadI mem) mask));
 5129 
 5130   ins_cost(125);
 5131   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5132   ins_encode %{
 5133     __ movzbl($dst$$Register, $mem$$Address);
 5134   %}
 5135   ins_pipe(ialu_reg_mem);
 5136 %}
 5137 
 5138 // Load Integer (32 bit signed) to Short (16 bit signed)
 5139 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5140   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5141 
 5142   ins_cost(125);
 5143   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5144   ins_encode %{
 5145     __ movswl($dst$$Register, $mem$$Address);
 5146   %}
 5147   ins_pipe(ialu_reg_mem);
 5148 %}
 5149 
 5150 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5151 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5152   match(Set dst (AndI (LoadI mem) mask));
 5153 
 5154   ins_cost(125);
 5155   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5156   ins_encode %{
 5157     __ movzwl($dst$$Register, $mem$$Address);
 5158   %}
 5159   ins_pipe(ialu_reg_mem);
 5160 %}
 5161 
 5162 // Load Integer into Long Register
 5163 instruct loadI2L(rRegL dst, memory mem)
 5164 %{
 5165   match(Set dst (ConvI2L (LoadI mem)));
 5166 
 5167   ins_cost(125);
 5168   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5169 
 5170   ins_encode %{
 5171     __ movslq($dst$$Register, $mem$$Address);
 5172   %}
 5173 
 5174   ins_pipe(ialu_reg_mem);
 5175 %}
 5176 
 5177 // Load Integer with mask 0xFF into Long Register
 5178 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5179   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5180 
 5181   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5182   ins_encode %{
 5183     __ movzbq($dst$$Register, $mem$$Address);
 5184   %}
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFFFF into Long Register
 5189 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5193   ins_encode %{
 5194     __ movzwq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with a 31-bit mask into Long Register
 5200 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202   effect(KILL cr);
 5203 
 5204   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5205             "andl    $dst, $mask" %}
 5206   ins_encode %{
 5207     Register Rdst = $dst$$Register;
 5208     __ movl(Rdst, $mem$$Address);
 5209     __ andl(Rdst, $mask$$constant);
 5210   %}
 5211   ins_pipe(ialu_reg_mem);
 5212 %}
 5213 
 5214 // Load Unsigned Integer into Long Register
 5215 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5216 %{
 5217   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5218 
 5219   ins_cost(125);
 5220   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5221 
 5222   ins_encode %{
 5223     __ movl($dst$$Register, $mem$$Address);
 5224   %}
 5225 
 5226   ins_pipe(ialu_reg_mem);
 5227 %}
 5228 
 5229 // Load Long
 5230 instruct loadL(rRegL dst, memory mem)
 5231 %{
 5232   match(Set dst (LoadL mem));
 5233 
 5234   ins_cost(125);
 5235   format %{ "movq    $dst, $mem\t# long" %}
 5236 
 5237   ins_encode %{
 5238     __ movq($dst$$Register, $mem$$Address);
 5239   %}
 5240 
 5241   ins_pipe(ialu_reg_mem); // XXX
 5242 %}
 5243 
 5244 // Load Range
 5245 instruct loadRange(rRegI dst, memory mem)
 5246 %{
 5247   match(Set dst (LoadRange mem));
 5248 
 5249   ins_cost(125); // XXX
 5250   format %{ "movl    $dst, $mem\t# range" %}
 5251   ins_encode %{
 5252     __ movl($dst$$Register, $mem$$Address);
 5253   %}
 5254   ins_pipe(ialu_reg_mem);
 5255 %}
 5256 
 5257 // Load Pointer
 5258 instruct loadP(rRegP dst, memory mem)
 5259 %{
 5260   match(Set dst (LoadP mem));
 5261   predicate(n->as_Load()->barrier_data() == 0);
 5262 
 5263   ins_cost(125); // XXX
 5264   format %{ "movq    $dst, $mem\t# ptr" %}
 5265   ins_encode %{
 5266     __ movq($dst$$Register, $mem$$Address);
 5267   %}
 5268   ins_pipe(ialu_reg_mem); // XXX
 5269 %}
 5270 
 5271 // Load Compressed Pointer
 5272 instruct loadN(rRegN dst, memory mem)
 5273 %{
 5274    match(Set dst (LoadN mem));
 5275 
 5276    ins_cost(125); // XXX
 5277    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5278    ins_encode %{
 5279      __ movl($dst$$Register, $mem$$Address);
 5280    %}
 5281    ins_pipe(ialu_reg_mem); // XXX
 5282 %}
 5283 
 5284 
 5285 // Load Klass Pointer
 5286 instruct loadKlass(rRegP dst, memory mem)
 5287 %{
 5288   match(Set dst (LoadKlass mem));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movq    $dst, $mem\t# class" %}
 5292   ins_encode %{
 5293     __ movq($dst$$Register, $mem$$Address);
 5294   %}
 5295   ins_pipe(ialu_reg_mem); // XXX
 5296 %}
 5297 
 5298 // Load narrow Klass Pointer
 5299 instruct loadNKlass(rRegN dst, memory mem)
 5300 %{
 5301   match(Set dst (LoadNKlass mem));
 5302 
 5303   ins_cost(125); // XXX
 5304   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5305   ins_encode %{
 5306     __ movl($dst$$Register, $mem$$Address);
 5307   %}
 5308   ins_pipe(ialu_reg_mem); // XXX
 5309 %}
 5310 
 5311 // Load Float
 5312 instruct loadF(regF dst, memory mem)
 5313 %{
 5314   match(Set dst (LoadF mem));
 5315 
 5316   ins_cost(145); // XXX
 5317   format %{ "movss   $dst, $mem\t# float" %}
 5318   ins_encode %{
 5319     __ movflt($dst$$XMMRegister, $mem$$Address);
 5320   %}
 5321   ins_pipe(pipe_slow); // XXX
 5322 %}
 5323 
 5324 // Load Double
 5325 instruct loadD_partial(regD dst, memory mem)
 5326 %{
 5327   predicate(!UseXmmLoadAndClearUpper);
 5328   match(Set dst (LoadD mem));
 5329 
 5330   ins_cost(145); // XXX
 5331   format %{ "movlpd  $dst, $mem\t# double" %}
 5332   ins_encode %{
 5333     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5334   %}
 5335   ins_pipe(pipe_slow); // XXX
 5336 %}
 5337 
 5338 instruct loadD(regD dst, memory mem)
 5339 %{
 5340   predicate(UseXmmLoadAndClearUpper);
 5341   match(Set dst (LoadD mem));
 5342 
 5343   ins_cost(145); // XXX
 5344   format %{ "movsd   $dst, $mem\t# double" %}
 5345   ins_encode %{
 5346     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5347   %}
 5348   ins_pipe(pipe_slow); // XXX
 5349 %}
 5350 
 5351 
 5352 // Following pseudo code describes the algorithm for max[FD]:
 5353 // Min algorithm is on similar lines
 5354 //  btmp = (b < +0.0) ? a : b
 5355 //  atmp = (b < +0.0) ? b : a
 5356 //  Tmp  = Max_Float(atmp , btmp)
 5357 //  Res  = (atmp == NaN) ? atmp : Tmp
 5358 
 5359 // max = java.lang.Math.max(float a, float b)
 5360 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5361   predicate(UseAVX > 0 && !n->is_reduction());
 5362   match(Set dst (MaxF a b));
 5363   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5364   format %{
 5365      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5366      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5367      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5368      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5369      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5370   %}
 5371   ins_encode %{
 5372     int vector_len = Assembler::AVX_128bit;
 5373     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5374     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5375     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5376     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5377     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5378  %}
 5379   ins_pipe( pipe_slow );
 5380 %}
 5381 
 5382 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5383   predicate(UseAVX > 0 && n->is_reduction());
 5384   match(Set dst (MaxF a b));
 5385   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5386 
 5387   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5388   ins_encode %{
 5389     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5390                     false /*min*/, true /*single*/);
 5391   %}
 5392   ins_pipe( pipe_slow );
 5393 %}
 5394 
 5395 // max = java.lang.Math.max(double a, double b)
 5396 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5397   predicate(UseAVX > 0 && !n->is_reduction());
 5398   match(Set dst (MaxD a b));
 5399   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5400   format %{
 5401      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5402      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5403      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5404      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5405      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5406   %}
 5407   ins_encode %{
 5408     int vector_len = Assembler::AVX_128bit;
 5409     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5410     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5411     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5412     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5413     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5414   %}
 5415   ins_pipe( pipe_slow );
 5416 %}
 5417 
 5418 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5419   predicate(UseAVX > 0 && n->is_reduction());
 5420   match(Set dst (MaxD a b));
 5421   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5422 
 5423   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5424   ins_encode %{
 5425     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5426                     false /*min*/, false /*single*/);
 5427   %}
 5428   ins_pipe( pipe_slow );
 5429 %}
 5430 
 5431 // min = java.lang.Math.min(float a, float b)
 5432 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5433   predicate(UseAVX > 0 && !n->is_reduction());
 5434   match(Set dst (MinF a b));
 5435   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5436   format %{
 5437      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5438      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5439      "vminss           $tmp,$atmp,$btmp           \n\t"
 5440      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5441      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5442   %}
 5443   ins_encode %{
 5444     int vector_len = Assembler::AVX_128bit;
 5445     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5446     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5447     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5448     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5449     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5450   %}
 5451   ins_pipe( pipe_slow );
 5452 %}
 5453 
 5454 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5455   predicate(UseAVX > 0 && n->is_reduction());
 5456   match(Set dst (MinF a b));
 5457   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5458 
 5459   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5460   ins_encode %{
 5461     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5462                     true /*min*/, true /*single*/);
 5463   %}
 5464   ins_pipe( pipe_slow );
 5465 %}
 5466 
 5467 // min = java.lang.Math.min(double a, double b)
 5468 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5469   predicate(UseAVX > 0 && !n->is_reduction());
 5470   match(Set dst (MinD a b));
 5471   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5472   format %{
 5473      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5474      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5475      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5476      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5477      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5478   %}
 5479   ins_encode %{
 5480     int vector_len = Assembler::AVX_128bit;
 5481     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5482     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5483     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5484     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5485     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5486   %}
 5487   ins_pipe( pipe_slow );
 5488 %}
 5489 
 5490 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5491   predicate(UseAVX > 0 && n->is_reduction());
 5492   match(Set dst (MinD a b));
 5493   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5494 
 5495   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5496   ins_encode %{
 5497     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5498                     true /*min*/, false /*single*/);
 5499   %}
 5500   ins_pipe( pipe_slow );
 5501 %}
 5502 
 5503 // Load Effective Address
 5504 instruct leaP8(rRegP dst, indOffset8 mem)
 5505 %{
 5506   match(Set dst mem);
 5507 
 5508   ins_cost(110); // XXX
 5509   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5510   ins_encode %{
 5511     __ leaq($dst$$Register, $mem$$Address);
 5512   %}
 5513   ins_pipe(ialu_reg_reg_fat);
 5514 %}
 5515 
 5516 instruct leaP32(rRegP dst, indOffset32 mem)
 5517 %{
 5518   match(Set dst mem);
 5519 
 5520   ins_cost(110);
 5521   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5522   ins_encode %{
 5523     __ leaq($dst$$Register, $mem$$Address);
 5524   %}
 5525   ins_pipe(ialu_reg_reg_fat);
 5526 %}
 5527 
 5528 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5529 %{
 5530   match(Set dst mem);
 5531 
 5532   ins_cost(110);
 5533   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5534   ins_encode %{
 5535     __ leaq($dst$$Register, $mem$$Address);
 5536   %}
 5537   ins_pipe(ialu_reg_reg_fat);
 5538 %}
 5539 
 5540 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5541 %{
 5542   match(Set dst mem);
 5543 
 5544   ins_cost(110);
 5545   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5546   ins_encode %{
 5547     __ leaq($dst$$Register, $mem$$Address);
 5548   %}
 5549   ins_pipe(ialu_reg_reg_fat);
 5550 %}
 5551 
 5552 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5553 %{
 5554   match(Set dst mem);
 5555 
 5556   ins_cost(110);
 5557   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5558   ins_encode %{
 5559     __ leaq($dst$$Register, $mem$$Address);
 5560   %}
 5561   ins_pipe(ialu_reg_reg_fat);
 5562 %}
 5563 
 5564 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5565 %{
 5566   match(Set dst mem);
 5567 
 5568   ins_cost(110);
 5569   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5570   ins_encode %{
 5571     __ leaq($dst$$Register, $mem$$Address);
 5572   %}
 5573   ins_pipe(ialu_reg_reg_fat);
 5574 %}
 5575 
 5576 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5577 %{
 5578   match(Set dst mem);
 5579 
 5580   ins_cost(110);
 5581   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5582   ins_encode %{
 5583     __ leaq($dst$$Register, $mem$$Address);
 5584   %}
 5585   ins_pipe(ialu_reg_reg_fat);
 5586 %}
 5587 
 5588 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5589 %{
 5590   match(Set dst mem);
 5591 
 5592   ins_cost(110);
 5593   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5594   ins_encode %{
 5595     __ leaq($dst$$Register, $mem$$Address);
 5596   %}
 5597   ins_pipe(ialu_reg_reg_fat);
 5598 %}
 5599 
 5600 // Load Effective Address which uses Narrow (32-bits) oop
 5601 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5602 %{
 5603   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5604   match(Set dst mem);
 5605 
 5606   ins_cost(110);
 5607   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5608   ins_encode %{
 5609     __ leaq($dst$$Register, $mem$$Address);
 5610   %}
 5611   ins_pipe(ialu_reg_reg_fat);
 5612 %}
 5613 
 5614 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5615 %{
 5616   predicate(CompressedOops::shift() == 0);
 5617   match(Set dst mem);
 5618 
 5619   ins_cost(110); // XXX
 5620   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5621   ins_encode %{
 5622     __ leaq($dst$$Register, $mem$$Address);
 5623   %}
 5624   ins_pipe(ialu_reg_reg_fat);
 5625 %}
 5626 
 5627 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5628 %{
 5629   predicate(CompressedOops::shift() == 0);
 5630   match(Set dst mem);
 5631 
 5632   ins_cost(110);
 5633   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5634   ins_encode %{
 5635     __ leaq($dst$$Register, $mem$$Address);
 5636   %}
 5637   ins_pipe(ialu_reg_reg_fat);
 5638 %}
 5639 
 5640 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5641 %{
 5642   predicate(CompressedOops::shift() == 0);
 5643   match(Set dst mem);
 5644 
 5645   ins_cost(110);
 5646   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5647   ins_encode %{
 5648     __ leaq($dst$$Register, $mem$$Address);
 5649   %}
 5650   ins_pipe(ialu_reg_reg_fat);
 5651 %}
 5652 
 5653 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5654 %{
 5655   predicate(CompressedOops::shift() == 0);
 5656   match(Set dst mem);
 5657 
 5658   ins_cost(110);
 5659   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5660   ins_encode %{
 5661     __ leaq($dst$$Register, $mem$$Address);
 5662   %}
 5663   ins_pipe(ialu_reg_reg_fat);
 5664 %}
 5665 
 5666 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5667 %{
 5668   predicate(CompressedOops::shift() == 0);
 5669   match(Set dst mem);
 5670 
 5671   ins_cost(110);
 5672   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5673   ins_encode %{
 5674     __ leaq($dst$$Register, $mem$$Address);
 5675   %}
 5676   ins_pipe(ialu_reg_reg_fat);
 5677 %}
 5678 
 5679 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5680 %{
 5681   predicate(CompressedOops::shift() == 0);
 5682   match(Set dst mem);
 5683 
 5684   ins_cost(110);
 5685   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5686   ins_encode %{
 5687     __ leaq($dst$$Register, $mem$$Address);
 5688   %}
 5689   ins_pipe(ialu_reg_reg_fat);
 5690 %}
 5691 
 5692 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5693 %{
 5694   predicate(CompressedOops::shift() == 0);
 5695   match(Set dst mem);
 5696 
 5697   ins_cost(110);
 5698   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5699   ins_encode %{
 5700     __ leaq($dst$$Register, $mem$$Address);
 5701   %}
 5702   ins_pipe(ialu_reg_reg_fat);
 5703 %}
 5704 
 5705 instruct loadConI(rRegI dst, immI src)
 5706 %{
 5707   match(Set dst src);
 5708 
 5709   format %{ "movl    $dst, $src\t# int" %}
 5710   ins_encode %{
 5711     __ movl($dst$$Register, $src$$constant);
 5712   %}
 5713   ins_pipe(ialu_reg_fat); // XXX
 5714 %}
 5715 
 5716 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5717 %{
 5718   match(Set dst src);
 5719   effect(KILL cr);
 5720 
 5721   ins_cost(50);
 5722   format %{ "xorl    $dst, $dst\t# int" %}
 5723   ins_encode %{
 5724     __ xorl($dst$$Register, $dst$$Register);
 5725   %}
 5726   ins_pipe(ialu_reg);
 5727 %}
 5728 
 5729 instruct loadConL(rRegL dst, immL src)
 5730 %{
 5731   match(Set dst src);
 5732 
 5733   ins_cost(150);
 5734   format %{ "movq    $dst, $src\t# long" %}
 5735   ins_encode %{
 5736     __ mov64($dst$$Register, $src$$constant);
 5737   %}
 5738   ins_pipe(ialu_reg);
 5739 %}
 5740 
 5741 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5742 %{
 5743   match(Set dst src);
 5744   effect(KILL cr);
 5745 
 5746   ins_cost(50);
 5747   format %{ "xorl    $dst, $dst\t# long" %}
 5748   ins_encode %{
 5749     __ xorl($dst$$Register, $dst$$Register);
 5750   %}
 5751   ins_pipe(ialu_reg); // XXX
 5752 %}
 5753 
 5754 instruct loadConUL32(rRegL dst, immUL32 src)
 5755 %{
 5756   match(Set dst src);
 5757 
 5758   ins_cost(60);
 5759   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5760   ins_encode %{
 5761     __ movl($dst$$Register, $src$$constant);
 5762   %}
 5763   ins_pipe(ialu_reg);
 5764 %}
 5765 
 5766 instruct loadConL32(rRegL dst, immL32 src)
 5767 %{
 5768   match(Set dst src);
 5769 
 5770   ins_cost(70);
 5771   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5772   ins_encode %{
 5773     __ movq($dst$$Register, $src$$constant);
 5774   %}
 5775   ins_pipe(ialu_reg);
 5776 %}
 5777 
 5778 instruct loadConP(rRegP dst, immP con) %{
 5779   match(Set dst con);
 5780 
 5781   format %{ "movq    $dst, $con\t# ptr" %}
 5782   ins_encode %{
 5783     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5784   %}
 5785   ins_pipe(ialu_reg_fat); // XXX
 5786 %}
 5787 
 5788 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5789 %{
 5790   match(Set dst src);
 5791   effect(KILL cr);
 5792 
 5793   ins_cost(50);
 5794   format %{ "xorl    $dst, $dst\t# ptr" %}
 5795   ins_encode %{
 5796     __ xorl($dst$$Register, $dst$$Register);
 5797   %}
 5798   ins_pipe(ialu_reg);
 5799 %}
 5800 
 5801 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5802 %{
 5803   match(Set dst src);
 5804   effect(KILL cr);
 5805 
 5806   ins_cost(60);
 5807   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5808   ins_encode %{
 5809     __ movl($dst$$Register, $src$$constant);
 5810   %}
 5811   ins_pipe(ialu_reg);
 5812 %}
 5813 
 5814 instruct loadConF(regF dst, immF con) %{
 5815   match(Set dst con);
 5816   ins_cost(125);
 5817   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5818   ins_encode %{
 5819     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5820   %}
 5821   ins_pipe(pipe_slow);
 5822 %}
 5823 
 5824 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5825   match(Set dst src);
 5826   effect(KILL cr);
 5827   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5828   ins_encode %{
 5829     __ xorq($dst$$Register, $dst$$Register);
 5830   %}
 5831   ins_pipe(ialu_reg);
 5832 %}
 5833 
 5834 instruct loadConN(rRegN dst, immN src) %{
 5835   match(Set dst src);
 5836 
 5837   ins_cost(125);
 5838   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5839   ins_encode %{
 5840     address con = (address)$src$$constant;
 5841     if (con == NULL) {
 5842       ShouldNotReachHere();
 5843     } else {
 5844       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5845     }
 5846   %}
 5847   ins_pipe(ialu_reg_fat); // XXX
 5848 %}
 5849 
 5850 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5851   match(Set dst src);
 5852 
 5853   ins_cost(125);
 5854   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5855   ins_encode %{
 5856     address con = (address)$src$$constant;
 5857     if (con == NULL) {
 5858       ShouldNotReachHere();
 5859     } else {
 5860       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5861     }
 5862   %}
 5863   ins_pipe(ialu_reg_fat); // XXX
 5864 %}
 5865 
 5866 instruct loadConF0(regF dst, immF0 src)
 5867 %{
 5868   match(Set dst src);
 5869   ins_cost(100);
 5870 
 5871   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5872   ins_encode %{
 5873     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5874   %}
 5875   ins_pipe(pipe_slow);
 5876 %}
 5877 
 5878 // Use the same format since predicate() can not be used here.
 5879 instruct loadConD(regD dst, immD con) %{
 5880   match(Set dst con);
 5881   ins_cost(125);
 5882   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5883   ins_encode %{
 5884     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5885   %}
 5886   ins_pipe(pipe_slow);
 5887 %}
 5888 
 5889 instruct loadConD0(regD dst, immD0 src)
 5890 %{
 5891   match(Set dst src);
 5892   ins_cost(100);
 5893 
 5894   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5895   ins_encode %{
 5896     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5897   %}
 5898   ins_pipe(pipe_slow);
 5899 %}
 5900 
 5901 instruct loadSSI(rRegI dst, stackSlotI src)
 5902 %{
 5903   match(Set dst src);
 5904 
 5905   ins_cost(125);
 5906   format %{ "movl    $dst, $src\t# int stk" %}
 5907   opcode(0x8B);
 5908   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5909   ins_pipe(ialu_reg_mem);
 5910 %}
 5911 
 5912 instruct loadSSL(rRegL dst, stackSlotL src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movq    $dst, $src\t# long stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSP(rRegP dst, stackSlotP src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movq    $dst, $src\t# ptr stk" %}
 5929   opcode(0x8B);
 5930   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5931   ins_pipe(ialu_reg_mem);
 5932 %}
 5933 
 5934 instruct loadSSF(regF dst, stackSlotF src)
 5935 %{
 5936   match(Set dst src);
 5937 
 5938   ins_cost(125);
 5939   format %{ "movss   $dst, $src\t# float stk" %}
 5940   ins_encode %{
 5941     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5942   %}
 5943   ins_pipe(pipe_slow); // XXX
 5944 %}
 5945 
 5946 // Use the same format since predicate() can not be used here.
 5947 instruct loadSSD(regD dst, stackSlotD src)
 5948 %{
 5949   match(Set dst src);
 5950 
 5951   ins_cost(125);
 5952   format %{ "movsd   $dst, $src\t# double stk" %}
 5953   ins_encode  %{
 5954     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5955   %}
 5956   ins_pipe(pipe_slow); // XXX
 5957 %}
 5958 
 5959 // Prefetch instructions for allocation.
 5960 // Must be safe to execute with invalid address (cannot fault).
 5961 
 5962 instruct prefetchAlloc( memory mem ) %{
 5963   predicate(AllocatePrefetchInstr==3);
 5964   match(PrefetchAllocation mem);
 5965   ins_cost(125);
 5966 
 5967   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5968   ins_encode %{
 5969     __ prefetchw($mem$$Address);
 5970   %}
 5971   ins_pipe(ialu_mem);
 5972 %}
 5973 
 5974 instruct prefetchAllocNTA( memory mem ) %{
 5975   predicate(AllocatePrefetchInstr==0);
 5976   match(PrefetchAllocation mem);
 5977   ins_cost(125);
 5978 
 5979   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5980   ins_encode %{
 5981     __ prefetchnta($mem$$Address);
 5982   %}
 5983   ins_pipe(ialu_mem);
 5984 %}
 5985 
 5986 instruct prefetchAllocT0( memory mem ) %{
 5987   predicate(AllocatePrefetchInstr==1);
 5988   match(PrefetchAllocation mem);
 5989   ins_cost(125);
 5990 
 5991   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5992   ins_encode %{
 5993     __ prefetcht0($mem$$Address);
 5994   %}
 5995   ins_pipe(ialu_mem);
 5996 %}
 5997 
 5998 instruct prefetchAllocT2( memory mem ) %{
 5999   predicate(AllocatePrefetchInstr==2);
 6000   match(PrefetchAllocation mem);
 6001   ins_cost(125);
 6002 
 6003   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6004   ins_encode %{
 6005     __ prefetcht2($mem$$Address);
 6006   %}
 6007   ins_pipe(ialu_mem);
 6008 %}
 6009 
 6010 //----------Store Instructions-------------------------------------------------
 6011 
 6012 // Store Byte
 6013 instruct storeB(memory mem, rRegI src)
 6014 %{
 6015   match(Set mem (StoreB mem src));
 6016 
 6017   ins_cost(125); // XXX
 6018   format %{ "movb    $mem, $src\t# byte" %}
 6019   ins_encode %{
 6020     __ movb($mem$$Address, $src$$Register);
 6021   %}
 6022   ins_pipe(ialu_mem_reg);
 6023 %}
 6024 
 6025 // Store Char/Short
 6026 instruct storeC(memory mem, rRegI src)
 6027 %{
 6028   match(Set mem (StoreC mem src));
 6029 
 6030   ins_cost(125); // XXX
 6031   format %{ "movw    $mem, $src\t# char/short" %}
 6032   ins_encode %{
 6033     __ movw($mem$$Address, $src$$Register);
 6034   %}
 6035   ins_pipe(ialu_mem_reg);
 6036 %}
 6037 
 6038 // Store Integer
 6039 instruct storeI(memory mem, rRegI src)
 6040 %{
 6041   match(Set mem (StoreI mem src));
 6042 
 6043   ins_cost(125); // XXX
 6044   format %{ "movl    $mem, $src\t# int" %}
 6045   ins_encode %{
 6046     __ movl($mem$$Address, $src$$Register);
 6047   %}
 6048   ins_pipe(ialu_mem_reg);
 6049 %}
 6050 
 6051 // Store Long
 6052 instruct storeL(memory mem, rRegL src)
 6053 %{
 6054   match(Set mem (StoreL mem src));
 6055 
 6056   ins_cost(125); // XXX
 6057   format %{ "movq    $mem, $src\t# long" %}
 6058   ins_encode %{
 6059     __ movq($mem$$Address, $src$$Register);
 6060   %}
 6061   ins_pipe(ialu_mem_reg); // XXX
 6062 %}
 6063 
 6064 // Store Pointer
 6065 instruct storeP(memory mem, any_RegP src)
 6066 %{
 6067   match(Set mem (StoreP mem src));
 6068 
 6069   ins_cost(125); // XXX
 6070   format %{ "movq    $mem, $src\t# ptr" %}
 6071   ins_encode %{
 6072     __ movq($mem$$Address, $src$$Register);
 6073   %}
 6074   ins_pipe(ialu_mem_reg);
 6075 %}
 6076 
 6077 instruct storeImmP0(memory mem, immP0 zero)
 6078 %{
 6079   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6080   match(Set mem (StoreP mem zero));
 6081 
 6082   ins_cost(125); // XXX
 6083   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6084   ins_encode %{
 6085     __ movq($mem$$Address, r12);
 6086   %}
 6087   ins_pipe(ialu_mem_reg);
 6088 %}
 6089 
 6090 // Store NULL Pointer, mark word, or other simple pointer constant.
 6091 instruct storeImmP(memory mem, immP31 src)
 6092 %{
 6093   match(Set mem (StoreP mem src));
 6094 
 6095   ins_cost(150); // XXX
 6096   format %{ "movq    $mem, $src\t# ptr" %}
 6097   ins_encode %{
 6098     __ movq($mem$$Address, $src$$constant);
 6099   %}
 6100   ins_pipe(ialu_mem_imm);
 6101 %}
 6102 
 6103 // Store Compressed Pointer
 6104 instruct storeN(memory mem, rRegN src)
 6105 %{
 6106   match(Set mem (StoreN mem src));
 6107 
 6108   ins_cost(125); // XXX
 6109   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6110   ins_encode %{
 6111     __ movl($mem$$Address, $src$$Register);
 6112   %}
 6113   ins_pipe(ialu_mem_reg);
 6114 %}
 6115 
 6116 instruct storeNKlass(memory mem, rRegN src)
 6117 %{
 6118   match(Set mem (StoreNKlass mem src));
 6119 
 6120   ins_cost(125); // XXX
 6121   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6122   ins_encode %{
 6123     __ movl($mem$$Address, $src$$Register);
 6124   %}
 6125   ins_pipe(ialu_mem_reg);
 6126 %}
 6127 
 6128 instruct storeImmN0(memory mem, immN0 zero)
 6129 %{
 6130   predicate(CompressedOops::base() == NULL);
 6131   match(Set mem (StoreN mem zero));
 6132 
 6133   ins_cost(125); // XXX
 6134   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6135   ins_encode %{
 6136     __ movl($mem$$Address, r12);
 6137   %}
 6138   ins_pipe(ialu_mem_reg);
 6139 %}
 6140 
 6141 instruct storeImmN(memory mem, immN src)
 6142 %{
 6143   match(Set mem (StoreN mem src));
 6144 
 6145   ins_cost(150); // XXX
 6146   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6147   ins_encode %{
 6148     address con = (address)$src$$constant;
 6149     if (con == NULL) {
 6150       __ movl($mem$$Address, 0);
 6151     } else {
 6152       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6153     }
 6154   %}
 6155   ins_pipe(ialu_mem_imm);
 6156 %}
 6157 
 6158 instruct storeImmNKlass(memory mem, immNKlass src)
 6159 %{
 6160   match(Set mem (StoreNKlass mem src));
 6161 
 6162   ins_cost(150); // XXX
 6163   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6164   ins_encode %{
 6165     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6166   %}
 6167   ins_pipe(ialu_mem_imm);
 6168 %}
 6169 
 6170 // Store Integer Immediate
 6171 instruct storeImmI0(memory mem, immI_0 zero)
 6172 %{
 6173   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6174   match(Set mem (StoreI mem zero));
 6175 
 6176   ins_cost(125); // XXX
 6177   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6178   ins_encode %{
 6179     __ movl($mem$$Address, r12);
 6180   %}
 6181   ins_pipe(ialu_mem_reg);
 6182 %}
 6183 
 6184 instruct storeImmI(memory mem, immI src)
 6185 %{
 6186   match(Set mem (StoreI mem src));
 6187 
 6188   ins_cost(150);
 6189   format %{ "movl    $mem, $src\t# int" %}
 6190   ins_encode %{
 6191     __ movl($mem$$Address, $src$$constant);
 6192   %}
 6193   ins_pipe(ialu_mem_imm);
 6194 %}
 6195 
 6196 // Store Long Immediate
 6197 instruct storeImmL0(memory mem, immL0 zero)
 6198 %{
 6199   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6200   match(Set mem (StoreL mem zero));
 6201 
 6202   ins_cost(125); // XXX
 6203   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6204   ins_encode %{
 6205     __ movq($mem$$Address, r12);
 6206   %}
 6207   ins_pipe(ialu_mem_reg);
 6208 %}
 6209 
 6210 instruct storeImmL(memory mem, immL32 src)
 6211 %{
 6212   match(Set mem (StoreL mem src));
 6213 
 6214   ins_cost(150);
 6215   format %{ "movq    $mem, $src\t# long" %}
 6216   ins_encode %{
 6217     __ movq($mem$$Address, $src$$constant);
 6218   %}
 6219   ins_pipe(ialu_mem_imm);
 6220 %}
 6221 
 6222 // Store Short/Char Immediate
 6223 instruct storeImmC0(memory mem, immI_0 zero)
 6224 %{
 6225   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6226   match(Set mem (StoreC mem zero));
 6227 
 6228   ins_cost(125); // XXX
 6229   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6230   ins_encode %{
 6231     __ movw($mem$$Address, r12);
 6232   %}
 6233   ins_pipe(ialu_mem_reg);
 6234 %}
 6235 
 6236 instruct storeImmI16(memory mem, immI16 src)
 6237 %{
 6238   predicate(UseStoreImmI16);
 6239   match(Set mem (StoreC mem src));
 6240 
 6241   ins_cost(150);
 6242   format %{ "movw    $mem, $src\t# short/char" %}
 6243   ins_encode %{
 6244     __ movw($mem$$Address, $src$$constant);
 6245   %}
 6246   ins_pipe(ialu_mem_imm);
 6247 %}
 6248 
 6249 // Store Byte Immediate
 6250 instruct storeImmB0(memory mem, immI_0 zero)
 6251 %{
 6252   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6253   match(Set mem (StoreB mem zero));
 6254 
 6255   ins_cost(125); // XXX
 6256   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6257   ins_encode %{
 6258     __ movb($mem$$Address, r12);
 6259   %}
 6260   ins_pipe(ialu_mem_reg);
 6261 %}
 6262 
 6263 instruct storeImmB(memory mem, immI8 src)
 6264 %{
 6265   match(Set mem (StoreB mem src));
 6266 
 6267   ins_cost(150); // XXX
 6268   format %{ "movb    $mem, $src\t# byte" %}
 6269   ins_encode %{
 6270     __ movb($mem$$Address, $src$$constant);
 6271   %}
 6272   ins_pipe(ialu_mem_imm);
 6273 %}
 6274 
 6275 // Store CMS card-mark Immediate
 6276 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6277 %{
 6278   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6279   match(Set mem (StoreCM mem zero));
 6280 
 6281   ins_cost(125); // XXX
 6282   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6283   ins_encode %{
 6284     __ movb($mem$$Address, r12);
 6285   %}
 6286   ins_pipe(ialu_mem_reg);
 6287 %}
 6288 
 6289 instruct storeImmCM0(memory mem, immI_0 src)
 6290 %{
 6291   match(Set mem (StoreCM mem src));
 6292 
 6293   ins_cost(150); // XXX
 6294   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6295   ins_encode %{
 6296     __ movb($mem$$Address, $src$$constant);
 6297   %}
 6298   ins_pipe(ialu_mem_imm);
 6299 %}
 6300 
 6301 // Store Float
 6302 instruct storeF(memory mem, regF src)
 6303 %{
 6304   match(Set mem (StoreF mem src));
 6305 
 6306   ins_cost(95); // XXX
 6307   format %{ "movss   $mem, $src\t# float" %}
 6308   ins_encode %{
 6309     __ movflt($mem$$Address, $src$$XMMRegister);
 6310   %}
 6311   ins_pipe(pipe_slow); // XXX
 6312 %}
 6313 
 6314 // Store immediate Float value (it is faster than store from XMM register)
 6315 instruct storeF0(memory mem, immF0 zero)
 6316 %{
 6317   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6318   match(Set mem (StoreF mem zero));
 6319 
 6320   ins_cost(25); // XXX
 6321   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6322   ins_encode %{
 6323     __ movl($mem$$Address, r12);
 6324   %}
 6325   ins_pipe(ialu_mem_reg);
 6326 %}
 6327 
 6328 instruct storeF_imm(memory mem, immF src)
 6329 %{
 6330   match(Set mem (StoreF mem src));
 6331 
 6332   ins_cost(50);
 6333   format %{ "movl    $mem, $src\t# float" %}
 6334   ins_encode %{
 6335     __ movl($mem$$Address, jint_cast($src$$constant));
 6336   %}
 6337   ins_pipe(ialu_mem_imm);
 6338 %}
 6339 
 6340 // Store Double
 6341 instruct storeD(memory mem, regD src)
 6342 %{
 6343   match(Set mem (StoreD mem src));
 6344 
 6345   ins_cost(95); // XXX
 6346   format %{ "movsd   $mem, $src\t# double" %}
 6347   ins_encode %{
 6348     __ movdbl($mem$$Address, $src$$XMMRegister);
 6349   %}
 6350   ins_pipe(pipe_slow); // XXX
 6351 %}
 6352 
 6353 // Store immediate double 0.0 (it is faster than store from XMM register)
 6354 instruct storeD0_imm(memory mem, immD0 src)
 6355 %{
 6356   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6357   match(Set mem (StoreD mem src));
 6358 
 6359   ins_cost(50);
 6360   format %{ "movq    $mem, $src\t# double 0." %}
 6361   ins_encode %{
 6362     __ movq($mem$$Address, $src$$constant);
 6363   %}
 6364   ins_pipe(ialu_mem_imm);
 6365 %}
 6366 
 6367 instruct storeD0(memory mem, immD0 zero)
 6368 %{
 6369   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6370   match(Set mem (StoreD mem zero));
 6371 
 6372   ins_cost(25); // XXX
 6373   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6374   ins_encode %{
 6375     __ movq($mem$$Address, r12);
 6376   %}
 6377   ins_pipe(ialu_mem_reg);
 6378 %}
 6379 
 6380 instruct storeSSI(stackSlotI dst, rRegI src)
 6381 %{
 6382   match(Set dst src);
 6383 
 6384   ins_cost(100);
 6385   format %{ "movl    $dst, $src\t# int stk" %}
 6386   opcode(0x89);
 6387   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6388   ins_pipe( ialu_mem_reg );
 6389 %}
 6390 
 6391 instruct storeSSL(stackSlotL dst, rRegL src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movq    $dst, $src\t# long stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe(ialu_mem_reg);
 6400 %}
 6401 
 6402 instruct storeSSP(stackSlotP dst, rRegP src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(100);
 6407   format %{ "movq    $dst, $src\t# ptr stk" %}
 6408   opcode(0x89);
 6409   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6410   ins_pipe(ialu_mem_reg);
 6411 %}
 6412 
 6413 instruct storeSSF(stackSlotF dst, regF src)
 6414 %{
 6415   match(Set dst src);
 6416 
 6417   ins_cost(95); // XXX
 6418   format %{ "movss   $dst, $src\t# float stk" %}
 6419   ins_encode %{
 6420     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6421   %}
 6422   ins_pipe(pipe_slow); // XXX
 6423 %}
 6424 
 6425 instruct storeSSD(stackSlotD dst, regD src)
 6426 %{
 6427   match(Set dst src);
 6428 
 6429   ins_cost(95); // XXX
 6430   format %{ "movsd   $dst, $src\t# double stk" %}
 6431   ins_encode %{
 6432     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6433   %}
 6434   ins_pipe(pipe_slow); // XXX
 6435 %}
 6436 
 6437 instruct cacheWB(indirect addr)
 6438 %{
 6439   predicate(VM_Version::supports_data_cache_line_flush());
 6440   match(CacheWB addr);
 6441 
 6442   ins_cost(100);
 6443   format %{"cache wb $addr" %}
 6444   ins_encode %{
 6445     assert($addr->index_position() < 0, "should be");
 6446     assert($addr$$disp == 0, "should be");
 6447     __ cache_wb(Address($addr$$base$$Register, 0));
 6448   %}
 6449   ins_pipe(pipe_slow); // XXX
 6450 %}
 6451 
 6452 instruct cacheWBPreSync()
 6453 %{
 6454   predicate(VM_Version::supports_data_cache_line_flush());
 6455   match(CacheWBPreSync);
 6456 
 6457   ins_cost(100);
 6458   format %{"cache wb presync" %}
 6459   ins_encode %{
 6460     __ cache_wbsync(true);
 6461   %}
 6462   ins_pipe(pipe_slow); // XXX
 6463 %}
 6464 
 6465 instruct cacheWBPostSync()
 6466 %{
 6467   predicate(VM_Version::supports_data_cache_line_flush());
 6468   match(CacheWBPostSync);
 6469 
 6470   ins_cost(100);
 6471   format %{"cache wb postsync" %}
 6472   ins_encode %{
 6473     __ cache_wbsync(false);
 6474   %}
 6475   ins_pipe(pipe_slow); // XXX
 6476 %}
 6477 
 6478 //----------BSWAP Instructions-------------------------------------------------
 6479 instruct bytes_reverse_int(rRegI dst) %{
 6480   match(Set dst (ReverseBytesI dst));
 6481 
 6482   format %{ "bswapl  $dst" %}
 6483   ins_encode %{
 6484     __ bswapl($dst$$Register);
 6485   %}
 6486   ins_pipe( ialu_reg );
 6487 %}
 6488 
 6489 instruct bytes_reverse_long(rRegL dst) %{
 6490   match(Set dst (ReverseBytesL dst));
 6491 
 6492   format %{ "bswapq  $dst" %}
 6493   ins_encode %{
 6494     __ bswapq($dst$$Register);
 6495   %}
 6496   ins_pipe( ialu_reg);
 6497 %}
 6498 
 6499 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6500   match(Set dst (ReverseBytesUS dst));
 6501   effect(KILL cr);
 6502 
 6503   format %{ "bswapl  $dst\n\t"
 6504             "shrl    $dst,16\n\t" %}
 6505   ins_encode %{
 6506     __ bswapl($dst$$Register);
 6507     __ shrl($dst$$Register, 16);
 6508   %}
 6509   ins_pipe( ialu_reg );
 6510 %}
 6511 
 6512 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6513   match(Set dst (ReverseBytesS dst));
 6514   effect(KILL cr);
 6515 
 6516   format %{ "bswapl  $dst\n\t"
 6517             "sar     $dst,16\n\t" %}
 6518   ins_encode %{
 6519     __ bswapl($dst$$Register);
 6520     __ sarl($dst$$Register, 16);
 6521   %}
 6522   ins_pipe( ialu_reg );
 6523 %}
 6524 
 6525 //---------- Zeros Count Instructions ------------------------------------------
 6526 
 6527 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6528   predicate(UseCountLeadingZerosInstruction);
 6529   match(Set dst (CountLeadingZerosI src));
 6530   effect(KILL cr);
 6531 
 6532   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6533   ins_encode %{
 6534     __ lzcntl($dst$$Register, $src$$Register);
 6535   %}
 6536   ins_pipe(ialu_reg);
 6537 %}
 6538 
 6539 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6540   predicate(UseCountLeadingZerosInstruction);
 6541   match(Set dst (CountLeadingZerosI (LoadI src)));
 6542   effect(KILL cr);
 6543   ins_cost(175);
 6544   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6545   ins_encode %{
 6546     __ lzcntl($dst$$Register, $src$$Address);
 6547   %}
 6548   ins_pipe(ialu_reg_mem);
 6549 %}
 6550 
 6551 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6552   predicate(!UseCountLeadingZerosInstruction);
 6553   match(Set dst (CountLeadingZerosI src));
 6554   effect(KILL cr);
 6555 
 6556   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6557             "jnz     skip\n\t"
 6558             "movl    $dst, -1\n"
 6559       "skip:\n\t"
 6560             "negl    $dst\n\t"
 6561             "addl    $dst, 31" %}
 6562   ins_encode %{
 6563     Register Rdst = $dst$$Register;
 6564     Register Rsrc = $src$$Register;
 6565     Label skip;
 6566     __ bsrl(Rdst, Rsrc);
 6567     __ jccb(Assembler::notZero, skip);
 6568     __ movl(Rdst, -1);
 6569     __ bind(skip);
 6570     __ negl(Rdst);
 6571     __ addl(Rdst, BitsPerInt - 1);
 6572   %}
 6573   ins_pipe(ialu_reg);
 6574 %}
 6575 
 6576 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6577   predicate(UseCountLeadingZerosInstruction);
 6578   match(Set dst (CountLeadingZerosL src));
 6579   effect(KILL cr);
 6580 
 6581   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6582   ins_encode %{
 6583     __ lzcntq($dst$$Register, $src$$Register);
 6584   %}
 6585   ins_pipe(ialu_reg);
 6586 %}
 6587 
 6588 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6589   predicate(UseCountLeadingZerosInstruction);
 6590   match(Set dst (CountLeadingZerosL (LoadL src)));
 6591   effect(KILL cr);
 6592   ins_cost(175);
 6593   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6594   ins_encode %{
 6595     __ lzcntq($dst$$Register, $src$$Address);
 6596   %}
 6597   ins_pipe(ialu_reg_mem);
 6598 %}
 6599 
 6600 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6601   predicate(!UseCountLeadingZerosInstruction);
 6602   match(Set dst (CountLeadingZerosL src));
 6603   effect(KILL cr);
 6604 
 6605   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6606             "jnz     skip\n\t"
 6607             "movl    $dst, -1\n"
 6608       "skip:\n\t"
 6609             "negl    $dst\n\t"
 6610             "addl    $dst, 63" %}
 6611   ins_encode %{
 6612     Register Rdst = $dst$$Register;
 6613     Register Rsrc = $src$$Register;
 6614     Label skip;
 6615     __ bsrq(Rdst, Rsrc);
 6616     __ jccb(Assembler::notZero, skip);
 6617     __ movl(Rdst, -1);
 6618     __ bind(skip);
 6619     __ negl(Rdst);
 6620     __ addl(Rdst, BitsPerLong - 1);
 6621   %}
 6622   ins_pipe(ialu_reg);
 6623 %}
 6624 
 6625 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6626   predicate(UseCountTrailingZerosInstruction);
 6627   match(Set dst (CountTrailingZerosI src));
 6628   effect(KILL cr);
 6629 
 6630   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6631   ins_encode %{
 6632     __ tzcntl($dst$$Register, $src$$Register);
 6633   %}
 6634   ins_pipe(ialu_reg);
 6635 %}
 6636 
 6637 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6638   predicate(UseCountTrailingZerosInstruction);
 6639   match(Set dst (CountTrailingZerosI (LoadI src)));
 6640   effect(KILL cr);
 6641   ins_cost(175);
 6642   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6643   ins_encode %{
 6644     __ tzcntl($dst$$Register, $src$$Address);
 6645   %}
 6646   ins_pipe(ialu_reg_mem);
 6647 %}
 6648 
 6649 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6650   predicate(!UseCountTrailingZerosInstruction);
 6651   match(Set dst (CountTrailingZerosI src));
 6652   effect(KILL cr);
 6653 
 6654   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6655             "jnz     done\n\t"
 6656             "movl    $dst, 32\n"
 6657       "done:" %}
 6658   ins_encode %{
 6659     Register Rdst = $dst$$Register;
 6660     Label done;
 6661     __ bsfl(Rdst, $src$$Register);
 6662     __ jccb(Assembler::notZero, done);
 6663     __ movl(Rdst, BitsPerInt);
 6664     __ bind(done);
 6665   %}
 6666   ins_pipe(ialu_reg);
 6667 %}
 6668 
 6669 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6670   predicate(UseCountTrailingZerosInstruction);
 6671   match(Set dst (CountTrailingZerosL src));
 6672   effect(KILL cr);
 6673 
 6674   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6675   ins_encode %{
 6676     __ tzcntq($dst$$Register, $src$$Register);
 6677   %}
 6678   ins_pipe(ialu_reg);
 6679 %}
 6680 
 6681 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6682   predicate(UseCountTrailingZerosInstruction);
 6683   match(Set dst (CountTrailingZerosL (LoadL src)));
 6684   effect(KILL cr);
 6685   ins_cost(175);
 6686   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6687   ins_encode %{
 6688     __ tzcntq($dst$$Register, $src$$Address);
 6689   %}
 6690   ins_pipe(ialu_reg_mem);
 6691 %}
 6692 
 6693 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6694   predicate(!UseCountTrailingZerosInstruction);
 6695   match(Set dst (CountTrailingZerosL src));
 6696   effect(KILL cr);
 6697 
 6698   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6699             "jnz     done\n\t"
 6700             "movl    $dst, 64\n"
 6701       "done:" %}
 6702   ins_encode %{
 6703     Register Rdst = $dst$$Register;
 6704     Label done;
 6705     __ bsfq(Rdst, $src$$Register);
 6706     __ jccb(Assembler::notZero, done);
 6707     __ movl(Rdst, BitsPerLong);
 6708     __ bind(done);
 6709   %}
 6710   ins_pipe(ialu_reg);
 6711 %}
 6712 
 6713 //--------------- Reverse Operation Instructions ----------------
 6714 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 6715   predicate(!VM_Version::supports_gfni());
 6716   match(Set dst (ReverseI src));
 6717   effect(TEMP dst, TEMP rtmp, KILL cr);
 6718   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 6719   ins_encode %{
 6720     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 6721   %}
 6722   ins_pipe( ialu_reg );
 6723 %}
 6724 
 6725 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6726   predicate(VM_Version::supports_gfni());
 6727   match(Set dst (ReverseI src));
 6728   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6729   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6730   ins_encode %{
 6731     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 6732   %}
 6733   ins_pipe( ialu_reg );
 6734 %}
 6735 
 6736 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 6737   predicate(!VM_Version::supports_gfni());
 6738   match(Set dst (ReverseL src));
 6739   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 6740   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 6741   ins_encode %{
 6742     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 6743   %}
 6744   ins_pipe( ialu_reg );
 6745 %}
 6746 
 6747 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6748   predicate(VM_Version::supports_gfni());
 6749   match(Set dst (ReverseL src));
 6750   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6751   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6752   ins_encode %{
 6753     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 6754   %}
 6755   ins_pipe( ialu_reg );
 6756 %}
 6757 
 6758 //---------- Population Count Instructions -------------------------------------
 6759 
 6760 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6761   predicate(UsePopCountInstruction);
 6762   match(Set dst (PopCountI src));
 6763   effect(KILL cr);
 6764 
 6765   format %{ "popcnt  $dst, $src" %}
 6766   ins_encode %{
 6767     __ popcntl($dst$$Register, $src$$Register);
 6768   %}
 6769   ins_pipe(ialu_reg);
 6770 %}
 6771 
 6772 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6773   predicate(UsePopCountInstruction);
 6774   match(Set dst (PopCountI (LoadI mem)));
 6775   effect(KILL cr);
 6776 
 6777   format %{ "popcnt  $dst, $mem" %}
 6778   ins_encode %{
 6779     __ popcntl($dst$$Register, $mem$$Address);
 6780   %}
 6781   ins_pipe(ialu_reg);
 6782 %}
 6783 
 6784 // Note: Long.bitCount(long) returns an int.
 6785 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6786   predicate(UsePopCountInstruction);
 6787   match(Set dst (PopCountL src));
 6788   effect(KILL cr);
 6789 
 6790   format %{ "popcnt  $dst, $src" %}
 6791   ins_encode %{
 6792     __ popcntq($dst$$Register, $src$$Register);
 6793   %}
 6794   ins_pipe(ialu_reg);
 6795 %}
 6796 
 6797 // Note: Long.bitCount(long) returns an int.
 6798 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6799   predicate(UsePopCountInstruction);
 6800   match(Set dst (PopCountL (LoadL mem)));
 6801   effect(KILL cr);
 6802 
 6803   format %{ "popcnt  $dst, $mem" %}
 6804   ins_encode %{
 6805     __ popcntq($dst$$Register, $mem$$Address);
 6806   %}
 6807   ins_pipe(ialu_reg);
 6808 %}
 6809 
 6810 
 6811 //----------MemBar Instructions-----------------------------------------------
 6812 // Memory barrier flavors
 6813 
 6814 instruct membar_acquire()
 6815 %{
 6816   match(MemBarAcquire);
 6817   match(LoadFence);
 6818   ins_cost(0);
 6819 
 6820   size(0);
 6821   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6822   ins_encode();
 6823   ins_pipe(empty);
 6824 %}
 6825 
 6826 instruct membar_acquire_lock()
 6827 %{
 6828   match(MemBarAcquireLock);
 6829   ins_cost(0);
 6830 
 6831   size(0);
 6832   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6833   ins_encode();
 6834   ins_pipe(empty);
 6835 %}
 6836 
 6837 instruct membar_release()
 6838 %{
 6839   match(MemBarRelease);
 6840   match(StoreFence);
 6841   ins_cost(0);
 6842 
 6843   size(0);
 6844   format %{ "MEMBAR-release ! (empty encoding)" %}
 6845   ins_encode();
 6846   ins_pipe(empty);
 6847 %}
 6848 
 6849 instruct membar_release_lock()
 6850 %{
 6851   match(MemBarReleaseLock);
 6852   ins_cost(0);
 6853 
 6854   size(0);
 6855   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6856   ins_encode();
 6857   ins_pipe(empty);
 6858 %}
 6859 
 6860 instruct membar_volatile(rFlagsReg cr) %{
 6861   match(MemBarVolatile);
 6862   effect(KILL cr);
 6863   ins_cost(400);
 6864 
 6865   format %{
 6866     $$template
 6867     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6868   %}
 6869   ins_encode %{
 6870     __ membar(Assembler::StoreLoad);
 6871   %}
 6872   ins_pipe(pipe_slow);
 6873 %}
 6874 
 6875 instruct unnecessary_membar_volatile()
 6876 %{
 6877   match(MemBarVolatile);
 6878   predicate(Matcher::post_store_load_barrier(n));
 6879   ins_cost(0);
 6880 
 6881   size(0);
 6882   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6883   ins_encode();
 6884   ins_pipe(empty);
 6885 %}
 6886 
 6887 instruct membar_storestore() %{
 6888   match(MemBarStoreStore);
 6889   match(StoreStoreFence);
 6890   ins_cost(0);
 6891 
 6892   size(0);
 6893   format %{ "MEMBAR-storestore (empty encoding)" %}
 6894   ins_encode( );
 6895   ins_pipe(empty);
 6896 %}
 6897 
 6898 //----------Move Instructions--------------------------------------------------
 6899 
 6900 instruct castX2P(rRegP dst, rRegL src)
 6901 %{
 6902   match(Set dst (CastX2P src));
 6903 
 6904   format %{ "movq    $dst, $src\t# long->ptr" %}
 6905   ins_encode %{
 6906     if ($dst$$reg != $src$$reg) {
 6907       __ movptr($dst$$Register, $src$$Register);
 6908     }
 6909   %}
 6910   ins_pipe(ialu_reg_reg); // XXX
 6911 %}
 6912 
 6913 instruct castP2X(rRegL dst, rRegP src)
 6914 %{
 6915   match(Set dst (CastP2X src));
 6916 
 6917   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6918   ins_encode %{
 6919     if ($dst$$reg != $src$$reg) {
 6920       __ movptr($dst$$Register, $src$$Register);
 6921     }
 6922   %}
 6923   ins_pipe(ialu_reg_reg); // XXX
 6924 %}
 6925 
 6926 // Convert oop into int for vectors alignment masking
 6927 instruct convP2I(rRegI dst, rRegP src)
 6928 %{
 6929   match(Set dst (ConvL2I (CastP2X src)));
 6930 
 6931   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6932   ins_encode %{
 6933     __ movl($dst$$Register, $src$$Register);
 6934   %}
 6935   ins_pipe(ialu_reg_reg); // XXX
 6936 %}
 6937 
 6938 // Convert compressed oop into int for vectors alignment masking
 6939 // in case of 32bit oops (heap < 4Gb).
 6940 instruct convN2I(rRegI dst, rRegN src)
 6941 %{
 6942   predicate(CompressedOops::shift() == 0);
 6943   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6944 
 6945   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6946   ins_encode %{
 6947     __ movl($dst$$Register, $src$$Register);
 6948   %}
 6949   ins_pipe(ialu_reg_reg); // XXX
 6950 %}
 6951 
 6952 // Convert oop pointer into compressed form
 6953 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6954   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6955   match(Set dst (EncodeP src));
 6956   effect(KILL cr);
 6957   format %{ "encode_heap_oop $dst,$src" %}
 6958   ins_encode %{
 6959     Register s = $src$$Register;
 6960     Register d = $dst$$Register;
 6961     if (s != d) {
 6962       __ movq(d, s);
 6963     }
 6964     __ encode_heap_oop(d);
 6965   %}
 6966   ins_pipe(ialu_reg_long);
 6967 %}
 6968 
 6969 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6970   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6971   match(Set dst (EncodeP src));
 6972   effect(KILL cr);
 6973   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6974   ins_encode %{
 6975     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6976   %}
 6977   ins_pipe(ialu_reg_long);
 6978 %}
 6979 
 6980 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6981   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6982             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6983   match(Set dst (DecodeN src));
 6984   effect(KILL cr);
 6985   format %{ "decode_heap_oop $dst,$src" %}
 6986   ins_encode %{
 6987     Register s = $src$$Register;
 6988     Register d = $dst$$Register;
 6989     if (s != d) {
 6990       __ movq(d, s);
 6991     }
 6992     __ decode_heap_oop(d);
 6993   %}
 6994   ins_pipe(ialu_reg_long);
 6995 %}
 6996 
 6997 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6998   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6999             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7000   match(Set dst (DecodeN src));
 7001   effect(KILL cr);
 7002   format %{ "decode_heap_oop_not_null $dst,$src" %}
 7003   ins_encode %{
 7004     Register s = $src$$Register;
 7005     Register d = $dst$$Register;
 7006     if (s != d) {
 7007       __ decode_heap_oop_not_null(d, s);
 7008     } else {
 7009       __ decode_heap_oop_not_null(d);
 7010     }
 7011   %}
 7012   ins_pipe(ialu_reg_long);
 7013 %}
 7014 
 7015 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7016   match(Set dst (EncodePKlass src));
 7017   effect(TEMP dst, KILL cr);
 7018   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7019   ins_encode %{
 7020     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7021   %}
 7022   ins_pipe(ialu_reg_long);
 7023 %}
 7024 
 7025 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7026   match(Set dst (DecodeNKlass src));
 7027   effect(TEMP dst, KILL cr);
 7028   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7029   ins_encode %{
 7030     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7031   %}
 7032   ins_pipe(ialu_reg_long);
 7033 %}
 7034 
 7035 //----------Conditional Move---------------------------------------------------
 7036 // Jump
 7037 // dummy instruction for generating temp registers
 7038 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7039   match(Jump (LShiftL switch_val shift));
 7040   ins_cost(350);
 7041   predicate(false);
 7042   effect(TEMP dest);
 7043 
 7044   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7045             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7046   ins_encode %{
 7047     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7048     // to do that and the compiler is using that register as one it can allocate.
 7049     // So we build it all by hand.
 7050     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7051     // ArrayAddress dispatch(table, index);
 7052     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7053     __ lea($dest$$Register, $constantaddress);
 7054     __ jmp(dispatch);
 7055   %}
 7056   ins_pipe(pipe_jmp);
 7057 %}
 7058 
 7059 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7060   match(Jump (AddL (LShiftL switch_val shift) offset));
 7061   ins_cost(350);
 7062   effect(TEMP dest);
 7063 
 7064   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7065             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7066   ins_encode %{
 7067     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7068     // to do that and the compiler is using that register as one it can allocate.
 7069     // So we build it all by hand.
 7070     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7071     // ArrayAddress dispatch(table, index);
 7072     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7073     __ lea($dest$$Register, $constantaddress);
 7074     __ jmp(dispatch);
 7075   %}
 7076   ins_pipe(pipe_jmp);
 7077 %}
 7078 
 7079 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7080   match(Jump switch_val);
 7081   ins_cost(350);
 7082   effect(TEMP dest);
 7083 
 7084   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7085             "jmp     [$dest + $switch_val]\n\t" %}
 7086   ins_encode %{
 7087     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7088     // to do that and the compiler is using that register as one it can allocate.
 7089     // So we build it all by hand.
 7090     // Address index(noreg, switch_reg, Address::times_1);
 7091     // ArrayAddress dispatch(table, index);
 7092     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7093     __ lea($dest$$Register, $constantaddress);
 7094     __ jmp(dispatch);
 7095   %}
 7096   ins_pipe(pipe_jmp);
 7097 %}
 7098 
 7099 // Conditional move
 7100 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7101 %{
 7102   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7103   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7104 
 7105   ins_cost(100); // XXX
 7106   format %{ "setbn$cop $dst\t# signed, int" %}
 7107   ins_encode %{
 7108     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7109     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7110   %}
 7111   ins_pipe(ialu_reg);
 7112 %}
 7113 
 7114 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7115 %{
 7116   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7117 
 7118   ins_cost(200); // XXX
 7119   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7120   ins_encode %{
 7121     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7122   %}
 7123   ins_pipe(pipe_cmov_reg);
 7124 %}
 7125 
 7126 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7127 %{
 7128   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7129   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7130 
 7131   ins_cost(100); // XXX
 7132   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7133   ins_encode %{
 7134     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7135     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7136   %}
 7137   ins_pipe(ialu_reg);
 7138 %}
 7139 
 7140 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7141   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7142 
 7143   ins_cost(200); // XXX
 7144   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7145   ins_encode %{
 7146     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7147   %}
 7148   ins_pipe(pipe_cmov_reg);
 7149 %}
 7150 
 7151 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7152 %{
 7153   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7154   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7155 
 7156   ins_cost(100); // XXX
 7157   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7158   ins_encode %{
 7159     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7160     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7161   %}
 7162   ins_pipe(ialu_reg);
 7163 %}
 7164 
 7165 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7166   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7167   ins_cost(200);
 7168   expand %{
 7169     cmovI_regU(cop, cr, dst, src);
 7170   %}
 7171 %}
 7172 
 7173 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7174   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7175   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7176 
 7177   ins_cost(200); // XXX
 7178   format %{ "cmovpl  $dst, $src\n\t"
 7179             "cmovnel $dst, $src" %}
 7180   ins_encode %{
 7181     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7182     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7183   %}
 7184   ins_pipe(pipe_cmov_reg);
 7185 %}
 7186 
 7187 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7188 // inputs of the CMove
 7189 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7190   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7191   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7192 
 7193   ins_cost(200); // XXX
 7194   format %{ "cmovpl  $dst, $src\n\t"
 7195             "cmovnel $dst, $src" %}
 7196   ins_encode %{
 7197     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7198     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7199   %}
 7200   ins_pipe(pipe_cmov_reg);
 7201 %}
 7202 
 7203 // Conditional move
 7204 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7205   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7206 
 7207   ins_cost(250); // XXX
 7208   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7209   ins_encode %{
 7210     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7211   %}
 7212   ins_pipe(pipe_cmov_mem);
 7213 %}
 7214 
 7215 // Conditional move
 7216 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7217 %{
 7218   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7219 
 7220   ins_cost(250); // XXX
 7221   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7222   ins_encode %{
 7223     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7224   %}
 7225   ins_pipe(pipe_cmov_mem);
 7226 %}
 7227 
 7228 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7229   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7230   ins_cost(250);
 7231   expand %{
 7232     cmovI_memU(cop, cr, dst, src);
 7233   %}
 7234 %}
 7235 
 7236 // Conditional move
 7237 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7238 %{
 7239   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7240 
 7241   ins_cost(200); // XXX
 7242   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7243   ins_encode %{
 7244     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7245   %}
 7246   ins_pipe(pipe_cmov_reg);
 7247 %}
 7248 
 7249 // Conditional move
 7250 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7251 %{
 7252   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7253 
 7254   ins_cost(200); // XXX
 7255   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7256   ins_encode %{
 7257     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7258   %}
 7259   ins_pipe(pipe_cmov_reg);
 7260 %}
 7261 
 7262 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7263   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7264   ins_cost(200);
 7265   expand %{
 7266     cmovN_regU(cop, cr, dst, src);
 7267   %}
 7268 %}
 7269 
 7270 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7271   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7272   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7273 
 7274   ins_cost(200); // XXX
 7275   format %{ "cmovpl  $dst, $src\n\t"
 7276             "cmovnel $dst, $src" %}
 7277   ins_encode %{
 7278     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7279     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7280   %}
 7281   ins_pipe(pipe_cmov_reg);
 7282 %}
 7283 
 7284 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7285 // inputs of the CMove
 7286 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7287   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7288   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7289 
 7290   ins_cost(200); // XXX
 7291   format %{ "cmovpl  $dst, $src\n\t"
 7292             "cmovnel $dst, $src" %}
 7293   ins_encode %{
 7294     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7295     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7296   %}
 7297   ins_pipe(pipe_cmov_reg);
 7298 %}
 7299 
 7300 // Conditional move
 7301 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7302 %{
 7303   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7304 
 7305   ins_cost(200); // XXX
 7306   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7307   ins_encode %{
 7308     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7309   %}
 7310   ins_pipe(pipe_cmov_reg);  // XXX
 7311 %}
 7312 
 7313 // Conditional move
 7314 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7315 %{
 7316   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7317 
 7318   ins_cost(200); // XXX
 7319   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7320   ins_encode %{
 7321     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7322   %}
 7323   ins_pipe(pipe_cmov_reg); // XXX
 7324 %}
 7325 
 7326 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7327   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7328   ins_cost(200);
 7329   expand %{
 7330     cmovP_regU(cop, cr, dst, src);
 7331   %}
 7332 %}
 7333 
 7334 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7335   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7336   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7337 
 7338   ins_cost(200); // XXX
 7339   format %{ "cmovpq  $dst, $src\n\t"
 7340             "cmovneq $dst, $src" %}
 7341   ins_encode %{
 7342     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7343     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7344   %}
 7345   ins_pipe(pipe_cmov_reg);
 7346 %}
 7347 
 7348 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7349 // inputs of the CMove
 7350 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7351   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7352   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7353 
 7354   ins_cost(200); // XXX
 7355   format %{ "cmovpq  $dst, $src\n\t"
 7356             "cmovneq $dst, $src" %}
 7357   ins_encode %{
 7358     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7359     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7360   %}
 7361   ins_pipe(pipe_cmov_reg);
 7362 %}
 7363 
 7364 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7365 // correctly meets the two pointer arguments; one is an incoming
 7366 // register but the other is a memory operand.  ALSO appears to
 7367 // be buggy with implicit null checks.
 7368 //
 7369 //// Conditional move
 7370 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7371 //%{
 7372 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7373 //  ins_cost(250);
 7374 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7375 //  opcode(0x0F,0x40);
 7376 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7377 //  ins_pipe( pipe_cmov_mem );
 7378 //%}
 7379 //
 7380 //// Conditional move
 7381 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7382 //%{
 7383 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7384 //  ins_cost(250);
 7385 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7386 //  opcode(0x0F,0x40);
 7387 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7388 //  ins_pipe( pipe_cmov_mem );
 7389 //%}
 7390 
 7391 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7392 %{
 7393   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7394   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7395 
 7396   ins_cost(100); // XXX
 7397   format %{ "setbn$cop $dst\t# signed, long" %}
 7398   ins_encode %{
 7399     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7400     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7401   %}
 7402   ins_pipe(ialu_reg);
 7403 %}
 7404 
 7405 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7406 %{
 7407   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7408 
 7409   ins_cost(200); // XXX
 7410   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7411   ins_encode %{
 7412     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7413   %}
 7414   ins_pipe(pipe_cmov_reg);  // XXX
 7415 %}
 7416 
 7417 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7418 %{
 7419   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7420 
 7421   ins_cost(200); // XXX
 7422   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7423   ins_encode %{
 7424     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7425   %}
 7426   ins_pipe(pipe_cmov_mem);  // XXX
 7427 %}
 7428 
 7429 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7430 %{
 7431   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7432   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7433 
 7434   ins_cost(100); // XXX
 7435   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7436   ins_encode %{
 7437     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7438     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7439   %}
 7440   ins_pipe(ialu_reg);
 7441 %}
 7442 
 7443 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7444 %{
 7445   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7446 
 7447   ins_cost(200); // XXX
 7448   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7449   ins_encode %{
 7450     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7451   %}
 7452   ins_pipe(pipe_cmov_reg); // XXX
 7453 %}
 7454 
 7455 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7456 %{
 7457   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7458   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7459 
 7460   ins_cost(100); // XXX
 7461   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7462   ins_encode %{
 7463     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7464     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7465   %}
 7466   ins_pipe(ialu_reg);
 7467 %}
 7468 
 7469 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7470   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7471   ins_cost(200);
 7472   expand %{
 7473     cmovL_regU(cop, cr, dst, src);
 7474   %}
 7475 %}
 7476 
 7477 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7478   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7479   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7480 
 7481   ins_cost(200); // XXX
 7482   format %{ "cmovpq  $dst, $src\n\t"
 7483             "cmovneq $dst, $src" %}
 7484   ins_encode %{
 7485     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7486     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7487   %}
 7488   ins_pipe(pipe_cmov_reg);
 7489 %}
 7490 
 7491 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7492 // inputs of the CMove
 7493 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7494   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7495   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7496 
 7497   ins_cost(200); // XXX
 7498   format %{ "cmovpq  $dst, $src\n\t"
 7499             "cmovneq $dst, $src" %}
 7500   ins_encode %{
 7501     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7502     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7503   %}
 7504   ins_pipe(pipe_cmov_reg);
 7505 %}
 7506 
 7507 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7508 %{
 7509   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7510 
 7511   ins_cost(200); // XXX
 7512   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7513   ins_encode %{
 7514     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7515   %}
 7516   ins_pipe(pipe_cmov_mem); // XXX
 7517 %}
 7518 
 7519 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7520   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7521   ins_cost(200);
 7522   expand %{
 7523     cmovL_memU(cop, cr, dst, src);
 7524   %}
 7525 %}
 7526 
 7527 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7528 %{
 7529   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7530 
 7531   ins_cost(200); // XXX
 7532   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7533             "movss     $dst, $src\n"
 7534     "skip:" %}
 7535   ins_encode %{
 7536     Label Lskip;
 7537     // Invert sense of branch from sense of CMOV
 7538     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7539     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7540     __ bind(Lskip);
 7541   %}
 7542   ins_pipe(pipe_slow);
 7543 %}
 7544 
 7545 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7546 // %{
 7547 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7548 
 7549 //   ins_cost(200); // XXX
 7550 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7551 //             "movss     $dst, $src\n"
 7552 //     "skip:" %}
 7553 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7554 //   ins_pipe(pipe_slow);
 7555 // %}
 7556 
 7557 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7558 %{
 7559   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7560 
 7561   ins_cost(200); // XXX
 7562   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7563             "movss     $dst, $src\n"
 7564     "skip:" %}
 7565   ins_encode %{
 7566     Label Lskip;
 7567     // Invert sense of branch from sense of CMOV
 7568     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7569     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7570     __ bind(Lskip);
 7571   %}
 7572   ins_pipe(pipe_slow);
 7573 %}
 7574 
 7575 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7576   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7577   ins_cost(200);
 7578   expand %{
 7579     cmovF_regU(cop, cr, dst, src);
 7580   %}
 7581 %}
 7582 
 7583 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7584 %{
 7585   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7586 
 7587   ins_cost(200); // XXX
 7588   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7589             "movsd     $dst, $src\n"
 7590     "skip:" %}
 7591   ins_encode %{
 7592     Label Lskip;
 7593     // Invert sense of branch from sense of CMOV
 7594     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7595     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7596     __ bind(Lskip);
 7597   %}
 7598   ins_pipe(pipe_slow);
 7599 %}
 7600 
 7601 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7602 %{
 7603   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7604 
 7605   ins_cost(200); // XXX
 7606   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7607             "movsd     $dst, $src\n"
 7608     "skip:" %}
 7609   ins_encode %{
 7610     Label Lskip;
 7611     // Invert sense of branch from sense of CMOV
 7612     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7613     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7614     __ bind(Lskip);
 7615   %}
 7616   ins_pipe(pipe_slow);
 7617 %}
 7618 
 7619 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7620   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7621   ins_cost(200);
 7622   expand %{
 7623     cmovD_regU(cop, cr, dst, src);
 7624   %}
 7625 %}
 7626 
 7627 //----------Arithmetic Instructions--------------------------------------------
 7628 //----------Addition Instructions----------------------------------------------
 7629 
 7630 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7631 %{
 7632   match(Set dst (AddI dst src));
 7633   effect(KILL cr);
 7634 
 7635   format %{ "addl    $dst, $src\t# int" %}
 7636   ins_encode %{
 7637     __ addl($dst$$Register, $src$$Register);
 7638   %}
 7639   ins_pipe(ialu_reg_reg);
 7640 %}
 7641 
 7642 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7643 %{
 7644   match(Set dst (AddI dst src));
 7645   effect(KILL cr);
 7646 
 7647   format %{ "addl    $dst, $src\t# int" %}
 7648   ins_encode %{
 7649     __ addl($dst$$Register, $src$$constant);
 7650   %}
 7651   ins_pipe( ialu_reg );
 7652 %}
 7653 
 7654 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7655 %{
 7656   match(Set dst (AddI dst (LoadI src)));
 7657   effect(KILL cr);
 7658 
 7659   ins_cost(150); // XXX
 7660   format %{ "addl    $dst, $src\t# int" %}
 7661   ins_encode %{
 7662     __ addl($dst$$Register, $src$$Address);
 7663   %}
 7664   ins_pipe(ialu_reg_mem);
 7665 %}
 7666 
 7667 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7668 %{
 7669   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7670   effect(KILL cr);
 7671 
 7672   ins_cost(150); // XXX
 7673   format %{ "addl    $dst, $src\t# int" %}
 7674   ins_encode %{
 7675     __ addl($dst$$Address, $src$$Register);
 7676   %}
 7677   ins_pipe(ialu_mem_reg);
 7678 %}
 7679 
 7680 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7681 %{
 7682   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7683   effect(KILL cr);
 7684 
 7685   ins_cost(125); // XXX
 7686   format %{ "addl    $dst, $src\t# int" %}
 7687   ins_encode %{
 7688     __ addl($dst$$Address, $src$$constant);
 7689   %}
 7690   ins_pipe(ialu_mem_imm);
 7691 %}
 7692 
 7693 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7694 %{
 7695   predicate(UseIncDec);
 7696   match(Set dst (AddI dst src));
 7697   effect(KILL cr);
 7698 
 7699   format %{ "incl    $dst\t# int" %}
 7700   ins_encode %{
 7701     __ incrementl($dst$$Register);
 7702   %}
 7703   ins_pipe(ialu_reg);
 7704 %}
 7705 
 7706 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7707 %{
 7708   predicate(UseIncDec);
 7709   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7710   effect(KILL cr);
 7711 
 7712   ins_cost(125); // XXX
 7713   format %{ "incl    $dst\t# int" %}
 7714   ins_encode %{
 7715     __ incrementl($dst$$Address);
 7716   %}
 7717   ins_pipe(ialu_mem_imm);
 7718 %}
 7719 
 7720 // XXX why does that use AddI
 7721 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7722 %{
 7723   predicate(UseIncDec);
 7724   match(Set dst (AddI dst src));
 7725   effect(KILL cr);
 7726 
 7727   format %{ "decl    $dst\t# int" %}
 7728   ins_encode %{
 7729     __ decrementl($dst$$Register);
 7730   %}
 7731   ins_pipe(ialu_reg);
 7732 %}
 7733 
 7734 // XXX why does that use AddI
 7735 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7736 %{
 7737   predicate(UseIncDec);
 7738   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7739   effect(KILL cr);
 7740 
 7741   ins_cost(125); // XXX
 7742   format %{ "decl    $dst\t# int" %}
 7743   ins_encode %{
 7744     __ decrementl($dst$$Address);
 7745   %}
 7746   ins_pipe(ialu_mem_imm);
 7747 %}
 7748 
 7749 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7750 %{
 7751   predicate(VM_Version::supports_fast_2op_lea());
 7752   match(Set dst (AddI (LShiftI index scale) disp));
 7753 
 7754   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7755   ins_encode %{
 7756     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7757     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7758   %}
 7759   ins_pipe(ialu_reg_reg);
 7760 %}
 7761 
 7762 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7763 %{
 7764   predicate(VM_Version::supports_fast_3op_lea());
 7765   match(Set dst (AddI (AddI base index) disp));
 7766 
 7767   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7768   ins_encode %{
 7769     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7770   %}
 7771   ins_pipe(ialu_reg_reg);
 7772 %}
 7773 
 7774 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7775 %{
 7776   predicate(VM_Version::supports_fast_2op_lea());
 7777   match(Set dst (AddI base (LShiftI index scale)));
 7778 
 7779   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7780   ins_encode %{
 7781     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7782     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7783   %}
 7784   ins_pipe(ialu_reg_reg);
 7785 %}
 7786 
 7787 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7788 %{
 7789   predicate(VM_Version::supports_fast_3op_lea());
 7790   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7791 
 7792   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7793   ins_encode %{
 7794     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7795     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7796   %}
 7797   ins_pipe(ialu_reg_reg);
 7798 %}
 7799 
 7800 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7801 %{
 7802   match(Set dst (AddL dst src));
 7803   effect(KILL cr);
 7804 
 7805   format %{ "addq    $dst, $src\t# long" %}
 7806   ins_encode %{
 7807     __ addq($dst$$Register, $src$$Register);
 7808   %}
 7809   ins_pipe(ialu_reg_reg);
 7810 %}
 7811 
 7812 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7813 %{
 7814   match(Set dst (AddL dst src));
 7815   effect(KILL cr);
 7816 
 7817   format %{ "addq    $dst, $src\t# long" %}
 7818   ins_encode %{
 7819     __ addq($dst$$Register, $src$$constant);
 7820   %}
 7821   ins_pipe( ialu_reg );
 7822 %}
 7823 
 7824 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7825 %{
 7826   match(Set dst (AddL dst (LoadL src)));
 7827   effect(KILL cr);
 7828 
 7829   ins_cost(150); // XXX
 7830   format %{ "addq    $dst, $src\t# long" %}
 7831   ins_encode %{
 7832     __ addq($dst$$Register, $src$$Address);
 7833   %}
 7834   ins_pipe(ialu_reg_mem);
 7835 %}
 7836 
 7837 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7838 %{
 7839   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7840   effect(KILL cr);
 7841 
 7842   ins_cost(150); // XXX
 7843   format %{ "addq    $dst, $src\t# long" %}
 7844   ins_encode %{
 7845     __ addq($dst$$Address, $src$$Register);
 7846   %}
 7847   ins_pipe(ialu_mem_reg);
 7848 %}
 7849 
 7850 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7851 %{
 7852   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7853   effect(KILL cr);
 7854 
 7855   ins_cost(125); // XXX
 7856   format %{ "addq    $dst, $src\t# long" %}
 7857   ins_encode %{
 7858     __ addq($dst$$Address, $src$$constant);
 7859   %}
 7860   ins_pipe(ialu_mem_imm);
 7861 %}
 7862 
 7863 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7864 %{
 7865   predicate(UseIncDec);
 7866   match(Set dst (AddL dst src));
 7867   effect(KILL cr);
 7868 
 7869   format %{ "incq    $dst\t# long" %}
 7870   ins_encode %{
 7871     __ incrementq($dst$$Register);
 7872   %}
 7873   ins_pipe(ialu_reg);
 7874 %}
 7875 
 7876 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7877 %{
 7878   predicate(UseIncDec);
 7879   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7880   effect(KILL cr);
 7881 
 7882   ins_cost(125); // XXX
 7883   format %{ "incq    $dst\t# long" %}
 7884   ins_encode %{
 7885     __ incrementq($dst$$Address);
 7886   %}
 7887   ins_pipe(ialu_mem_imm);
 7888 %}
 7889 
 7890 // XXX why does that use AddL
 7891 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7892 %{
 7893   predicate(UseIncDec);
 7894   match(Set dst (AddL dst src));
 7895   effect(KILL cr);
 7896 
 7897   format %{ "decq    $dst\t# long" %}
 7898   ins_encode %{
 7899     __ decrementq($dst$$Register);
 7900   %}
 7901   ins_pipe(ialu_reg);
 7902 %}
 7903 
 7904 // XXX why does that use AddL
 7905 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7906 %{
 7907   predicate(UseIncDec);
 7908   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7909   effect(KILL cr);
 7910 
 7911   ins_cost(125); // XXX
 7912   format %{ "decq    $dst\t# long" %}
 7913   ins_encode %{
 7914     __ decrementq($dst$$Address);
 7915   %}
 7916   ins_pipe(ialu_mem_imm);
 7917 %}
 7918 
 7919 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7920 %{
 7921   predicate(VM_Version::supports_fast_2op_lea());
 7922   match(Set dst (AddL (LShiftL index scale) disp));
 7923 
 7924   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7925   ins_encode %{
 7926     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7927     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7928   %}
 7929   ins_pipe(ialu_reg_reg);
 7930 %}
 7931 
 7932 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7933 %{
 7934   predicate(VM_Version::supports_fast_3op_lea());
 7935   match(Set dst (AddL (AddL base index) disp));
 7936 
 7937   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7938   ins_encode %{
 7939     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7940   %}
 7941   ins_pipe(ialu_reg_reg);
 7942 %}
 7943 
 7944 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7945 %{
 7946   predicate(VM_Version::supports_fast_2op_lea());
 7947   match(Set dst (AddL base (LShiftL index scale)));
 7948 
 7949   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7950   ins_encode %{
 7951     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7952     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7953   %}
 7954   ins_pipe(ialu_reg_reg);
 7955 %}
 7956 
 7957 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7958 %{
 7959   predicate(VM_Version::supports_fast_3op_lea());
 7960   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7961 
 7962   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7963   ins_encode %{
 7964     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7965     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7966   %}
 7967   ins_pipe(ialu_reg_reg);
 7968 %}
 7969 
 7970 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7971 %{
 7972   match(Set dst (AddP dst src));
 7973   effect(KILL cr);
 7974 
 7975   format %{ "addq    $dst, $src\t# ptr" %}
 7976   ins_encode %{
 7977     __ addq($dst$$Register, $src$$Register);
 7978   %}
 7979   ins_pipe(ialu_reg_reg);
 7980 %}
 7981 
 7982 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7983 %{
 7984   match(Set dst (AddP dst src));
 7985   effect(KILL cr);
 7986 
 7987   format %{ "addq    $dst, $src\t# ptr" %}
 7988   ins_encode %{
 7989     __ addq($dst$$Register, $src$$constant);
 7990   %}
 7991   ins_pipe( ialu_reg );
 7992 %}
 7993 
 7994 // XXX addP mem ops ????
 7995 
 7996 instruct checkCastPP(rRegP dst)
 7997 %{
 7998   match(Set dst (CheckCastPP dst));
 7999 
 8000   size(0);
 8001   format %{ "# checkcastPP of $dst" %}
 8002   ins_encode(/* empty encoding */);
 8003   ins_pipe(empty);
 8004 %}
 8005 
 8006 instruct castPP(rRegP dst)
 8007 %{
 8008   match(Set dst (CastPP dst));
 8009 
 8010   size(0);
 8011   format %{ "# castPP of $dst" %}
 8012   ins_encode(/* empty encoding */);
 8013   ins_pipe(empty);
 8014 %}
 8015 
 8016 instruct castII(rRegI dst)
 8017 %{
 8018   match(Set dst (CastII dst));
 8019 
 8020   size(0);
 8021   format %{ "# castII of $dst" %}
 8022   ins_encode(/* empty encoding */);
 8023   ins_cost(0);
 8024   ins_pipe(empty);
 8025 %}
 8026 
 8027 instruct castLL(rRegL dst)
 8028 %{
 8029   match(Set dst (CastLL dst));
 8030 
 8031   size(0);
 8032   format %{ "# castLL of $dst" %}
 8033   ins_encode(/* empty encoding */);
 8034   ins_cost(0);
 8035   ins_pipe(empty);
 8036 %}
 8037 
 8038 instruct castFF(regF dst)
 8039 %{
 8040   match(Set dst (CastFF dst));
 8041 
 8042   size(0);
 8043   format %{ "# castFF of $dst" %}
 8044   ins_encode(/* empty encoding */);
 8045   ins_cost(0);
 8046   ins_pipe(empty);
 8047 %}
 8048 
 8049 instruct castDD(regD dst)
 8050 %{
 8051   match(Set dst (CastDD dst));
 8052 
 8053   size(0);
 8054   format %{ "# castDD of $dst" %}
 8055   ins_encode(/* empty encoding */);
 8056   ins_cost(0);
 8057   ins_pipe(empty);
 8058 %}
 8059 
 8060 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8061 instruct compareAndSwapP(rRegI res,
 8062                          memory mem_ptr,
 8063                          rax_RegP oldval, rRegP newval,
 8064                          rFlagsReg cr)
 8065 %{
 8066   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8067   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8068   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8069   effect(KILL cr, KILL oldval);
 8070 
 8071   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8072             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8073             "sete    $res\n\t"
 8074             "movzbl  $res, $res" %}
 8075   ins_encode %{
 8076     __ lock();
 8077     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8078     __ sete($res$$Register);
 8079     __ movzbl($res$$Register, $res$$Register);
 8080   %}
 8081   ins_pipe( pipe_cmpxchg );
 8082 %}
 8083 
 8084 instruct compareAndSwapL(rRegI res,
 8085                          memory mem_ptr,
 8086                          rax_RegL oldval, rRegL newval,
 8087                          rFlagsReg cr)
 8088 %{
 8089   predicate(VM_Version::supports_cx8());
 8090   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8091   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8092   effect(KILL cr, KILL oldval);
 8093 
 8094   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8095             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8096             "sete    $res\n\t"
 8097             "movzbl  $res, $res" %}
 8098   ins_encode %{
 8099     __ lock();
 8100     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8101     __ sete($res$$Register);
 8102     __ movzbl($res$$Register, $res$$Register);
 8103   %}
 8104   ins_pipe( pipe_cmpxchg );
 8105 %}
 8106 
 8107 instruct compareAndSwapI(rRegI res,
 8108                          memory mem_ptr,
 8109                          rax_RegI oldval, rRegI newval,
 8110                          rFlagsReg cr)
 8111 %{
 8112   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8113   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8114   effect(KILL cr, KILL oldval);
 8115 
 8116   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8117             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8118             "sete    $res\n\t"
 8119             "movzbl  $res, $res" %}
 8120   ins_encode %{
 8121     __ lock();
 8122     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8123     __ sete($res$$Register);
 8124     __ movzbl($res$$Register, $res$$Register);
 8125   %}
 8126   ins_pipe( pipe_cmpxchg );
 8127 %}
 8128 
 8129 instruct compareAndSwapB(rRegI res,
 8130                          memory mem_ptr,
 8131                          rax_RegI oldval, rRegI newval,
 8132                          rFlagsReg cr)
 8133 %{
 8134   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8135   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8136   effect(KILL cr, KILL oldval);
 8137 
 8138   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8139             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8140             "sete    $res\n\t"
 8141             "movzbl  $res, $res" %}
 8142   ins_encode %{
 8143     __ lock();
 8144     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8145     __ sete($res$$Register);
 8146     __ movzbl($res$$Register, $res$$Register);
 8147   %}
 8148   ins_pipe( pipe_cmpxchg );
 8149 %}
 8150 
 8151 instruct compareAndSwapS(rRegI res,
 8152                          memory mem_ptr,
 8153                          rax_RegI oldval, rRegI newval,
 8154                          rFlagsReg cr)
 8155 %{
 8156   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8157   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8158   effect(KILL cr, KILL oldval);
 8159 
 8160   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8161             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8162             "sete    $res\n\t"
 8163             "movzbl  $res, $res" %}
 8164   ins_encode %{
 8165     __ lock();
 8166     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8167     __ sete($res$$Register);
 8168     __ movzbl($res$$Register, $res$$Register);
 8169   %}
 8170   ins_pipe( pipe_cmpxchg );
 8171 %}
 8172 
 8173 instruct compareAndSwapN(rRegI res,
 8174                           memory mem_ptr,
 8175                           rax_RegN oldval, rRegN newval,
 8176                           rFlagsReg cr) %{
 8177   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8178   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8179   effect(KILL cr, KILL oldval);
 8180 
 8181   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8182             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8183             "sete    $res\n\t"
 8184             "movzbl  $res, $res" %}
 8185   ins_encode %{
 8186     __ lock();
 8187     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8188     __ sete($res$$Register);
 8189     __ movzbl($res$$Register, $res$$Register);
 8190   %}
 8191   ins_pipe( pipe_cmpxchg );
 8192 %}
 8193 
 8194 instruct compareAndExchangeB(
 8195                          memory mem_ptr,
 8196                          rax_RegI oldval, rRegI newval,
 8197                          rFlagsReg cr)
 8198 %{
 8199   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8200   effect(KILL cr);
 8201 
 8202   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8203             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8204   ins_encode %{
 8205     __ lock();
 8206     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8207   %}
 8208   ins_pipe( pipe_cmpxchg );
 8209 %}
 8210 
 8211 instruct compareAndExchangeS(
 8212                          memory mem_ptr,
 8213                          rax_RegI oldval, rRegI newval,
 8214                          rFlagsReg cr)
 8215 %{
 8216   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8217   effect(KILL cr);
 8218 
 8219   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8220             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8221   ins_encode %{
 8222     __ lock();
 8223     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8224   %}
 8225   ins_pipe( pipe_cmpxchg );
 8226 %}
 8227 
 8228 instruct compareAndExchangeI(
 8229                          memory mem_ptr,
 8230                          rax_RegI oldval, rRegI newval,
 8231                          rFlagsReg cr)
 8232 %{
 8233   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8234   effect(KILL cr);
 8235 
 8236   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8237             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8238   ins_encode %{
 8239     __ lock();
 8240     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8241   %}
 8242   ins_pipe( pipe_cmpxchg );
 8243 %}
 8244 
 8245 instruct compareAndExchangeL(
 8246                          memory mem_ptr,
 8247                          rax_RegL oldval, rRegL newval,
 8248                          rFlagsReg cr)
 8249 %{
 8250   predicate(VM_Version::supports_cx8());
 8251   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8252   effect(KILL cr);
 8253 
 8254   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8255             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8256   ins_encode %{
 8257     __ lock();
 8258     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8259   %}
 8260   ins_pipe( pipe_cmpxchg );
 8261 %}
 8262 
 8263 instruct compareAndExchangeN(
 8264                           memory mem_ptr,
 8265                           rax_RegN oldval, rRegN newval,
 8266                           rFlagsReg cr) %{
 8267   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8268   effect(KILL cr);
 8269 
 8270   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8271             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8272   ins_encode %{
 8273     __ lock();
 8274     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8275   %}
 8276   ins_pipe( pipe_cmpxchg );
 8277 %}
 8278 
 8279 instruct compareAndExchangeP(
 8280                          memory mem_ptr,
 8281                          rax_RegP oldval, rRegP newval,
 8282                          rFlagsReg cr)
 8283 %{
 8284   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8285   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8286   effect(KILL cr);
 8287 
 8288   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8289             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8290   ins_encode %{
 8291     __ lock();
 8292     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8293   %}
 8294   ins_pipe( pipe_cmpxchg );
 8295 %}
 8296 
 8297 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8298   predicate(n->as_LoadStore()->result_not_used());
 8299   match(Set dummy (GetAndAddB mem add));
 8300   effect(KILL cr);
 8301   format %{ "ADDB  [$mem],$add" %}
 8302   ins_encode %{
 8303     __ lock();
 8304     __ addb($mem$$Address, $add$$constant);
 8305   %}
 8306   ins_pipe( pipe_cmpxchg );
 8307 %}
 8308 
 8309 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8310   match(Set newval (GetAndAddB mem newval));
 8311   effect(KILL cr);
 8312   format %{ "XADDB  [$mem],$newval" %}
 8313   ins_encode %{
 8314     __ lock();
 8315     __ xaddb($mem$$Address, $newval$$Register);
 8316   %}
 8317   ins_pipe( pipe_cmpxchg );
 8318 %}
 8319 
 8320 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8321   predicate(n->as_LoadStore()->result_not_used());
 8322   match(Set dummy (GetAndAddS mem add));
 8323   effect(KILL cr);
 8324   format %{ "ADDW  [$mem],$add" %}
 8325   ins_encode %{
 8326     __ lock();
 8327     __ addw($mem$$Address, $add$$constant);
 8328   %}
 8329   ins_pipe( pipe_cmpxchg );
 8330 %}
 8331 
 8332 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8333   match(Set newval (GetAndAddS mem newval));
 8334   effect(KILL cr);
 8335   format %{ "XADDW  [$mem],$newval" %}
 8336   ins_encode %{
 8337     __ lock();
 8338     __ xaddw($mem$$Address, $newval$$Register);
 8339   %}
 8340   ins_pipe( pipe_cmpxchg );
 8341 %}
 8342 
 8343 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8344   predicate(n->as_LoadStore()->result_not_used());
 8345   match(Set dummy (GetAndAddI mem add));
 8346   effect(KILL cr);
 8347   format %{ "ADDL  [$mem],$add" %}
 8348   ins_encode %{
 8349     __ lock();
 8350     __ addl($mem$$Address, $add$$constant);
 8351   %}
 8352   ins_pipe( pipe_cmpxchg );
 8353 %}
 8354 
 8355 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8356   match(Set newval (GetAndAddI mem newval));
 8357   effect(KILL cr);
 8358   format %{ "XADDL  [$mem],$newval" %}
 8359   ins_encode %{
 8360     __ lock();
 8361     __ xaddl($mem$$Address, $newval$$Register);
 8362   %}
 8363   ins_pipe( pipe_cmpxchg );
 8364 %}
 8365 
 8366 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8367   predicate(n->as_LoadStore()->result_not_used());
 8368   match(Set dummy (GetAndAddL mem add));
 8369   effect(KILL cr);
 8370   format %{ "ADDQ  [$mem],$add" %}
 8371   ins_encode %{
 8372     __ lock();
 8373     __ addq($mem$$Address, $add$$constant);
 8374   %}
 8375   ins_pipe( pipe_cmpxchg );
 8376 %}
 8377 
 8378 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8379   match(Set newval (GetAndAddL mem newval));
 8380   effect(KILL cr);
 8381   format %{ "XADDQ  [$mem],$newval" %}
 8382   ins_encode %{
 8383     __ lock();
 8384     __ xaddq($mem$$Address, $newval$$Register);
 8385   %}
 8386   ins_pipe( pipe_cmpxchg );
 8387 %}
 8388 
 8389 instruct xchgB( memory mem, rRegI newval) %{
 8390   match(Set newval (GetAndSetB mem newval));
 8391   format %{ "XCHGB  $newval,[$mem]" %}
 8392   ins_encode %{
 8393     __ xchgb($newval$$Register, $mem$$Address);
 8394   %}
 8395   ins_pipe( pipe_cmpxchg );
 8396 %}
 8397 
 8398 instruct xchgS( memory mem, rRegI newval) %{
 8399   match(Set newval (GetAndSetS mem newval));
 8400   format %{ "XCHGW  $newval,[$mem]" %}
 8401   ins_encode %{
 8402     __ xchgw($newval$$Register, $mem$$Address);
 8403   %}
 8404   ins_pipe( pipe_cmpxchg );
 8405 %}
 8406 
 8407 instruct xchgI( memory mem, rRegI newval) %{
 8408   match(Set newval (GetAndSetI mem newval));
 8409   format %{ "XCHGL  $newval,[$mem]" %}
 8410   ins_encode %{
 8411     __ xchgl($newval$$Register, $mem$$Address);
 8412   %}
 8413   ins_pipe( pipe_cmpxchg );
 8414 %}
 8415 
 8416 instruct xchgL( memory mem, rRegL newval) %{
 8417   match(Set newval (GetAndSetL mem newval));
 8418   format %{ "XCHGL  $newval,[$mem]" %}
 8419   ins_encode %{
 8420     __ xchgq($newval$$Register, $mem$$Address);
 8421   %}
 8422   ins_pipe( pipe_cmpxchg );
 8423 %}
 8424 
 8425 instruct xchgP( memory mem, rRegP newval) %{
 8426   match(Set newval (GetAndSetP mem newval));
 8427   predicate(n->as_LoadStore()->barrier_data() == 0);
 8428   format %{ "XCHGQ  $newval,[$mem]" %}
 8429   ins_encode %{
 8430     __ xchgq($newval$$Register, $mem$$Address);
 8431   %}
 8432   ins_pipe( pipe_cmpxchg );
 8433 %}
 8434 
 8435 instruct xchgN( memory mem, rRegN newval) %{
 8436   match(Set newval (GetAndSetN mem newval));
 8437   format %{ "XCHGL  $newval,$mem]" %}
 8438   ins_encode %{
 8439     __ xchgl($newval$$Register, $mem$$Address);
 8440   %}
 8441   ins_pipe( pipe_cmpxchg );
 8442 %}
 8443 
 8444 //----------Abs Instructions-------------------------------------------
 8445 
 8446 // Integer Absolute Instructions
 8447 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8448 %{
 8449   match(Set dst (AbsI src));
 8450   effect(TEMP dst, TEMP tmp, KILL cr);
 8451   format %{ "movl $tmp, $src\n\t"
 8452             "sarl $tmp, 31\n\t"
 8453             "movl $dst, $src\n\t"
 8454             "xorl $dst, $tmp\n\t"
 8455             "subl $dst, $tmp\n"
 8456           %}
 8457   ins_encode %{
 8458     __ movl($tmp$$Register, $src$$Register);
 8459     __ sarl($tmp$$Register, 31);
 8460     __ movl($dst$$Register, $src$$Register);
 8461     __ xorl($dst$$Register, $tmp$$Register);
 8462     __ subl($dst$$Register, $tmp$$Register);
 8463   %}
 8464 
 8465   ins_pipe(ialu_reg_reg);
 8466 %}
 8467 
 8468 // Long Absolute Instructions
 8469 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8470 %{
 8471   match(Set dst (AbsL src));
 8472   effect(TEMP dst, TEMP tmp, KILL cr);
 8473   format %{ "movq $tmp, $src\n\t"
 8474             "sarq $tmp, 63\n\t"
 8475             "movq $dst, $src\n\t"
 8476             "xorq $dst, $tmp\n\t"
 8477             "subq $dst, $tmp\n"
 8478           %}
 8479   ins_encode %{
 8480     __ movq($tmp$$Register, $src$$Register);
 8481     __ sarq($tmp$$Register, 63);
 8482     __ movq($dst$$Register, $src$$Register);
 8483     __ xorq($dst$$Register, $tmp$$Register);
 8484     __ subq($dst$$Register, $tmp$$Register);
 8485   %}
 8486 
 8487   ins_pipe(ialu_reg_reg);
 8488 %}
 8489 
 8490 //----------Subtraction Instructions-------------------------------------------
 8491 
 8492 // Integer Subtraction Instructions
 8493 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8494 %{
 8495   match(Set dst (SubI dst src));
 8496   effect(KILL cr);
 8497 
 8498   format %{ "subl    $dst, $src\t# int" %}
 8499   ins_encode %{
 8500     __ subl($dst$$Register, $src$$Register);
 8501   %}
 8502   ins_pipe(ialu_reg_reg);
 8503 %}
 8504 
 8505 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8506 %{
 8507   match(Set dst (SubI dst (LoadI src)));
 8508   effect(KILL cr);
 8509 
 8510   ins_cost(150);
 8511   format %{ "subl    $dst, $src\t# int" %}
 8512   ins_encode %{
 8513     __ subl($dst$$Register, $src$$Address);
 8514   %}
 8515   ins_pipe(ialu_reg_mem);
 8516 %}
 8517 
 8518 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8519 %{
 8520   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8521   effect(KILL cr);
 8522 
 8523   ins_cost(150);
 8524   format %{ "subl    $dst, $src\t# int" %}
 8525   ins_encode %{
 8526     __ subl($dst$$Address, $src$$Register);
 8527   %}
 8528   ins_pipe(ialu_mem_reg);
 8529 %}
 8530 
 8531 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8532 %{
 8533   match(Set dst (SubL dst src));
 8534   effect(KILL cr);
 8535 
 8536   format %{ "subq    $dst, $src\t# long" %}
 8537   ins_encode %{
 8538     __ subq($dst$$Register, $src$$Register);
 8539   %}
 8540   ins_pipe(ialu_reg_reg);
 8541 %}
 8542 
 8543 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8544 %{
 8545   match(Set dst (SubL dst (LoadL src)));
 8546   effect(KILL cr);
 8547 
 8548   ins_cost(150);
 8549   format %{ "subq    $dst, $src\t# long" %}
 8550   ins_encode %{
 8551     __ subq($dst$$Register, $src$$Address);
 8552   %}
 8553   ins_pipe(ialu_reg_mem);
 8554 %}
 8555 
 8556 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8557 %{
 8558   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8559   effect(KILL cr);
 8560 
 8561   ins_cost(150);
 8562   format %{ "subq    $dst, $src\t# long" %}
 8563   ins_encode %{
 8564     __ subq($dst$$Address, $src$$Register);
 8565   %}
 8566   ins_pipe(ialu_mem_reg);
 8567 %}
 8568 
 8569 // Subtract from a pointer
 8570 // XXX hmpf???
 8571 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8572 %{
 8573   match(Set dst (AddP dst (SubI zero src)));
 8574   effect(KILL cr);
 8575 
 8576   format %{ "subq    $dst, $src\t# ptr - int" %}
 8577   opcode(0x2B);
 8578   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8579   ins_pipe(ialu_reg_reg);
 8580 %}
 8581 
 8582 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8583 %{
 8584   match(Set dst (SubI zero dst));
 8585   effect(KILL cr);
 8586 
 8587   format %{ "negl    $dst\t# int" %}
 8588   ins_encode %{
 8589     __ negl($dst$$Register);
 8590   %}
 8591   ins_pipe(ialu_reg);
 8592 %}
 8593 
 8594 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8595 %{
 8596   match(Set dst (NegI dst));
 8597   effect(KILL cr);
 8598 
 8599   format %{ "negl    $dst\t# int" %}
 8600   ins_encode %{
 8601     __ negl($dst$$Register);
 8602   %}
 8603   ins_pipe(ialu_reg);
 8604 %}
 8605 
 8606 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8607 %{
 8608   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8609   effect(KILL cr);
 8610 
 8611   format %{ "negl    $dst\t# int" %}
 8612   ins_encode %{
 8613     __ negl($dst$$Address);
 8614   %}
 8615   ins_pipe(ialu_reg);
 8616 %}
 8617 
 8618 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8619 %{
 8620   match(Set dst (SubL zero dst));
 8621   effect(KILL cr);
 8622 
 8623   format %{ "negq    $dst\t# long" %}
 8624   ins_encode %{
 8625     __ negq($dst$$Register);
 8626   %}
 8627   ins_pipe(ialu_reg);
 8628 %}
 8629 
 8630 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8631 %{
 8632   match(Set dst (NegL dst));
 8633   effect(KILL cr);
 8634 
 8635   format %{ "negq    $dst\t# int" %}
 8636   ins_encode %{
 8637     __ negq($dst$$Register);
 8638   %}
 8639   ins_pipe(ialu_reg);
 8640 %}
 8641 
 8642 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8643 %{
 8644   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8645   effect(KILL cr);
 8646 
 8647   format %{ "negq    $dst\t# long" %}
 8648   ins_encode %{
 8649     __ negq($dst$$Address);
 8650   %}
 8651   ins_pipe(ialu_reg);
 8652 %}
 8653 
 8654 //----------Multiplication/Division Instructions-------------------------------
 8655 // Integer Multiplication Instructions
 8656 // Multiply Register
 8657 
 8658 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8659 %{
 8660   match(Set dst (MulI dst src));
 8661   effect(KILL cr);
 8662 
 8663   ins_cost(300);
 8664   format %{ "imull   $dst, $src\t# int" %}
 8665   ins_encode %{
 8666     __ imull($dst$$Register, $src$$Register);
 8667   %}
 8668   ins_pipe(ialu_reg_reg_alu0);
 8669 %}
 8670 
 8671 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8672 %{
 8673   match(Set dst (MulI src imm));
 8674   effect(KILL cr);
 8675 
 8676   ins_cost(300);
 8677   format %{ "imull   $dst, $src, $imm\t# int" %}
 8678   ins_encode %{
 8679     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8680   %}
 8681   ins_pipe(ialu_reg_reg_alu0);
 8682 %}
 8683 
 8684 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8685 %{
 8686   match(Set dst (MulI dst (LoadI src)));
 8687   effect(KILL cr);
 8688 
 8689   ins_cost(350);
 8690   format %{ "imull   $dst, $src\t# int" %}
 8691   ins_encode %{
 8692     __ imull($dst$$Register, $src$$Address);
 8693   %}
 8694   ins_pipe(ialu_reg_mem_alu0);
 8695 %}
 8696 
 8697 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8698 %{
 8699   match(Set dst (MulI (LoadI src) imm));
 8700   effect(KILL cr);
 8701 
 8702   ins_cost(300);
 8703   format %{ "imull   $dst, $src, $imm\t# int" %}
 8704   ins_encode %{
 8705     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8706   %}
 8707   ins_pipe(ialu_reg_mem_alu0);
 8708 %}
 8709 
 8710 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8711 %{
 8712   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8713   effect(KILL cr, KILL src2);
 8714 
 8715   expand %{ mulI_rReg(dst, src1, cr);
 8716            mulI_rReg(src2, src3, cr);
 8717            addI_rReg(dst, src2, cr); %}
 8718 %}
 8719 
 8720 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8721 %{
 8722   match(Set dst (MulL dst src));
 8723   effect(KILL cr);
 8724 
 8725   ins_cost(300);
 8726   format %{ "imulq   $dst, $src\t# long" %}
 8727   ins_encode %{
 8728     __ imulq($dst$$Register, $src$$Register);
 8729   %}
 8730   ins_pipe(ialu_reg_reg_alu0);
 8731 %}
 8732 
 8733 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8734 %{
 8735   match(Set dst (MulL src imm));
 8736   effect(KILL cr);
 8737 
 8738   ins_cost(300);
 8739   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8740   ins_encode %{
 8741     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8742   %}
 8743   ins_pipe(ialu_reg_reg_alu0);
 8744 %}
 8745 
 8746 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8747 %{
 8748   match(Set dst (MulL dst (LoadL src)));
 8749   effect(KILL cr);
 8750 
 8751   ins_cost(350);
 8752   format %{ "imulq   $dst, $src\t# long" %}
 8753   ins_encode %{
 8754     __ imulq($dst$$Register, $src$$Address);
 8755   %}
 8756   ins_pipe(ialu_reg_mem_alu0);
 8757 %}
 8758 
 8759 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8760 %{
 8761   match(Set dst (MulL (LoadL src) imm));
 8762   effect(KILL cr);
 8763 
 8764   ins_cost(300);
 8765   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8766   ins_encode %{
 8767     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8768   %}
 8769   ins_pipe(ialu_reg_mem_alu0);
 8770 %}
 8771 
 8772 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8773 %{
 8774   match(Set dst (MulHiL src rax));
 8775   effect(USE_KILL rax, KILL cr);
 8776 
 8777   ins_cost(300);
 8778   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8779   ins_encode %{
 8780     __ imulq($src$$Register);
 8781   %}
 8782   ins_pipe(ialu_reg_reg_alu0);
 8783 %}
 8784 
 8785 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8786 %{
 8787   match(Set dst (UMulHiL src rax));
 8788   effect(USE_KILL rax, KILL cr);
 8789 
 8790   ins_cost(300);
 8791   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8792   ins_encode %{
 8793     __ mulq($src$$Register);
 8794   %}
 8795   ins_pipe(ialu_reg_reg_alu0);
 8796 %}
 8797 
 8798 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8799                    rFlagsReg cr)
 8800 %{
 8801   match(Set rax (DivI rax div));
 8802   effect(KILL rdx, KILL cr);
 8803 
 8804   ins_cost(30*100+10*100); // XXX
 8805   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8806             "jne,s   normal\n\t"
 8807             "xorl    rdx, rdx\n\t"
 8808             "cmpl    $div, -1\n\t"
 8809             "je,s    done\n"
 8810     "normal: cdql\n\t"
 8811             "idivl   $div\n"
 8812     "done:"        %}
 8813   ins_encode(cdql_enc(div));
 8814   ins_pipe(ialu_reg_reg_alu0);
 8815 %}
 8816 
 8817 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8818                    rFlagsReg cr)
 8819 %{
 8820   match(Set rax (DivL rax div));
 8821   effect(KILL rdx, KILL cr);
 8822 
 8823   ins_cost(30*100+10*100); // XXX
 8824   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8825             "cmpq    rax, rdx\n\t"
 8826             "jne,s   normal\n\t"
 8827             "xorl    rdx, rdx\n\t"
 8828             "cmpq    $div, -1\n\t"
 8829             "je,s    done\n"
 8830     "normal: cdqq\n\t"
 8831             "idivq   $div\n"
 8832     "done:"        %}
 8833   ins_encode(cdqq_enc(div));
 8834   ins_pipe(ialu_reg_reg_alu0);
 8835 %}
 8836 
 8837 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8838 %{
 8839   match(Set rax (UDivI rax div));
 8840   effect(KILL rdx, KILL cr);
 8841 
 8842   ins_cost(300);
 8843   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8844   ins_encode %{
 8845     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8846   %}
 8847   ins_pipe(ialu_reg_reg_alu0);
 8848 %}
 8849 
 8850 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8851 %{
 8852   match(Set rax (UDivL rax div));
 8853   effect(KILL rdx, KILL cr);
 8854 
 8855   ins_cost(300);
 8856   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8857   ins_encode %{
 8858      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8859   %}
 8860   ins_pipe(ialu_reg_reg_alu0);
 8861 %}
 8862 
 8863 // Integer DIVMOD with Register, both quotient and mod results
 8864 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8865                              rFlagsReg cr)
 8866 %{
 8867   match(DivModI rax div);
 8868   effect(KILL cr);
 8869 
 8870   ins_cost(30*100+10*100); // XXX
 8871   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8872             "jne,s   normal\n\t"
 8873             "xorl    rdx, rdx\n\t"
 8874             "cmpl    $div, -1\n\t"
 8875             "je,s    done\n"
 8876     "normal: cdql\n\t"
 8877             "idivl   $div\n"
 8878     "done:"        %}
 8879   ins_encode(cdql_enc(div));
 8880   ins_pipe(pipe_slow);
 8881 %}
 8882 
 8883 // Long DIVMOD with Register, both quotient and mod results
 8884 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8885                              rFlagsReg cr)
 8886 %{
 8887   match(DivModL rax div);
 8888   effect(KILL cr);
 8889 
 8890   ins_cost(30*100+10*100); // XXX
 8891   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8892             "cmpq    rax, rdx\n\t"
 8893             "jne,s   normal\n\t"
 8894             "xorl    rdx, rdx\n\t"
 8895             "cmpq    $div, -1\n\t"
 8896             "je,s    done\n"
 8897     "normal: cdqq\n\t"
 8898             "idivq   $div\n"
 8899     "done:"        %}
 8900   ins_encode(cdqq_enc(div));
 8901   ins_pipe(pipe_slow);
 8902 %}
 8903 
 8904 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8905 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8906                               no_rax_rdx_RegI div, rFlagsReg cr)
 8907 %{
 8908   match(UDivModI rax div);
 8909   effect(TEMP tmp, KILL cr);
 8910 
 8911   ins_cost(300);
 8912   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8913             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8914           %}
 8915   ins_encode %{
 8916     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8917   %}
 8918   ins_pipe(pipe_slow);
 8919 %}
 8920 
 8921 // Unsigned long DIVMOD with Register, both quotient and mod results
 8922 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8923                               no_rax_rdx_RegL div, rFlagsReg cr)
 8924 %{
 8925   match(UDivModL rax div);
 8926   effect(TEMP tmp, KILL cr);
 8927 
 8928   ins_cost(300);
 8929   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8930             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8931           %}
 8932   ins_encode %{
 8933     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8934   %}
 8935   ins_pipe(pipe_slow);
 8936 %}
 8937 
 8938 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8939                    rFlagsReg cr)
 8940 %{
 8941   match(Set rdx (ModI rax div));
 8942   effect(KILL rax, KILL cr);
 8943 
 8944   ins_cost(300); // XXX
 8945   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8946             "jne,s   normal\n\t"
 8947             "xorl    rdx, rdx\n\t"
 8948             "cmpl    $div, -1\n\t"
 8949             "je,s    done\n"
 8950     "normal: cdql\n\t"
 8951             "idivl   $div\n"
 8952     "done:"        %}
 8953   ins_encode(cdql_enc(div));
 8954   ins_pipe(ialu_reg_reg_alu0);
 8955 %}
 8956 
 8957 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8958                    rFlagsReg cr)
 8959 %{
 8960   match(Set rdx (ModL rax div));
 8961   effect(KILL rax, KILL cr);
 8962 
 8963   ins_cost(300); // XXX
 8964   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8965             "cmpq    rax, rdx\n\t"
 8966             "jne,s   normal\n\t"
 8967             "xorl    rdx, rdx\n\t"
 8968             "cmpq    $div, -1\n\t"
 8969             "je,s    done\n"
 8970     "normal: cdqq\n\t"
 8971             "idivq   $div\n"
 8972     "done:"        %}
 8973   ins_encode(cdqq_enc(div));
 8974   ins_pipe(ialu_reg_reg_alu0);
 8975 %}
 8976 
 8977 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8978 %{
 8979   match(Set rdx (UModI rax div));
 8980   effect(KILL rax, KILL cr);
 8981 
 8982   ins_cost(300);
 8983   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8984   ins_encode %{
 8985     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8986   %}
 8987   ins_pipe(ialu_reg_reg_alu0);
 8988 %}
 8989 
 8990 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8991 %{
 8992   match(Set rdx (UModL rax div));
 8993   effect(KILL rax, KILL cr);
 8994 
 8995   ins_cost(300);
 8996   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8997   ins_encode %{
 8998     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8999   %}
 9000   ins_pipe(ialu_reg_reg_alu0);
 9001 %}
 9002 
 9003 // Integer Shift Instructions
 9004 // Shift Left by 8-bit immediate
 9005 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9006 %{
 9007   match(Set dst (LShiftI dst shift));
 9008   effect(KILL cr);
 9009 
 9010   format %{ "sall    $dst, $shift" %}
 9011   ins_encode %{
 9012     __ sall($dst$$Register, $shift$$constant);
 9013   %}
 9014   ins_pipe(ialu_reg);
 9015 %}
 9016 
 9017 // Shift Left by 8-bit immediate
 9018 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9019 %{
 9020   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9021   effect(KILL cr);
 9022 
 9023   format %{ "sall    $dst, $shift" %}
 9024   ins_encode %{
 9025     __ sall($dst$$Address, $shift$$constant);
 9026   %}
 9027   ins_pipe(ialu_mem_imm);
 9028 %}
 9029 
 9030 // Shift Left by variable
 9031 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9032 %{
 9033   predicate(!VM_Version::supports_bmi2());
 9034   match(Set dst (LShiftI dst shift));
 9035   effect(KILL cr);
 9036 
 9037   format %{ "sall    $dst, $shift" %}
 9038   ins_encode %{
 9039     __ sall($dst$$Register);
 9040   %}
 9041   ins_pipe(ialu_reg_reg);
 9042 %}
 9043 
 9044 // Shift Left by variable
 9045 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9046 %{
 9047   predicate(!VM_Version::supports_bmi2());
 9048   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9049   effect(KILL cr);
 9050 
 9051   format %{ "sall    $dst, $shift" %}
 9052   ins_encode %{
 9053     __ sall($dst$$Address);
 9054   %}
 9055   ins_pipe(ialu_mem_reg);
 9056 %}
 9057 
 9058 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9059 %{
 9060   predicate(VM_Version::supports_bmi2());
 9061   match(Set dst (LShiftI src shift));
 9062 
 9063   format %{ "shlxl   $dst, $src, $shift" %}
 9064   ins_encode %{
 9065     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9066   %}
 9067   ins_pipe(ialu_reg_reg);
 9068 %}
 9069 
 9070 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9071 %{
 9072   predicate(VM_Version::supports_bmi2());
 9073   match(Set dst (LShiftI (LoadI src) shift));
 9074   ins_cost(175);
 9075   format %{ "shlxl   $dst, $src, $shift" %}
 9076   ins_encode %{
 9077     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9078   %}
 9079   ins_pipe(ialu_reg_mem);
 9080 %}
 9081 
 9082 // Arithmetic Shift Right by 8-bit immediate
 9083 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9084 %{
 9085   match(Set dst (RShiftI dst shift));
 9086   effect(KILL cr);
 9087 
 9088   format %{ "sarl    $dst, $shift" %}
 9089   ins_encode %{
 9090     __ sarl($dst$$Register, $shift$$constant);
 9091   %}
 9092   ins_pipe(ialu_mem_imm);
 9093 %}
 9094 
 9095 // Arithmetic Shift Right by 8-bit immediate
 9096 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9097 %{
 9098   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9099   effect(KILL cr);
 9100 
 9101   format %{ "sarl    $dst, $shift" %}
 9102   ins_encode %{
 9103     __ sarl($dst$$Address, $shift$$constant);
 9104   %}
 9105   ins_pipe(ialu_mem_imm);
 9106 %}
 9107 
 9108 // Arithmetic Shift Right by variable
 9109 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9110 %{
 9111   predicate(!VM_Version::supports_bmi2());
 9112   match(Set dst (RShiftI dst shift));
 9113   effect(KILL cr);
 9114   format %{ "sarl    $dst, $shift" %}
 9115   ins_encode %{
 9116     __ sarl($dst$$Register);
 9117   %}
 9118   ins_pipe(ialu_reg_reg);
 9119 %}
 9120 
 9121 // Arithmetic Shift Right by variable
 9122 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9123 %{
 9124   predicate(!VM_Version::supports_bmi2());
 9125   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9126   effect(KILL cr);
 9127 
 9128   format %{ "sarl    $dst, $shift" %}
 9129   ins_encode %{
 9130     __ sarl($dst$$Address);
 9131   %}
 9132   ins_pipe(ialu_mem_reg);
 9133 %}
 9134 
 9135 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9136 %{
 9137   predicate(VM_Version::supports_bmi2());
 9138   match(Set dst (RShiftI src shift));
 9139 
 9140   format %{ "sarxl   $dst, $src, $shift" %}
 9141   ins_encode %{
 9142     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9143   %}
 9144   ins_pipe(ialu_reg_reg);
 9145 %}
 9146 
 9147 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9148 %{
 9149   predicate(VM_Version::supports_bmi2());
 9150   match(Set dst (RShiftI (LoadI src) shift));
 9151   ins_cost(175);
 9152   format %{ "sarxl   $dst, $src, $shift" %}
 9153   ins_encode %{
 9154     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9155   %}
 9156   ins_pipe(ialu_reg_mem);
 9157 %}
 9158 
 9159 // Logical Shift Right by 8-bit immediate
 9160 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9161 %{
 9162   match(Set dst (URShiftI dst shift));
 9163   effect(KILL cr);
 9164 
 9165   format %{ "shrl    $dst, $shift" %}
 9166   ins_encode %{
 9167     __ shrl($dst$$Register, $shift$$constant);
 9168   %}
 9169   ins_pipe(ialu_reg);
 9170 %}
 9171 
 9172 // Logical Shift Right by 8-bit immediate
 9173 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9174 %{
 9175   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9176   effect(KILL cr);
 9177 
 9178   format %{ "shrl    $dst, $shift" %}
 9179   ins_encode %{
 9180     __ shrl($dst$$Address, $shift$$constant);
 9181   %}
 9182   ins_pipe(ialu_mem_imm);
 9183 %}
 9184 
 9185 // Logical Shift Right by variable
 9186 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9187 %{
 9188   predicate(!VM_Version::supports_bmi2());
 9189   match(Set dst (URShiftI dst shift));
 9190   effect(KILL cr);
 9191 
 9192   format %{ "shrl    $dst, $shift" %}
 9193   ins_encode %{
 9194     __ shrl($dst$$Register);
 9195   %}
 9196   ins_pipe(ialu_reg_reg);
 9197 %}
 9198 
 9199 // Logical Shift Right by variable
 9200 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9201 %{
 9202   predicate(!VM_Version::supports_bmi2());
 9203   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9204   effect(KILL cr);
 9205 
 9206   format %{ "shrl    $dst, $shift" %}
 9207   ins_encode %{
 9208     __ shrl($dst$$Address);
 9209   %}
 9210   ins_pipe(ialu_mem_reg);
 9211 %}
 9212 
 9213 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9214 %{
 9215   predicate(VM_Version::supports_bmi2());
 9216   match(Set dst (URShiftI src shift));
 9217 
 9218   format %{ "shrxl   $dst, $src, $shift" %}
 9219   ins_encode %{
 9220     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9221   %}
 9222   ins_pipe(ialu_reg_reg);
 9223 %}
 9224 
 9225 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9226 %{
 9227   predicate(VM_Version::supports_bmi2());
 9228   match(Set dst (URShiftI (LoadI src) shift));
 9229   ins_cost(175);
 9230   format %{ "shrxl   $dst, $src, $shift" %}
 9231   ins_encode %{
 9232     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9233   %}
 9234   ins_pipe(ialu_reg_mem);
 9235 %}
 9236 
 9237 // Long Shift Instructions
 9238 // Shift Left by 8-bit immediate
 9239 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9240 %{
 9241   match(Set dst (LShiftL dst shift));
 9242   effect(KILL cr);
 9243 
 9244   format %{ "salq    $dst, $shift" %}
 9245   ins_encode %{
 9246     __ salq($dst$$Register, $shift$$constant);
 9247   %}
 9248   ins_pipe(ialu_reg);
 9249 %}
 9250 
 9251 // Shift Left by 8-bit immediate
 9252 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9253 %{
 9254   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9255   effect(KILL cr);
 9256 
 9257   format %{ "salq    $dst, $shift" %}
 9258   ins_encode %{
 9259     __ salq($dst$$Address, $shift$$constant);
 9260   %}
 9261   ins_pipe(ialu_mem_imm);
 9262 %}
 9263 
 9264 // Shift Left by variable
 9265 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9266 %{
 9267   predicate(!VM_Version::supports_bmi2());
 9268   match(Set dst (LShiftL dst shift));
 9269   effect(KILL cr);
 9270 
 9271   format %{ "salq    $dst, $shift" %}
 9272   ins_encode %{
 9273     __ salq($dst$$Register);
 9274   %}
 9275   ins_pipe(ialu_reg_reg);
 9276 %}
 9277 
 9278 // Shift Left by variable
 9279 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9280 %{
 9281   predicate(!VM_Version::supports_bmi2());
 9282   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9283   effect(KILL cr);
 9284 
 9285   format %{ "salq    $dst, $shift" %}
 9286   ins_encode %{
 9287     __ salq($dst$$Address);
 9288   %}
 9289   ins_pipe(ialu_mem_reg);
 9290 %}
 9291 
 9292 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9293 %{
 9294   predicate(VM_Version::supports_bmi2());
 9295   match(Set dst (LShiftL src shift));
 9296 
 9297   format %{ "shlxq   $dst, $src, $shift" %}
 9298   ins_encode %{
 9299     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9300   %}
 9301   ins_pipe(ialu_reg_reg);
 9302 %}
 9303 
 9304 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9305 %{
 9306   predicate(VM_Version::supports_bmi2());
 9307   match(Set dst (LShiftL (LoadL src) shift));
 9308   ins_cost(175);
 9309   format %{ "shlxq   $dst, $src, $shift" %}
 9310   ins_encode %{
 9311     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9312   %}
 9313   ins_pipe(ialu_reg_mem);
 9314 %}
 9315 
 9316 // Arithmetic Shift Right by 8-bit immediate
 9317 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9318 %{
 9319   match(Set dst (RShiftL dst shift));
 9320   effect(KILL cr);
 9321 
 9322   format %{ "sarq    $dst, $shift" %}
 9323   ins_encode %{
 9324     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9325   %}
 9326   ins_pipe(ialu_mem_imm);
 9327 %}
 9328 
 9329 // Arithmetic Shift Right by 8-bit immediate
 9330 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9331 %{
 9332   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9333   effect(KILL cr);
 9334 
 9335   format %{ "sarq    $dst, $shift" %}
 9336   ins_encode %{
 9337     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9338   %}
 9339   ins_pipe(ialu_mem_imm);
 9340 %}
 9341 
 9342 // Arithmetic Shift Right by variable
 9343 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9344 %{
 9345   predicate(!VM_Version::supports_bmi2());
 9346   match(Set dst (RShiftL dst shift));
 9347   effect(KILL cr);
 9348 
 9349   format %{ "sarq    $dst, $shift" %}
 9350   ins_encode %{
 9351     __ sarq($dst$$Register);
 9352   %}
 9353   ins_pipe(ialu_reg_reg);
 9354 %}
 9355 
 9356 // Arithmetic Shift Right by variable
 9357 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9358 %{
 9359   predicate(!VM_Version::supports_bmi2());
 9360   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9361   effect(KILL cr);
 9362 
 9363   format %{ "sarq    $dst, $shift" %}
 9364   ins_encode %{
 9365     __ sarq($dst$$Address);
 9366   %}
 9367   ins_pipe(ialu_mem_reg);
 9368 %}
 9369 
 9370 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9371 %{
 9372   predicate(VM_Version::supports_bmi2());
 9373   match(Set dst (RShiftL src shift));
 9374 
 9375   format %{ "sarxq   $dst, $src, $shift" %}
 9376   ins_encode %{
 9377     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9378   %}
 9379   ins_pipe(ialu_reg_reg);
 9380 %}
 9381 
 9382 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9383 %{
 9384   predicate(VM_Version::supports_bmi2());
 9385   match(Set dst (RShiftL (LoadL src) shift));
 9386   ins_cost(175);
 9387   format %{ "sarxq   $dst, $src, $shift" %}
 9388   ins_encode %{
 9389     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9390   %}
 9391   ins_pipe(ialu_reg_mem);
 9392 %}
 9393 
 9394 // Logical Shift Right by 8-bit immediate
 9395 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9396 %{
 9397   match(Set dst (URShiftL dst shift));
 9398   effect(KILL cr);
 9399 
 9400   format %{ "shrq    $dst, $shift" %}
 9401   ins_encode %{
 9402     __ shrq($dst$$Register, $shift$$constant);
 9403   %}
 9404   ins_pipe(ialu_reg);
 9405 %}
 9406 
 9407 // Logical Shift Right by 8-bit immediate
 9408 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9409 %{
 9410   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9411   effect(KILL cr);
 9412 
 9413   format %{ "shrq    $dst, $shift" %}
 9414   ins_encode %{
 9415     __ shrq($dst$$Address, $shift$$constant);
 9416   %}
 9417   ins_pipe(ialu_mem_imm);
 9418 %}
 9419 
 9420 // Logical Shift Right by variable
 9421 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9422 %{
 9423   predicate(!VM_Version::supports_bmi2());
 9424   match(Set dst (URShiftL dst shift));
 9425   effect(KILL cr);
 9426 
 9427   format %{ "shrq    $dst, $shift" %}
 9428   ins_encode %{
 9429     __ shrq($dst$$Register);
 9430   %}
 9431   ins_pipe(ialu_reg_reg);
 9432 %}
 9433 
 9434 // Logical Shift Right by variable
 9435 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9436 %{
 9437   predicate(!VM_Version::supports_bmi2());
 9438   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9439   effect(KILL cr);
 9440 
 9441   format %{ "shrq    $dst, $shift" %}
 9442   ins_encode %{
 9443     __ shrq($dst$$Address);
 9444   %}
 9445   ins_pipe(ialu_mem_reg);
 9446 %}
 9447 
 9448 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9449 %{
 9450   predicate(VM_Version::supports_bmi2());
 9451   match(Set dst (URShiftL src shift));
 9452 
 9453   format %{ "shrxq   $dst, $src, $shift" %}
 9454   ins_encode %{
 9455     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9456   %}
 9457   ins_pipe(ialu_reg_reg);
 9458 %}
 9459 
 9460 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9461 %{
 9462   predicate(VM_Version::supports_bmi2());
 9463   match(Set dst (URShiftL (LoadL src) shift));
 9464   ins_cost(175);
 9465   format %{ "shrxq   $dst, $src, $shift" %}
 9466   ins_encode %{
 9467     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9468   %}
 9469   ins_pipe(ialu_reg_mem);
 9470 %}
 9471 
 9472 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9473 // This idiom is used by the compiler for the i2b bytecode.
 9474 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9475 %{
 9476   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9477 
 9478   format %{ "movsbl  $dst, $src\t# i2b" %}
 9479   ins_encode %{
 9480     __ movsbl($dst$$Register, $src$$Register);
 9481   %}
 9482   ins_pipe(ialu_reg_reg);
 9483 %}
 9484 
 9485 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9486 // This idiom is used by the compiler the i2s bytecode.
 9487 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9488 %{
 9489   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9490 
 9491   format %{ "movswl  $dst, $src\t# i2s" %}
 9492   ins_encode %{
 9493     __ movswl($dst$$Register, $src$$Register);
 9494   %}
 9495   ins_pipe(ialu_reg_reg);
 9496 %}
 9497 
 9498 // ROL/ROR instructions
 9499 
 9500 // Rotate left by constant.
 9501 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9502 %{
 9503   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9504   match(Set dst (RotateLeft dst shift));
 9505   effect(KILL cr);
 9506   format %{ "roll    $dst, $shift" %}
 9507   ins_encode %{
 9508     __ roll($dst$$Register, $shift$$constant);
 9509   %}
 9510   ins_pipe(ialu_reg);
 9511 %}
 9512 
 9513 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9514 %{
 9515   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9516   match(Set dst (RotateLeft src shift));
 9517   format %{ "rolxl   $dst, $src, $shift" %}
 9518   ins_encode %{
 9519     int shift = 32 - ($shift$$constant & 31);
 9520     __ rorxl($dst$$Register, $src$$Register, shift);
 9521   %}
 9522   ins_pipe(ialu_reg_reg);
 9523 %}
 9524 
 9525 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9526 %{
 9527   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9528   match(Set dst (RotateLeft (LoadI src) shift));
 9529   ins_cost(175);
 9530   format %{ "rolxl   $dst, $src, $shift" %}
 9531   ins_encode %{
 9532     int shift = 32 - ($shift$$constant & 31);
 9533     __ rorxl($dst$$Register, $src$$Address, shift);
 9534   %}
 9535   ins_pipe(ialu_reg_mem);
 9536 %}
 9537 
 9538 // Rotate Left by variable
 9539 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9540 %{
 9541   predicate(n->bottom_type()->basic_type() == T_INT);
 9542   match(Set dst (RotateLeft dst shift));
 9543   effect(KILL cr);
 9544   format %{ "roll    $dst, $shift" %}
 9545   ins_encode %{
 9546     __ roll($dst$$Register);
 9547   %}
 9548   ins_pipe(ialu_reg_reg);
 9549 %}
 9550 
 9551 // Rotate Right by constant.
 9552 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9553 %{
 9554   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9555   match(Set dst (RotateRight dst shift));
 9556   effect(KILL cr);
 9557   format %{ "rorl    $dst, $shift" %}
 9558   ins_encode %{
 9559     __ rorl($dst$$Register, $shift$$constant);
 9560   %}
 9561   ins_pipe(ialu_reg);
 9562 %}
 9563 
 9564 // Rotate Right by constant.
 9565 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9566 %{
 9567   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9568   match(Set dst (RotateRight src shift));
 9569   format %{ "rorxl   $dst, $src, $shift" %}
 9570   ins_encode %{
 9571     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9572   %}
 9573   ins_pipe(ialu_reg_reg);
 9574 %}
 9575 
 9576 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9577 %{
 9578   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9579   match(Set dst (RotateRight (LoadI src) shift));
 9580   ins_cost(175);
 9581   format %{ "rorxl   $dst, $src, $shift" %}
 9582   ins_encode %{
 9583     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9584   %}
 9585   ins_pipe(ialu_reg_mem);
 9586 %}
 9587 
 9588 // Rotate Right by variable
 9589 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9590 %{
 9591   predicate(n->bottom_type()->basic_type() == T_INT);
 9592   match(Set dst (RotateRight dst shift));
 9593   effect(KILL cr);
 9594   format %{ "rorl    $dst, $shift" %}
 9595   ins_encode %{
 9596     __ rorl($dst$$Register);
 9597   %}
 9598   ins_pipe(ialu_reg_reg);
 9599 %}
 9600 
 9601 // Rotate Left by constant.
 9602 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9603 %{
 9604   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9605   match(Set dst (RotateLeft dst shift));
 9606   effect(KILL cr);
 9607   format %{ "rolq    $dst, $shift" %}
 9608   ins_encode %{
 9609     __ rolq($dst$$Register, $shift$$constant);
 9610   %}
 9611   ins_pipe(ialu_reg);
 9612 %}
 9613 
 9614 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9615 %{
 9616   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9617   match(Set dst (RotateLeft src shift));
 9618   format %{ "rolxq   $dst, $src, $shift" %}
 9619   ins_encode %{
 9620     int shift = 64 - ($shift$$constant & 63);
 9621     __ rorxq($dst$$Register, $src$$Register, shift);
 9622   %}
 9623   ins_pipe(ialu_reg_reg);
 9624 %}
 9625 
 9626 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9627 %{
 9628   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9629   match(Set dst (RotateLeft (LoadL src) shift));
 9630   ins_cost(175);
 9631   format %{ "rolxq   $dst, $src, $shift" %}
 9632   ins_encode %{
 9633     int shift = 64 - ($shift$$constant & 63);
 9634     __ rorxq($dst$$Register, $src$$Address, shift);
 9635   %}
 9636   ins_pipe(ialu_reg_mem);
 9637 %}
 9638 
 9639 // Rotate Left by variable
 9640 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9641 %{
 9642   predicate(n->bottom_type()->basic_type() == T_LONG);
 9643   match(Set dst (RotateLeft dst shift));
 9644   effect(KILL cr);
 9645   format %{ "rolq    $dst, $shift" %}
 9646   ins_encode %{
 9647     __ rolq($dst$$Register);
 9648   %}
 9649   ins_pipe(ialu_reg_reg);
 9650 %}
 9651 
 9652 // Rotate Right by constant.
 9653 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9654 %{
 9655   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9656   match(Set dst (RotateRight dst shift));
 9657   effect(KILL cr);
 9658   format %{ "rorq    $dst, $shift" %}
 9659   ins_encode %{
 9660     __ rorq($dst$$Register, $shift$$constant);
 9661   %}
 9662   ins_pipe(ialu_reg);
 9663 %}
 9664 
 9665 // Rotate Right by constant
 9666 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9667 %{
 9668   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9669   match(Set dst (RotateRight src shift));
 9670   format %{ "rorxq   $dst, $src, $shift" %}
 9671   ins_encode %{
 9672     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9673   %}
 9674   ins_pipe(ialu_reg_reg);
 9675 %}
 9676 
 9677 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9678 %{
 9679   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9680   match(Set dst (RotateRight (LoadL src) shift));
 9681   ins_cost(175);
 9682   format %{ "rorxq   $dst, $src, $shift" %}
 9683   ins_encode %{
 9684     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9685   %}
 9686   ins_pipe(ialu_reg_mem);
 9687 %}
 9688 
 9689 // Rotate Right by variable
 9690 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9691 %{
 9692   predicate(n->bottom_type()->basic_type() == T_LONG);
 9693   match(Set dst (RotateRight dst shift));
 9694   effect(KILL cr);
 9695   format %{ "rorq    $dst, $shift" %}
 9696   ins_encode %{
 9697     __ rorq($dst$$Register);
 9698   %}
 9699   ins_pipe(ialu_reg_reg);
 9700 %}
 9701 
 9702 //----------------------------- CompressBits/ExpandBits ------------------------
 9703 
 9704 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9705   predicate(n->bottom_type()->isa_long());
 9706   match(Set dst (CompressBits src mask));
 9707   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9708   ins_encode %{
 9709     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9710   %}
 9711   ins_pipe( pipe_slow );
 9712 %}
 9713 
 9714 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9715   predicate(n->bottom_type()->isa_long());
 9716   match(Set dst (ExpandBits src mask));
 9717   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9718   ins_encode %{
 9719     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9720   %}
 9721   ins_pipe( pipe_slow );
 9722 %}
 9723 
 9724 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9725   predicate(n->bottom_type()->isa_long());
 9726   match(Set dst (CompressBits src (LoadL mask)));
 9727   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9728   ins_encode %{
 9729     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 9730   %}
 9731   ins_pipe( pipe_slow );
 9732 %}
 9733 
 9734 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9735   predicate(n->bottom_type()->isa_long());
 9736   match(Set dst (ExpandBits src (LoadL mask)));
 9737   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9738   ins_encode %{
 9739     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 9740   %}
 9741   ins_pipe( pipe_slow );
 9742 %}
 9743 
 9744 
 9745 // Logical Instructions
 9746 
 9747 // Integer Logical Instructions
 9748 
 9749 // And Instructions
 9750 // And Register with Register
 9751 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9752 %{
 9753   match(Set dst (AndI dst src));
 9754   effect(KILL cr);
 9755 
 9756   format %{ "andl    $dst, $src\t# int" %}
 9757   ins_encode %{
 9758     __ andl($dst$$Register, $src$$Register);
 9759   %}
 9760   ins_pipe(ialu_reg_reg);
 9761 %}
 9762 
 9763 // And Register with Immediate 255
 9764 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 9765 %{
 9766   match(Set dst (AndI src mask));
 9767 
 9768   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 9769   ins_encode %{
 9770     __ movzbl($dst$$Register, $src$$Register);
 9771   %}
 9772   ins_pipe(ialu_reg);
 9773 %}
 9774 
 9775 // And Register with Immediate 255 and promote to long
 9776 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9777 %{
 9778   match(Set dst (ConvI2L (AndI src mask)));
 9779 
 9780   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9781   ins_encode %{
 9782     __ movzbl($dst$$Register, $src$$Register);
 9783   %}
 9784   ins_pipe(ialu_reg);
 9785 %}
 9786 
 9787 // And Register with Immediate 65535
 9788 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 9789 %{
 9790   match(Set dst (AndI src mask));
 9791 
 9792   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 9793   ins_encode %{
 9794     __ movzwl($dst$$Register, $src$$Register);
 9795   %}
 9796   ins_pipe(ialu_reg);
 9797 %}
 9798 
 9799 // And Register with Immediate 65535 and promote to long
 9800 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9801 %{
 9802   match(Set dst (ConvI2L (AndI src mask)));
 9803 
 9804   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9805   ins_encode %{
 9806     __ movzwl($dst$$Register, $src$$Register);
 9807   %}
 9808   ins_pipe(ialu_reg);
 9809 %}
 9810 
 9811 // Can skip int2long conversions after AND with small bitmask
 9812 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9813 %{
 9814   predicate(VM_Version::supports_bmi2());
 9815   ins_cost(125);
 9816   effect(TEMP tmp, KILL cr);
 9817   match(Set dst (ConvI2L (AndI src mask)));
 9818   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9819   ins_encode %{
 9820     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9821     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9822   %}
 9823   ins_pipe(ialu_reg_reg);
 9824 %}
 9825 
 9826 // And Register with Immediate
 9827 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9828 %{
 9829   match(Set dst (AndI dst src));
 9830   effect(KILL cr);
 9831 
 9832   format %{ "andl    $dst, $src\t# int" %}
 9833   ins_encode %{
 9834     __ andl($dst$$Register, $src$$constant);
 9835   %}
 9836   ins_pipe(ialu_reg);
 9837 %}
 9838 
 9839 // And Register with Memory
 9840 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9841 %{
 9842   match(Set dst (AndI dst (LoadI src)));
 9843   effect(KILL cr);
 9844 
 9845   ins_cost(150);
 9846   format %{ "andl    $dst, $src\t# int" %}
 9847   ins_encode %{
 9848     __ andl($dst$$Register, $src$$Address);
 9849   %}
 9850   ins_pipe(ialu_reg_mem);
 9851 %}
 9852 
 9853 // And Memory with Register
 9854 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9855 %{
 9856   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9857   effect(KILL cr);
 9858 
 9859   ins_cost(150);
 9860   format %{ "andb    $dst, $src\t# byte" %}
 9861   ins_encode %{
 9862     __ andb($dst$$Address, $src$$Register);
 9863   %}
 9864   ins_pipe(ialu_mem_reg);
 9865 %}
 9866 
 9867 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9868 %{
 9869   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9870   effect(KILL cr);
 9871 
 9872   ins_cost(150);
 9873   format %{ "andl    $dst, $src\t# int" %}
 9874   ins_encode %{
 9875     __ andl($dst$$Address, $src$$Register);
 9876   %}
 9877   ins_pipe(ialu_mem_reg);
 9878 %}
 9879 
 9880 // And Memory with Immediate
 9881 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9882 %{
 9883   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9884   effect(KILL cr);
 9885 
 9886   ins_cost(125);
 9887   format %{ "andl    $dst, $src\t# int" %}
 9888   ins_encode %{
 9889     __ andl($dst$$Address, $src$$constant);
 9890   %}
 9891   ins_pipe(ialu_mem_imm);
 9892 %}
 9893 
 9894 // BMI1 instructions
 9895 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9896   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9897   predicate(UseBMI1Instructions);
 9898   effect(KILL cr);
 9899 
 9900   ins_cost(125);
 9901   format %{ "andnl  $dst, $src1, $src2" %}
 9902 
 9903   ins_encode %{
 9904     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9905   %}
 9906   ins_pipe(ialu_reg_mem);
 9907 %}
 9908 
 9909 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9910   match(Set dst (AndI (XorI src1 minus_1) src2));
 9911   predicate(UseBMI1Instructions);
 9912   effect(KILL cr);
 9913 
 9914   format %{ "andnl  $dst, $src1, $src2" %}
 9915 
 9916   ins_encode %{
 9917     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9918   %}
 9919   ins_pipe(ialu_reg);
 9920 %}
 9921 
 9922 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9923   match(Set dst (AndI (SubI imm_zero src) src));
 9924   predicate(UseBMI1Instructions);
 9925   effect(KILL cr);
 9926 
 9927   format %{ "blsil  $dst, $src" %}
 9928 
 9929   ins_encode %{
 9930     __ blsil($dst$$Register, $src$$Register);
 9931   %}
 9932   ins_pipe(ialu_reg);
 9933 %}
 9934 
 9935 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9936   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9937   predicate(UseBMI1Instructions);
 9938   effect(KILL cr);
 9939 
 9940   ins_cost(125);
 9941   format %{ "blsil  $dst, $src" %}
 9942 
 9943   ins_encode %{
 9944     __ blsil($dst$$Register, $src$$Address);
 9945   %}
 9946   ins_pipe(ialu_reg_mem);
 9947 %}
 9948 
 9949 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9950 %{
 9951   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9952   predicate(UseBMI1Instructions);
 9953   effect(KILL cr);
 9954 
 9955   ins_cost(125);
 9956   format %{ "blsmskl $dst, $src" %}
 9957 
 9958   ins_encode %{
 9959     __ blsmskl($dst$$Register, $src$$Address);
 9960   %}
 9961   ins_pipe(ialu_reg_mem);
 9962 %}
 9963 
 9964 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9965 %{
 9966   match(Set dst (XorI (AddI src minus_1) src));
 9967   predicate(UseBMI1Instructions);
 9968   effect(KILL cr);
 9969 
 9970   format %{ "blsmskl $dst, $src" %}
 9971 
 9972   ins_encode %{
 9973     __ blsmskl($dst$$Register, $src$$Register);
 9974   %}
 9975 
 9976   ins_pipe(ialu_reg);
 9977 %}
 9978 
 9979 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9980 %{
 9981   match(Set dst (AndI (AddI src minus_1) src) );
 9982   predicate(UseBMI1Instructions);
 9983   effect(KILL cr);
 9984 
 9985   format %{ "blsrl  $dst, $src" %}
 9986 
 9987   ins_encode %{
 9988     __ blsrl($dst$$Register, $src$$Register);
 9989   %}
 9990 
 9991   ins_pipe(ialu_reg_mem);
 9992 %}
 9993 
 9994 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9995 %{
 9996   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9997   predicate(UseBMI1Instructions);
 9998   effect(KILL cr);
 9999 
10000   ins_cost(125);
10001   format %{ "blsrl  $dst, $src" %}
10002 
10003   ins_encode %{
10004     __ blsrl($dst$$Register, $src$$Address);
10005   %}
10006 
10007   ins_pipe(ialu_reg);
10008 %}
10009 
10010 // Or Instructions
10011 // Or Register with Register
10012 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10013 %{
10014   match(Set dst (OrI dst src));
10015   effect(KILL cr);
10016 
10017   format %{ "orl     $dst, $src\t# int" %}
10018   ins_encode %{
10019     __ orl($dst$$Register, $src$$Register);
10020   %}
10021   ins_pipe(ialu_reg_reg);
10022 %}
10023 
10024 // Or Register with Immediate
10025 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10026 %{
10027   match(Set dst (OrI dst src));
10028   effect(KILL cr);
10029 
10030   format %{ "orl     $dst, $src\t# int" %}
10031   ins_encode %{
10032     __ orl($dst$$Register, $src$$constant);
10033   %}
10034   ins_pipe(ialu_reg);
10035 %}
10036 
10037 // Or Register with Memory
10038 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10039 %{
10040   match(Set dst (OrI dst (LoadI src)));
10041   effect(KILL cr);
10042 
10043   ins_cost(150);
10044   format %{ "orl     $dst, $src\t# int" %}
10045   ins_encode %{
10046     __ orl($dst$$Register, $src$$Address);
10047   %}
10048   ins_pipe(ialu_reg_mem);
10049 %}
10050 
10051 // Or Memory with Register
10052 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10053 %{
10054   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10055   effect(KILL cr);
10056 
10057   ins_cost(150);
10058   format %{ "orb    $dst, $src\t# byte" %}
10059   ins_encode %{
10060     __ orb($dst$$Address, $src$$Register);
10061   %}
10062   ins_pipe(ialu_mem_reg);
10063 %}
10064 
10065 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10066 %{
10067   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10068   effect(KILL cr);
10069 
10070   ins_cost(150);
10071   format %{ "orl     $dst, $src\t# int" %}
10072   ins_encode %{
10073     __ orl($dst$$Address, $src$$Register);
10074   %}
10075   ins_pipe(ialu_mem_reg);
10076 %}
10077 
10078 // Or Memory with Immediate
10079 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10080 %{
10081   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10082   effect(KILL cr);
10083 
10084   ins_cost(125);
10085   format %{ "orl     $dst, $src\t# int" %}
10086   ins_encode %{
10087     __ orl($dst$$Address, $src$$constant);
10088   %}
10089   ins_pipe(ialu_mem_imm);
10090 %}
10091 
10092 // Xor Instructions
10093 // Xor Register with Register
10094 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10095 %{
10096   match(Set dst (XorI dst src));
10097   effect(KILL cr);
10098 
10099   format %{ "xorl    $dst, $src\t# int" %}
10100   ins_encode %{
10101     __ xorl($dst$$Register, $src$$Register);
10102   %}
10103   ins_pipe(ialu_reg_reg);
10104 %}
10105 
10106 // Xor Register with Immediate -1
10107 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10108   match(Set dst (XorI dst imm));
10109 
10110   format %{ "not    $dst" %}
10111   ins_encode %{
10112      __ notl($dst$$Register);
10113   %}
10114   ins_pipe(ialu_reg);
10115 %}
10116 
10117 // Xor Register with Immediate
10118 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10119 %{
10120   match(Set dst (XorI dst src));
10121   effect(KILL cr);
10122 
10123   format %{ "xorl    $dst, $src\t# int" %}
10124   ins_encode %{
10125     __ xorl($dst$$Register, $src$$constant);
10126   %}
10127   ins_pipe(ialu_reg);
10128 %}
10129 
10130 // Xor Register with Memory
10131 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10132 %{
10133   match(Set dst (XorI dst (LoadI src)));
10134   effect(KILL cr);
10135 
10136   ins_cost(150);
10137   format %{ "xorl    $dst, $src\t# int" %}
10138   ins_encode %{
10139     __ xorl($dst$$Register, $src$$Address);
10140   %}
10141   ins_pipe(ialu_reg_mem);
10142 %}
10143 
10144 // Xor Memory with Register
10145 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10146 %{
10147   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10148   effect(KILL cr);
10149 
10150   ins_cost(150);
10151   format %{ "xorb    $dst, $src\t# byte" %}
10152   ins_encode %{
10153     __ xorb($dst$$Address, $src$$Register);
10154   %}
10155   ins_pipe(ialu_mem_reg);
10156 %}
10157 
10158 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10159 %{
10160   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10161   effect(KILL cr);
10162 
10163   ins_cost(150);
10164   format %{ "xorl    $dst, $src\t# int" %}
10165   ins_encode %{
10166     __ xorl($dst$$Address, $src$$Register);
10167   %}
10168   ins_pipe(ialu_mem_reg);
10169 %}
10170 
10171 // Xor Memory with Immediate
10172 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10173 %{
10174   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10175   effect(KILL cr);
10176 
10177   ins_cost(125);
10178   format %{ "xorl    $dst, $src\t# int" %}
10179   ins_encode %{
10180     __ xorl($dst$$Address, $src$$constant);
10181   %}
10182   ins_pipe(ialu_mem_imm);
10183 %}
10184 
10185 
10186 // Long Logical Instructions
10187 
10188 // And Instructions
10189 // And Register with Register
10190 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10191 %{
10192   match(Set dst (AndL dst src));
10193   effect(KILL cr);
10194 
10195   format %{ "andq    $dst, $src\t# long" %}
10196   ins_encode %{
10197     __ andq($dst$$Register, $src$$Register);
10198   %}
10199   ins_pipe(ialu_reg_reg);
10200 %}
10201 
10202 // And Register with Immediate 255
10203 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10204 %{
10205   match(Set dst (AndL src mask));
10206 
10207   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10208   ins_encode %{
10209     // movzbl zeroes out the upper 32-bit and does not need REX.W
10210     __ movzbl($dst$$Register, $src$$Register);
10211   %}
10212   ins_pipe(ialu_reg);
10213 %}
10214 
10215 // And Register with Immediate 65535
10216 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10217 %{
10218   match(Set dst (AndL src mask));
10219 
10220   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10221   ins_encode %{
10222     // movzwl zeroes out the upper 32-bit and does not need REX.W
10223     __ movzwl($dst$$Register, $src$$Register);
10224   %}
10225   ins_pipe(ialu_reg);
10226 %}
10227 
10228 // And Register with Immediate
10229 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10230 %{
10231   match(Set dst (AndL dst src));
10232   effect(KILL cr);
10233 
10234   format %{ "andq    $dst, $src\t# long" %}
10235   ins_encode %{
10236     __ andq($dst$$Register, $src$$constant);
10237   %}
10238   ins_pipe(ialu_reg);
10239 %}
10240 
10241 // And Register with Memory
10242 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10243 %{
10244   match(Set dst (AndL dst (LoadL src)));
10245   effect(KILL cr);
10246 
10247   ins_cost(150);
10248   format %{ "andq    $dst, $src\t# long" %}
10249   ins_encode %{
10250     __ andq($dst$$Register, $src$$Address);
10251   %}
10252   ins_pipe(ialu_reg_mem);
10253 %}
10254 
10255 // And Memory with Register
10256 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10257 %{
10258   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10259   effect(KILL cr);
10260 
10261   ins_cost(150);
10262   format %{ "andq    $dst, $src\t# long" %}
10263   ins_encode %{
10264     __ andq($dst$$Address, $src$$Register);
10265   %}
10266   ins_pipe(ialu_mem_reg);
10267 %}
10268 
10269 // And Memory with Immediate
10270 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10271 %{
10272   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10273   effect(KILL cr);
10274 
10275   ins_cost(125);
10276   format %{ "andq    $dst, $src\t# long" %}
10277   ins_encode %{
10278     __ andq($dst$$Address, $src$$constant);
10279   %}
10280   ins_pipe(ialu_mem_imm);
10281 %}
10282 
10283 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10284 %{
10285   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10286   // because AND/OR works well enough for 8/32-bit values.
10287   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10288 
10289   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10290   effect(KILL cr);
10291 
10292   ins_cost(125);
10293   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10294   ins_encode %{
10295     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10296   %}
10297   ins_pipe(ialu_mem_imm);
10298 %}
10299 
10300 // BMI1 instructions
10301 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10302   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10303   predicate(UseBMI1Instructions);
10304   effect(KILL cr);
10305 
10306   ins_cost(125);
10307   format %{ "andnq  $dst, $src1, $src2" %}
10308 
10309   ins_encode %{
10310     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10311   %}
10312   ins_pipe(ialu_reg_mem);
10313 %}
10314 
10315 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10316   match(Set dst (AndL (XorL src1 minus_1) src2));
10317   predicate(UseBMI1Instructions);
10318   effect(KILL cr);
10319 
10320   format %{ "andnq  $dst, $src1, $src2" %}
10321 
10322   ins_encode %{
10323   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10324   %}
10325   ins_pipe(ialu_reg_mem);
10326 %}
10327 
10328 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10329   match(Set dst (AndL (SubL imm_zero src) src));
10330   predicate(UseBMI1Instructions);
10331   effect(KILL cr);
10332 
10333   format %{ "blsiq  $dst, $src" %}
10334 
10335   ins_encode %{
10336     __ blsiq($dst$$Register, $src$$Register);
10337   %}
10338   ins_pipe(ialu_reg);
10339 %}
10340 
10341 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10342   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10343   predicate(UseBMI1Instructions);
10344   effect(KILL cr);
10345 
10346   ins_cost(125);
10347   format %{ "blsiq  $dst, $src" %}
10348 
10349   ins_encode %{
10350     __ blsiq($dst$$Register, $src$$Address);
10351   %}
10352   ins_pipe(ialu_reg_mem);
10353 %}
10354 
10355 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10356 %{
10357   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10358   predicate(UseBMI1Instructions);
10359   effect(KILL cr);
10360 
10361   ins_cost(125);
10362   format %{ "blsmskq $dst, $src" %}
10363 
10364   ins_encode %{
10365     __ blsmskq($dst$$Register, $src$$Address);
10366   %}
10367   ins_pipe(ialu_reg_mem);
10368 %}
10369 
10370 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10371 %{
10372   match(Set dst (XorL (AddL src minus_1) src));
10373   predicate(UseBMI1Instructions);
10374   effect(KILL cr);
10375 
10376   format %{ "blsmskq $dst, $src" %}
10377 
10378   ins_encode %{
10379     __ blsmskq($dst$$Register, $src$$Register);
10380   %}
10381 
10382   ins_pipe(ialu_reg);
10383 %}
10384 
10385 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10386 %{
10387   match(Set dst (AndL (AddL src minus_1) src) );
10388   predicate(UseBMI1Instructions);
10389   effect(KILL cr);
10390 
10391   format %{ "blsrq  $dst, $src" %}
10392 
10393   ins_encode %{
10394     __ blsrq($dst$$Register, $src$$Register);
10395   %}
10396 
10397   ins_pipe(ialu_reg);
10398 %}
10399 
10400 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10401 %{
10402   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10403   predicate(UseBMI1Instructions);
10404   effect(KILL cr);
10405 
10406   ins_cost(125);
10407   format %{ "blsrq  $dst, $src" %}
10408 
10409   ins_encode %{
10410     __ blsrq($dst$$Register, $src$$Address);
10411   %}
10412 
10413   ins_pipe(ialu_reg);
10414 %}
10415 
10416 // Or Instructions
10417 // Or Register with Register
10418 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10419 %{
10420   match(Set dst (OrL dst src));
10421   effect(KILL cr);
10422 
10423   format %{ "orq     $dst, $src\t# long" %}
10424   ins_encode %{
10425     __ orq($dst$$Register, $src$$Register);
10426   %}
10427   ins_pipe(ialu_reg_reg);
10428 %}
10429 
10430 // Use any_RegP to match R15 (TLS register) without spilling.
10431 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10432   match(Set dst (OrL dst (CastP2X src)));
10433   effect(KILL cr);
10434 
10435   format %{ "orq     $dst, $src\t# long" %}
10436   ins_encode %{
10437     __ orq($dst$$Register, $src$$Register);
10438   %}
10439   ins_pipe(ialu_reg_reg);
10440 %}
10441 
10442 
10443 // Or Register with Immediate
10444 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10445 %{
10446   match(Set dst (OrL dst src));
10447   effect(KILL cr);
10448 
10449   format %{ "orq     $dst, $src\t# long" %}
10450   ins_encode %{
10451     __ orq($dst$$Register, $src$$constant);
10452   %}
10453   ins_pipe(ialu_reg);
10454 %}
10455 
10456 // Or Register with Memory
10457 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10458 %{
10459   match(Set dst (OrL dst (LoadL src)));
10460   effect(KILL cr);
10461 
10462   ins_cost(150);
10463   format %{ "orq     $dst, $src\t# long" %}
10464   ins_encode %{
10465     __ orq($dst$$Register, $src$$Address);
10466   %}
10467   ins_pipe(ialu_reg_mem);
10468 %}
10469 
10470 // Or Memory with Register
10471 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10472 %{
10473   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10474   effect(KILL cr);
10475 
10476   ins_cost(150);
10477   format %{ "orq     $dst, $src\t# long" %}
10478   ins_encode %{
10479     __ orq($dst$$Address, $src$$Register);
10480   %}
10481   ins_pipe(ialu_mem_reg);
10482 %}
10483 
10484 // Or Memory with Immediate
10485 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10486 %{
10487   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10488   effect(KILL cr);
10489 
10490   ins_cost(125);
10491   format %{ "orq     $dst, $src\t# long" %}
10492   ins_encode %{
10493     __ orq($dst$$Address, $src$$constant);
10494   %}
10495   ins_pipe(ialu_mem_imm);
10496 %}
10497 
10498 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10499 %{
10500   // con should be a pure 64-bit power of 2 immediate
10501   // because AND/OR works well enough for 8/32-bit values.
10502   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10503 
10504   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10505   effect(KILL cr);
10506 
10507   ins_cost(125);
10508   format %{ "btsq    $dst, log2($con)\t# long" %}
10509   ins_encode %{
10510     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10511   %}
10512   ins_pipe(ialu_mem_imm);
10513 %}
10514 
10515 // Xor Instructions
10516 // Xor Register with Register
10517 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10518 %{
10519   match(Set dst (XorL dst src));
10520   effect(KILL cr);
10521 
10522   format %{ "xorq    $dst, $src\t# long" %}
10523   ins_encode %{
10524     __ xorq($dst$$Register, $src$$Register);
10525   %}
10526   ins_pipe(ialu_reg_reg);
10527 %}
10528 
10529 // Xor Register with Immediate -1
10530 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10531   match(Set dst (XorL dst imm));
10532 
10533   format %{ "notq   $dst" %}
10534   ins_encode %{
10535      __ notq($dst$$Register);
10536   %}
10537   ins_pipe(ialu_reg);
10538 %}
10539 
10540 // Xor Register with Immediate
10541 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10542 %{
10543   match(Set dst (XorL dst src));
10544   effect(KILL cr);
10545 
10546   format %{ "xorq    $dst, $src\t# long" %}
10547   ins_encode %{
10548     __ xorq($dst$$Register, $src$$constant);
10549   %}
10550   ins_pipe(ialu_reg);
10551 %}
10552 
10553 // Xor Register with Memory
10554 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10555 %{
10556   match(Set dst (XorL dst (LoadL src)));
10557   effect(KILL cr);
10558 
10559   ins_cost(150);
10560   format %{ "xorq    $dst, $src\t# long" %}
10561   ins_encode %{
10562     __ xorq($dst$$Register, $src$$Address);
10563   %}
10564   ins_pipe(ialu_reg_mem);
10565 %}
10566 
10567 // Xor Memory with Register
10568 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10569 %{
10570   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10571   effect(KILL cr);
10572 
10573   ins_cost(150);
10574   format %{ "xorq    $dst, $src\t# long" %}
10575   ins_encode %{
10576     __ xorq($dst$$Address, $src$$Register);
10577   %}
10578   ins_pipe(ialu_mem_reg);
10579 %}
10580 
10581 // Xor Memory with Immediate
10582 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10583 %{
10584   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10585   effect(KILL cr);
10586 
10587   ins_cost(125);
10588   format %{ "xorq    $dst, $src\t# long" %}
10589   ins_encode %{
10590     __ xorq($dst$$Address, $src$$constant);
10591   %}
10592   ins_pipe(ialu_mem_imm);
10593 %}
10594 
10595 // Convert Int to Boolean
10596 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10597 %{
10598   match(Set dst (Conv2B src));
10599   effect(KILL cr);
10600 
10601   format %{ "testl   $src, $src\t# ci2b\n\t"
10602             "setnz   $dst\n\t"
10603             "movzbl  $dst, $dst" %}
10604   ins_encode %{
10605     __ testl($src$$Register, $src$$Register);
10606     __ set_byte_if_not_zero($dst$$Register);
10607     __ movzbl($dst$$Register, $dst$$Register);
10608   %}
10609   ins_pipe(pipe_slow); // XXX
10610 %}
10611 
10612 // Convert Pointer to Boolean
10613 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10614 %{
10615   match(Set dst (Conv2B src));
10616   effect(KILL cr);
10617 
10618   format %{ "testq   $src, $src\t# cp2b\n\t"
10619             "setnz   $dst\n\t"
10620             "movzbl  $dst, $dst" %}
10621   ins_encode %{
10622     __ testq($src$$Register, $src$$Register);
10623     __ set_byte_if_not_zero($dst$$Register);
10624     __ movzbl($dst$$Register, $dst$$Register);
10625   %}
10626   ins_pipe(pipe_slow); // XXX
10627 %}
10628 
10629 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10630 %{
10631   match(Set dst (CmpLTMask p q));
10632   effect(KILL cr);
10633 
10634   ins_cost(400);
10635   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10636             "setlt   $dst\n\t"
10637             "movzbl  $dst, $dst\n\t"
10638             "negl    $dst" %}
10639   ins_encode %{
10640     __ cmpl($p$$Register, $q$$Register);
10641     __ setl($dst$$Register);
10642     __ movzbl($dst$$Register, $dst$$Register);
10643     __ negl($dst$$Register);
10644   %}
10645   ins_pipe(pipe_slow);
10646 %}
10647 
10648 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10649 %{
10650   match(Set dst (CmpLTMask dst zero));
10651   effect(KILL cr);
10652 
10653   ins_cost(100);
10654   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10655   ins_encode %{
10656     __ sarl($dst$$Register, 31);
10657   %}
10658   ins_pipe(ialu_reg);
10659 %}
10660 
10661 /* Better to save a register than avoid a branch */
10662 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10663 %{
10664   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10665   effect(KILL cr);
10666   ins_cost(300);
10667   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10668             "jge     done\n\t"
10669             "addl    $p,$y\n"
10670             "done:   " %}
10671   ins_encode %{
10672     Register Rp = $p$$Register;
10673     Register Rq = $q$$Register;
10674     Register Ry = $y$$Register;
10675     Label done;
10676     __ subl(Rp, Rq);
10677     __ jccb(Assembler::greaterEqual, done);
10678     __ addl(Rp, Ry);
10679     __ bind(done);
10680   %}
10681   ins_pipe(pipe_cmplt);
10682 %}
10683 
10684 /* Better to save a register than avoid a branch */
10685 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10686 %{
10687   match(Set y (AndI (CmpLTMask p q) y));
10688   effect(KILL cr);
10689 
10690   ins_cost(300);
10691 
10692   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10693             "jlt     done\n\t"
10694             "xorl    $y, $y\n"
10695             "done:   " %}
10696   ins_encode %{
10697     Register Rp = $p$$Register;
10698     Register Rq = $q$$Register;
10699     Register Ry = $y$$Register;
10700     Label done;
10701     __ cmpl(Rp, Rq);
10702     __ jccb(Assembler::less, done);
10703     __ xorl(Ry, Ry);
10704     __ bind(done);
10705   %}
10706   ins_pipe(pipe_cmplt);
10707 %}
10708 
10709 
10710 //---------- FP Instructions------------------------------------------------
10711 
10712 // Really expensive, avoid
10713 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10714 %{
10715   match(Set cr (CmpF src1 src2));
10716 
10717   ins_cost(500);
10718   format %{ "ucomiss $src1, $src2\n\t"
10719             "jnp,s   exit\n\t"
10720             "pushfq\t# saw NaN, set CF\n\t"
10721             "andq    [rsp], #0xffffff2b\n\t"
10722             "popfq\n"
10723     "exit:" %}
10724   ins_encode %{
10725     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10726     emit_cmpfp_fixup(_masm);
10727   %}
10728   ins_pipe(pipe_slow);
10729 %}
10730 
10731 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10732   match(Set cr (CmpF src1 src2));
10733 
10734   ins_cost(100);
10735   format %{ "ucomiss $src1, $src2" %}
10736   ins_encode %{
10737     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10738   %}
10739   ins_pipe(pipe_slow);
10740 %}
10741 
10742 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10743   match(Set cr (CmpF src1 (LoadF src2)));
10744 
10745   ins_cost(100);
10746   format %{ "ucomiss $src1, $src2" %}
10747   ins_encode %{
10748     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10749   %}
10750   ins_pipe(pipe_slow);
10751 %}
10752 
10753 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10754   match(Set cr (CmpF src con));
10755   ins_cost(100);
10756   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10757   ins_encode %{
10758     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10759   %}
10760   ins_pipe(pipe_slow);
10761 %}
10762 
10763 // Really expensive, avoid
10764 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10765 %{
10766   match(Set cr (CmpD src1 src2));
10767 
10768   ins_cost(500);
10769   format %{ "ucomisd $src1, $src2\n\t"
10770             "jnp,s   exit\n\t"
10771             "pushfq\t# saw NaN, set CF\n\t"
10772             "andq    [rsp], #0xffffff2b\n\t"
10773             "popfq\n"
10774     "exit:" %}
10775   ins_encode %{
10776     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10777     emit_cmpfp_fixup(_masm);
10778   %}
10779   ins_pipe(pipe_slow);
10780 %}
10781 
10782 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10783   match(Set cr (CmpD src1 src2));
10784 
10785   ins_cost(100);
10786   format %{ "ucomisd $src1, $src2 test" %}
10787   ins_encode %{
10788     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10789   %}
10790   ins_pipe(pipe_slow);
10791 %}
10792 
10793 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10794   match(Set cr (CmpD src1 (LoadD src2)));
10795 
10796   ins_cost(100);
10797   format %{ "ucomisd $src1, $src2" %}
10798   ins_encode %{
10799     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10800   %}
10801   ins_pipe(pipe_slow);
10802 %}
10803 
10804 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10805   match(Set cr (CmpD src con));
10806   ins_cost(100);
10807   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10808   ins_encode %{
10809     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10810   %}
10811   ins_pipe(pipe_slow);
10812 %}
10813 
10814 // Compare into -1,0,1
10815 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10816 %{
10817   match(Set dst (CmpF3 src1 src2));
10818   effect(KILL cr);
10819 
10820   ins_cost(275);
10821   format %{ "ucomiss $src1, $src2\n\t"
10822             "movl    $dst, #-1\n\t"
10823             "jp,s    done\n\t"
10824             "jb,s    done\n\t"
10825             "setne   $dst\n\t"
10826             "movzbl  $dst, $dst\n"
10827     "done:" %}
10828   ins_encode %{
10829     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10830     emit_cmpfp3(_masm, $dst$$Register);
10831   %}
10832   ins_pipe(pipe_slow);
10833 %}
10834 
10835 // Compare into -1,0,1
10836 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10837 %{
10838   match(Set dst (CmpF3 src1 (LoadF src2)));
10839   effect(KILL cr);
10840 
10841   ins_cost(275);
10842   format %{ "ucomiss $src1, $src2\n\t"
10843             "movl    $dst, #-1\n\t"
10844             "jp,s    done\n\t"
10845             "jb,s    done\n\t"
10846             "setne   $dst\n\t"
10847             "movzbl  $dst, $dst\n"
10848     "done:" %}
10849   ins_encode %{
10850     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10851     emit_cmpfp3(_masm, $dst$$Register);
10852   %}
10853   ins_pipe(pipe_slow);
10854 %}
10855 
10856 // Compare into -1,0,1
10857 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10858   match(Set dst (CmpF3 src con));
10859   effect(KILL cr);
10860 
10861   ins_cost(275);
10862   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10863             "movl    $dst, #-1\n\t"
10864             "jp,s    done\n\t"
10865             "jb,s    done\n\t"
10866             "setne   $dst\n\t"
10867             "movzbl  $dst, $dst\n"
10868     "done:" %}
10869   ins_encode %{
10870     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10871     emit_cmpfp3(_masm, $dst$$Register);
10872   %}
10873   ins_pipe(pipe_slow);
10874 %}
10875 
10876 // Compare into -1,0,1
10877 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10878 %{
10879   match(Set dst (CmpD3 src1 src2));
10880   effect(KILL cr);
10881 
10882   ins_cost(275);
10883   format %{ "ucomisd $src1, $src2\n\t"
10884             "movl    $dst, #-1\n\t"
10885             "jp,s    done\n\t"
10886             "jb,s    done\n\t"
10887             "setne   $dst\n\t"
10888             "movzbl  $dst, $dst\n"
10889     "done:" %}
10890   ins_encode %{
10891     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10892     emit_cmpfp3(_masm, $dst$$Register);
10893   %}
10894   ins_pipe(pipe_slow);
10895 %}
10896 
10897 // Compare into -1,0,1
10898 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10899 %{
10900   match(Set dst (CmpD3 src1 (LoadD src2)));
10901   effect(KILL cr);
10902 
10903   ins_cost(275);
10904   format %{ "ucomisd $src1, $src2\n\t"
10905             "movl    $dst, #-1\n\t"
10906             "jp,s    done\n\t"
10907             "jb,s    done\n\t"
10908             "setne   $dst\n\t"
10909             "movzbl  $dst, $dst\n"
10910     "done:" %}
10911   ins_encode %{
10912     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10913     emit_cmpfp3(_masm, $dst$$Register);
10914   %}
10915   ins_pipe(pipe_slow);
10916 %}
10917 
10918 // Compare into -1,0,1
10919 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10920   match(Set dst (CmpD3 src con));
10921   effect(KILL cr);
10922 
10923   ins_cost(275);
10924   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10925             "movl    $dst, #-1\n\t"
10926             "jp,s    done\n\t"
10927             "jb,s    done\n\t"
10928             "setne   $dst\n\t"
10929             "movzbl  $dst, $dst\n"
10930     "done:" %}
10931   ins_encode %{
10932     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10933     emit_cmpfp3(_masm, $dst$$Register);
10934   %}
10935   ins_pipe(pipe_slow);
10936 %}
10937 
10938 //----------Arithmetic Conversion Instructions---------------------------------
10939 
10940 instruct convF2D_reg_reg(regD dst, regF src)
10941 %{
10942   match(Set dst (ConvF2D src));
10943 
10944   format %{ "cvtss2sd $dst, $src" %}
10945   ins_encode %{
10946     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10947   %}
10948   ins_pipe(pipe_slow); // XXX
10949 %}
10950 
10951 instruct convF2D_reg_mem(regD dst, memory src)
10952 %{
10953   match(Set dst (ConvF2D (LoadF src)));
10954 
10955   format %{ "cvtss2sd $dst, $src" %}
10956   ins_encode %{
10957     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10958   %}
10959   ins_pipe(pipe_slow); // XXX
10960 %}
10961 
10962 instruct convD2F_reg_reg(regF dst, regD src)
10963 %{
10964   match(Set dst (ConvD2F src));
10965 
10966   format %{ "cvtsd2ss $dst, $src" %}
10967   ins_encode %{
10968     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10969   %}
10970   ins_pipe(pipe_slow); // XXX
10971 %}
10972 
10973 instruct convD2F_reg_mem(regF dst, memory src)
10974 %{
10975   match(Set dst (ConvD2F (LoadD src)));
10976 
10977   format %{ "cvtsd2ss $dst, $src" %}
10978   ins_encode %{
10979     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10980   %}
10981   ins_pipe(pipe_slow); // XXX
10982 %}
10983 
10984 // XXX do mem variants
10985 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10986 %{
10987   match(Set dst (ConvF2I src));
10988   effect(KILL cr);
10989   format %{ "convert_f2i $dst,$src" %}
10990   ins_encode %{
10991     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10992   %}
10993   ins_pipe(pipe_slow);
10994 %}
10995 
10996 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10997 %{
10998   match(Set dst (ConvF2L src));
10999   effect(KILL cr);
11000   format %{ "convert_f2l $dst,$src"%}
11001   ins_encode %{
11002     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11003   %}
11004   ins_pipe(pipe_slow);
11005 %}
11006 
11007 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11008 %{
11009   match(Set dst (ConvD2I src));
11010   effect(KILL cr);
11011   format %{ "convert_d2i $dst,$src"%}
11012   ins_encode %{
11013     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11014   %}
11015   ins_pipe(pipe_slow);
11016 %}
11017 
11018 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11019 %{
11020   match(Set dst (ConvD2L src));
11021   effect(KILL cr);
11022   format %{ "convert_d2l $dst,$src"%}
11023   ins_encode %{
11024     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11025   %}
11026   ins_pipe(pipe_slow);
11027 %}
11028 
11029 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11030 %{
11031   match(Set dst (RoundD src));
11032   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11033   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11034   ins_encode %{
11035     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11036   %}
11037   ins_pipe(pipe_slow);
11038 %}
11039 
11040 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11041 %{
11042   match(Set dst (RoundF src));
11043   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11044   format %{ "round_float $dst,$src" %}
11045   ins_encode %{
11046     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11047   %}
11048   ins_pipe(pipe_slow);
11049 %}
11050 
11051 instruct convI2F_reg_reg(regF dst, rRegI src)
11052 %{
11053   predicate(!UseXmmI2F);
11054   match(Set dst (ConvI2F src));
11055 
11056   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11057   ins_encode %{
11058     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11059   %}
11060   ins_pipe(pipe_slow); // XXX
11061 %}
11062 
11063 instruct convI2F_reg_mem(regF dst, memory src)
11064 %{
11065   match(Set dst (ConvI2F (LoadI src)));
11066 
11067   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11068   ins_encode %{
11069     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11070   %}
11071   ins_pipe(pipe_slow); // XXX
11072 %}
11073 
11074 instruct convI2D_reg_reg(regD dst, rRegI src)
11075 %{
11076   predicate(!UseXmmI2D);
11077   match(Set dst (ConvI2D src));
11078 
11079   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11080   ins_encode %{
11081     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11082   %}
11083   ins_pipe(pipe_slow); // XXX
11084 %}
11085 
11086 instruct convI2D_reg_mem(regD dst, memory src)
11087 %{
11088   match(Set dst (ConvI2D (LoadI src)));
11089 
11090   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11091   ins_encode %{
11092     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11093   %}
11094   ins_pipe(pipe_slow); // XXX
11095 %}
11096 
11097 instruct convXI2F_reg(regF dst, rRegI src)
11098 %{
11099   predicate(UseXmmI2F);
11100   match(Set dst (ConvI2F src));
11101 
11102   format %{ "movdl $dst, $src\n\t"
11103             "cvtdq2psl $dst, $dst\t# i2f" %}
11104   ins_encode %{
11105     __ movdl($dst$$XMMRegister, $src$$Register);
11106     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11107   %}
11108   ins_pipe(pipe_slow); // XXX
11109 %}
11110 
11111 instruct convXI2D_reg(regD dst, rRegI src)
11112 %{
11113   predicate(UseXmmI2D);
11114   match(Set dst (ConvI2D src));
11115 
11116   format %{ "movdl $dst, $src\n\t"
11117             "cvtdq2pdl $dst, $dst\t# i2d" %}
11118   ins_encode %{
11119     __ movdl($dst$$XMMRegister, $src$$Register);
11120     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11121   %}
11122   ins_pipe(pipe_slow); // XXX
11123 %}
11124 
11125 instruct convL2F_reg_reg(regF dst, rRegL src)
11126 %{
11127   match(Set dst (ConvL2F src));
11128 
11129   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11130   ins_encode %{
11131     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11132   %}
11133   ins_pipe(pipe_slow); // XXX
11134 %}
11135 
11136 instruct convL2F_reg_mem(regF dst, memory src)
11137 %{
11138   match(Set dst (ConvL2F (LoadL src)));
11139 
11140   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11141   ins_encode %{
11142     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11143   %}
11144   ins_pipe(pipe_slow); // XXX
11145 %}
11146 
11147 instruct convL2D_reg_reg(regD dst, rRegL src)
11148 %{
11149   match(Set dst (ConvL2D src));
11150 
11151   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11152   ins_encode %{
11153     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11154   %}
11155   ins_pipe(pipe_slow); // XXX
11156 %}
11157 
11158 instruct convL2D_reg_mem(regD dst, memory src)
11159 %{
11160   match(Set dst (ConvL2D (LoadL src)));
11161 
11162   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11163   ins_encode %{
11164     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11165   %}
11166   ins_pipe(pipe_slow); // XXX
11167 %}
11168 
11169 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11170 %{
11171   match(Set dst (ConvI2L src));
11172 
11173   ins_cost(125);
11174   format %{ "movslq  $dst, $src\t# i2l" %}
11175   ins_encode %{
11176     __ movslq($dst$$Register, $src$$Register);
11177   %}
11178   ins_pipe(ialu_reg_reg);
11179 %}
11180 
11181 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11182 // %{
11183 //   match(Set dst (ConvI2L src));
11184 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11185 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11186 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11187 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11188 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11189 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11190 
11191 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11192 //   ins_encode(enc_copy(dst, src));
11193 // //   opcode(0x63); // needs REX.W
11194 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11195 //   ins_pipe(ialu_reg_reg);
11196 // %}
11197 
11198 // Zero-extend convert int to long
11199 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11200 %{
11201   match(Set dst (AndL (ConvI2L src) mask));
11202 
11203   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11204   ins_encode %{
11205     if ($dst$$reg != $src$$reg) {
11206       __ movl($dst$$Register, $src$$Register);
11207     }
11208   %}
11209   ins_pipe(ialu_reg_reg);
11210 %}
11211 
11212 // Zero-extend convert int to long
11213 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11214 %{
11215   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11216 
11217   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11218   ins_encode %{
11219     __ movl($dst$$Register, $src$$Address);
11220   %}
11221   ins_pipe(ialu_reg_mem);
11222 %}
11223 
11224 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11225 %{
11226   match(Set dst (AndL src mask));
11227 
11228   format %{ "movl    $dst, $src\t# zero-extend long" %}
11229   ins_encode %{
11230     __ movl($dst$$Register, $src$$Register);
11231   %}
11232   ins_pipe(ialu_reg_reg);
11233 %}
11234 
11235 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11236 %{
11237   match(Set dst (ConvL2I src));
11238 
11239   format %{ "movl    $dst, $src\t# l2i" %}
11240   ins_encode %{
11241     __ movl($dst$$Register, $src$$Register);
11242   %}
11243   ins_pipe(ialu_reg_reg);
11244 %}
11245 
11246 
11247 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11248   match(Set dst (MoveF2I src));
11249   effect(DEF dst, USE src);
11250 
11251   ins_cost(125);
11252   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11253   ins_encode %{
11254     __ movl($dst$$Register, Address(rsp, $src$$disp));
11255   %}
11256   ins_pipe(ialu_reg_mem);
11257 %}
11258 
11259 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11260   match(Set dst (MoveI2F src));
11261   effect(DEF dst, USE src);
11262 
11263   ins_cost(125);
11264   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11265   ins_encode %{
11266     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11267   %}
11268   ins_pipe(pipe_slow);
11269 %}
11270 
11271 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11272   match(Set dst (MoveD2L src));
11273   effect(DEF dst, USE src);
11274 
11275   ins_cost(125);
11276   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11277   ins_encode %{
11278     __ movq($dst$$Register, Address(rsp, $src$$disp));
11279   %}
11280   ins_pipe(ialu_reg_mem);
11281 %}
11282 
11283 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11284   predicate(!UseXmmLoadAndClearUpper);
11285   match(Set dst (MoveL2D src));
11286   effect(DEF dst, USE src);
11287 
11288   ins_cost(125);
11289   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11290   ins_encode %{
11291     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11292   %}
11293   ins_pipe(pipe_slow);
11294 %}
11295 
11296 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11297   predicate(UseXmmLoadAndClearUpper);
11298   match(Set dst (MoveL2D src));
11299   effect(DEF dst, USE src);
11300 
11301   ins_cost(125);
11302   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11303   ins_encode %{
11304     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11305   %}
11306   ins_pipe(pipe_slow);
11307 %}
11308 
11309 
11310 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11311   match(Set dst (MoveF2I src));
11312   effect(DEF dst, USE src);
11313 
11314   ins_cost(95); // XXX
11315   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11316   ins_encode %{
11317     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11318   %}
11319   ins_pipe(pipe_slow);
11320 %}
11321 
11322 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11323   match(Set dst (MoveI2F src));
11324   effect(DEF dst, USE src);
11325 
11326   ins_cost(100);
11327   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11328   ins_encode %{
11329     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11330   %}
11331   ins_pipe( ialu_mem_reg );
11332 %}
11333 
11334 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11335   match(Set dst (MoveD2L src));
11336   effect(DEF dst, USE src);
11337 
11338   ins_cost(95); // XXX
11339   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11340   ins_encode %{
11341     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11342   %}
11343   ins_pipe(pipe_slow);
11344 %}
11345 
11346 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11347   match(Set dst (MoveL2D src));
11348   effect(DEF dst, USE src);
11349 
11350   ins_cost(100);
11351   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11352   ins_encode %{
11353     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11354   %}
11355   ins_pipe(ialu_mem_reg);
11356 %}
11357 
11358 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11359   match(Set dst (MoveF2I src));
11360   effect(DEF dst, USE src);
11361   ins_cost(85);
11362   format %{ "movd    $dst,$src\t# MoveF2I" %}
11363   ins_encode %{
11364     __ movdl($dst$$Register, $src$$XMMRegister);
11365   %}
11366   ins_pipe( pipe_slow );
11367 %}
11368 
11369 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11370   match(Set dst (MoveD2L src));
11371   effect(DEF dst, USE src);
11372   ins_cost(85);
11373   format %{ "movd    $dst,$src\t# MoveD2L" %}
11374   ins_encode %{
11375     __ movdq($dst$$Register, $src$$XMMRegister);
11376   %}
11377   ins_pipe( pipe_slow );
11378 %}
11379 
11380 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11381   match(Set dst (MoveI2F src));
11382   effect(DEF dst, USE src);
11383   ins_cost(100);
11384   format %{ "movd    $dst,$src\t# MoveI2F" %}
11385   ins_encode %{
11386     __ movdl($dst$$XMMRegister, $src$$Register);
11387   %}
11388   ins_pipe( pipe_slow );
11389 %}
11390 
11391 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11392   match(Set dst (MoveL2D src));
11393   effect(DEF dst, USE src);
11394   ins_cost(100);
11395   format %{ "movd    $dst,$src\t# MoveL2D" %}
11396   ins_encode %{
11397      __ movdq($dst$$XMMRegister, $src$$Register);
11398   %}
11399   ins_pipe( pipe_slow );
11400 %}
11401 
11402 // Fast clearing of an array
11403 // Small ClearArray non-AVX512.
11404 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11405                   Universe dummy, rFlagsReg cr)
11406 %{
11407   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11408   match(Set dummy (ClearArray cnt base));
11409   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11410 
11411   format %{ $$template
11412     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11413     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11414     $$emit$$"jg      LARGE\n\t"
11415     $$emit$$"dec     rcx\n\t"
11416     $$emit$$"js      DONE\t# Zero length\n\t"
11417     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11418     $$emit$$"dec     rcx\n\t"
11419     $$emit$$"jge     LOOP\n\t"
11420     $$emit$$"jmp     DONE\n\t"
11421     $$emit$$"# LARGE:\n\t"
11422     if (UseFastStosb) {
11423        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11424        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11425     } else if (UseXMMForObjInit) {
11426        $$emit$$"mov     rdi,rax\n\t"
11427        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11428        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11429        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11430        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11431        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11432        $$emit$$"add     0x40,rax\n\t"
11433        $$emit$$"# L_zero_64_bytes:\n\t"
11434        $$emit$$"sub     0x8,rcx\n\t"
11435        $$emit$$"jge     L_loop\n\t"
11436        $$emit$$"add     0x4,rcx\n\t"
11437        $$emit$$"jl      L_tail\n\t"
11438        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11439        $$emit$$"add     0x20,rax\n\t"
11440        $$emit$$"sub     0x4,rcx\n\t"
11441        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11442        $$emit$$"add     0x4,rcx\n\t"
11443        $$emit$$"jle     L_end\n\t"
11444        $$emit$$"dec     rcx\n\t"
11445        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11446        $$emit$$"vmovq   xmm0,(rax)\n\t"
11447        $$emit$$"add     0x8,rax\n\t"
11448        $$emit$$"dec     rcx\n\t"
11449        $$emit$$"jge     L_sloop\n\t"
11450        $$emit$$"# L_end:\n\t"
11451     } else {
11452        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11453     }
11454     $$emit$$"# DONE"
11455   %}
11456   ins_encode %{
11457     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11458                  $tmp$$XMMRegister, false, knoreg);
11459   %}
11460   ins_pipe(pipe_slow);
11461 %}
11462 
11463 // Small ClearArray AVX512 non-constant length.
11464 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11465                        Universe dummy, rFlagsReg cr)
11466 %{
11467   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11468   match(Set dummy (ClearArray cnt base));
11469   ins_cost(125);
11470   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11471 
11472   format %{ $$template
11473     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11474     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11475     $$emit$$"jg      LARGE\n\t"
11476     $$emit$$"dec     rcx\n\t"
11477     $$emit$$"js      DONE\t# Zero length\n\t"
11478     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11479     $$emit$$"dec     rcx\n\t"
11480     $$emit$$"jge     LOOP\n\t"
11481     $$emit$$"jmp     DONE\n\t"
11482     $$emit$$"# LARGE:\n\t"
11483     if (UseFastStosb) {
11484        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11485        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11486     } else if (UseXMMForObjInit) {
11487        $$emit$$"mov     rdi,rax\n\t"
11488        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11489        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11490        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11491        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11492        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11493        $$emit$$"add     0x40,rax\n\t"
11494        $$emit$$"# L_zero_64_bytes:\n\t"
11495        $$emit$$"sub     0x8,rcx\n\t"
11496        $$emit$$"jge     L_loop\n\t"
11497        $$emit$$"add     0x4,rcx\n\t"
11498        $$emit$$"jl      L_tail\n\t"
11499        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11500        $$emit$$"add     0x20,rax\n\t"
11501        $$emit$$"sub     0x4,rcx\n\t"
11502        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11503        $$emit$$"add     0x4,rcx\n\t"
11504        $$emit$$"jle     L_end\n\t"
11505        $$emit$$"dec     rcx\n\t"
11506        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11507        $$emit$$"vmovq   xmm0,(rax)\n\t"
11508        $$emit$$"add     0x8,rax\n\t"
11509        $$emit$$"dec     rcx\n\t"
11510        $$emit$$"jge     L_sloop\n\t"
11511        $$emit$$"# L_end:\n\t"
11512     } else {
11513        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11514     }
11515     $$emit$$"# DONE"
11516   %}
11517   ins_encode %{
11518     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11519                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11520   %}
11521   ins_pipe(pipe_slow);
11522 %}
11523 
11524 // Large ClearArray non-AVX512.
11525 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11526                         Universe dummy, rFlagsReg cr)
11527 %{
11528   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11529   match(Set dummy (ClearArray cnt base));
11530   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11531 
11532   format %{ $$template
11533     if (UseFastStosb) {
11534        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11535        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11536        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11537     } else if (UseXMMForObjInit) {
11538        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11539        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11540        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11541        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11542        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11543        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11544        $$emit$$"add     0x40,rax\n\t"
11545        $$emit$$"# L_zero_64_bytes:\n\t"
11546        $$emit$$"sub     0x8,rcx\n\t"
11547        $$emit$$"jge     L_loop\n\t"
11548        $$emit$$"add     0x4,rcx\n\t"
11549        $$emit$$"jl      L_tail\n\t"
11550        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11551        $$emit$$"add     0x20,rax\n\t"
11552        $$emit$$"sub     0x4,rcx\n\t"
11553        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11554        $$emit$$"add     0x4,rcx\n\t"
11555        $$emit$$"jle     L_end\n\t"
11556        $$emit$$"dec     rcx\n\t"
11557        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11558        $$emit$$"vmovq   xmm0,(rax)\n\t"
11559        $$emit$$"add     0x8,rax\n\t"
11560        $$emit$$"dec     rcx\n\t"
11561        $$emit$$"jge     L_sloop\n\t"
11562        $$emit$$"# L_end:\n\t"
11563     } else {
11564        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11565        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11566     }
11567   %}
11568   ins_encode %{
11569     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11570                  $tmp$$XMMRegister, true, knoreg);
11571   %}
11572   ins_pipe(pipe_slow);
11573 %}
11574 
11575 // Large ClearArray AVX512.
11576 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11577                              Universe dummy, rFlagsReg cr)
11578 %{
11579   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11580   match(Set dummy (ClearArray cnt base));
11581   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11582 
11583   format %{ $$template
11584     if (UseFastStosb) {
11585        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11586        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11587        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11588     } else if (UseXMMForObjInit) {
11589        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11590        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11591        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11592        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11593        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11594        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11595        $$emit$$"add     0x40,rax\n\t"
11596        $$emit$$"# L_zero_64_bytes:\n\t"
11597        $$emit$$"sub     0x8,rcx\n\t"
11598        $$emit$$"jge     L_loop\n\t"
11599        $$emit$$"add     0x4,rcx\n\t"
11600        $$emit$$"jl      L_tail\n\t"
11601        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11602        $$emit$$"add     0x20,rax\n\t"
11603        $$emit$$"sub     0x4,rcx\n\t"
11604        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11605        $$emit$$"add     0x4,rcx\n\t"
11606        $$emit$$"jle     L_end\n\t"
11607        $$emit$$"dec     rcx\n\t"
11608        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11609        $$emit$$"vmovq   xmm0,(rax)\n\t"
11610        $$emit$$"add     0x8,rax\n\t"
11611        $$emit$$"dec     rcx\n\t"
11612        $$emit$$"jge     L_sloop\n\t"
11613        $$emit$$"# L_end:\n\t"
11614     } else {
11615        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11616        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11617     }
11618   %}
11619   ins_encode %{
11620     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11621                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11622   %}
11623   ins_pipe(pipe_slow);
11624 %}
11625 
11626 // Small ClearArray AVX512 constant length.
11627 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11628 %{
11629   predicate(!((ClearArrayNode*)n)->is_large() &&
11630               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11631   match(Set dummy (ClearArray cnt base));
11632   ins_cost(100);
11633   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11634   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11635   ins_encode %{
11636    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11637   %}
11638   ins_pipe(pipe_slow);
11639 %}
11640 
11641 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11642                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11643 %{
11644   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11645   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11646   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11647 
11648   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11649   ins_encode %{
11650     __ string_compare($str1$$Register, $str2$$Register,
11651                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11652                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11653   %}
11654   ins_pipe( pipe_slow );
11655 %}
11656 
11657 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11658                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11659 %{
11660   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11661   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11662   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11663 
11664   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11665   ins_encode %{
11666     __ string_compare($str1$$Register, $str2$$Register,
11667                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11668                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11669   %}
11670   ins_pipe( pipe_slow );
11671 %}
11672 
11673 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11674                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11675 %{
11676   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11677   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11678   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11679 
11680   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11681   ins_encode %{
11682     __ string_compare($str1$$Register, $str2$$Register,
11683                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11684                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11685   %}
11686   ins_pipe( pipe_slow );
11687 %}
11688 
11689 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11690                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11691 %{
11692   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11693   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11694   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11695 
11696   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11697   ins_encode %{
11698     __ string_compare($str1$$Register, $str2$$Register,
11699                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11700                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11701   %}
11702   ins_pipe( pipe_slow );
11703 %}
11704 
11705 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11706                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11707 %{
11708   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11709   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11710   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11711 
11712   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11713   ins_encode %{
11714     __ string_compare($str1$$Register, $str2$$Register,
11715                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11716                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11717   %}
11718   ins_pipe( pipe_slow );
11719 %}
11720 
11721 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11722                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11723 %{
11724   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11725   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11726   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11727 
11728   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11729   ins_encode %{
11730     __ string_compare($str1$$Register, $str2$$Register,
11731                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11732                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11733   %}
11734   ins_pipe( pipe_slow );
11735 %}
11736 
11737 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11738                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11739 %{
11740   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11741   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11742   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11743 
11744   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11745   ins_encode %{
11746     __ string_compare($str2$$Register, $str1$$Register,
11747                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11748                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11749   %}
11750   ins_pipe( pipe_slow );
11751 %}
11752 
11753 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11754                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11755 %{
11756   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11757   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11758   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11759 
11760   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11761   ins_encode %{
11762     __ string_compare($str2$$Register, $str1$$Register,
11763                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11764                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11765   %}
11766   ins_pipe( pipe_slow );
11767 %}
11768 
11769 // fast search of substring with known size.
11770 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11771                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11772 %{
11773   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11774   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11775   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11776 
11777   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11778   ins_encode %{
11779     int icnt2 = (int)$int_cnt2$$constant;
11780     if (icnt2 >= 16) {
11781       // IndexOf for constant substrings with size >= 16 elements
11782       // which don't need to be loaded through stack.
11783       __ string_indexofC8($str1$$Register, $str2$$Register,
11784                           $cnt1$$Register, $cnt2$$Register,
11785                           icnt2, $result$$Register,
11786                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11787     } else {
11788       // Small strings are loaded through stack if they cross page boundary.
11789       __ string_indexof($str1$$Register, $str2$$Register,
11790                         $cnt1$$Register, $cnt2$$Register,
11791                         icnt2, $result$$Register,
11792                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11793     }
11794   %}
11795   ins_pipe( pipe_slow );
11796 %}
11797 
11798 // fast search of substring with known size.
11799 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11800                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11801 %{
11802   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11803   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11804   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11805 
11806   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11807   ins_encode %{
11808     int icnt2 = (int)$int_cnt2$$constant;
11809     if (icnt2 >= 8) {
11810       // IndexOf for constant substrings with size >= 8 elements
11811       // which don't need to be loaded through stack.
11812       __ string_indexofC8($str1$$Register, $str2$$Register,
11813                           $cnt1$$Register, $cnt2$$Register,
11814                           icnt2, $result$$Register,
11815                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11816     } else {
11817       // Small strings are loaded through stack if they cross page boundary.
11818       __ string_indexof($str1$$Register, $str2$$Register,
11819                         $cnt1$$Register, $cnt2$$Register,
11820                         icnt2, $result$$Register,
11821                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11822     }
11823   %}
11824   ins_pipe( pipe_slow );
11825 %}
11826 
11827 // fast search of substring with known size.
11828 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11829                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11830 %{
11831   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11832   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11833   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11834 
11835   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11836   ins_encode %{
11837     int icnt2 = (int)$int_cnt2$$constant;
11838     if (icnt2 >= 8) {
11839       // IndexOf for constant substrings with size >= 8 elements
11840       // which don't need to be loaded through stack.
11841       __ string_indexofC8($str1$$Register, $str2$$Register,
11842                           $cnt1$$Register, $cnt2$$Register,
11843                           icnt2, $result$$Register,
11844                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11845     } else {
11846       // Small strings are loaded through stack if they cross page boundary.
11847       __ string_indexof($str1$$Register, $str2$$Register,
11848                         $cnt1$$Register, $cnt2$$Register,
11849                         icnt2, $result$$Register,
11850                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11851     }
11852   %}
11853   ins_pipe( pipe_slow );
11854 %}
11855 
11856 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11857                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11858 %{
11859   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11860   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11861   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11862 
11863   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11864   ins_encode %{
11865     __ string_indexof($str1$$Register, $str2$$Register,
11866                       $cnt1$$Register, $cnt2$$Register,
11867                       (-1), $result$$Register,
11868                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11869   %}
11870   ins_pipe( pipe_slow );
11871 %}
11872 
11873 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11874                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11875 %{
11876   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11877   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11878   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11879 
11880   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11881   ins_encode %{
11882     __ string_indexof($str1$$Register, $str2$$Register,
11883                       $cnt1$$Register, $cnt2$$Register,
11884                       (-1), $result$$Register,
11885                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11886   %}
11887   ins_pipe( pipe_slow );
11888 %}
11889 
11890 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11891                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11892 %{
11893   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11894   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11895   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11896 
11897   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11898   ins_encode %{
11899     __ string_indexof($str1$$Register, $str2$$Register,
11900                       $cnt1$$Register, $cnt2$$Register,
11901                       (-1), $result$$Register,
11902                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11903   %}
11904   ins_pipe( pipe_slow );
11905 %}
11906 
11907 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11908                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11909 %{
11910   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11911   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11912   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11913   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11914   ins_encode %{
11915     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11916                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11917   %}
11918   ins_pipe( pipe_slow );
11919 %}
11920 
11921 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11922                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11923 %{
11924   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11925   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11926   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11927   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11928   ins_encode %{
11929     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11930                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11931   %}
11932   ins_pipe( pipe_slow );
11933 %}
11934 
11935 // fast string equals
11936 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11937                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11938 %{
11939   predicate(!VM_Version::supports_avx512vlbw());
11940   match(Set result (StrEquals (Binary str1 str2) cnt));
11941   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11942 
11943   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11944   ins_encode %{
11945     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11946                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11947                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11948   %}
11949   ins_pipe( pipe_slow );
11950 %}
11951 
11952 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11953                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11954 %{
11955   predicate(VM_Version::supports_avx512vlbw());
11956   match(Set result (StrEquals (Binary str1 str2) cnt));
11957   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11958 
11959   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11960   ins_encode %{
11961     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11962                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11963                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11964   %}
11965   ins_pipe( pipe_slow );
11966 %}
11967 
11968 // fast array equals
11969 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11970                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11971 %{
11972   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11973   match(Set result (AryEq ary1 ary2));
11974   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11975 
11976   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11977   ins_encode %{
11978     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11979                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11980                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11981   %}
11982   ins_pipe( pipe_slow );
11983 %}
11984 
11985 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11986                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11987 %{
11988   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11989   match(Set result (AryEq ary1 ary2));
11990   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11991 
11992   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11993   ins_encode %{
11994     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11995                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11996                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11997   %}
11998   ins_pipe( pipe_slow );
11999 %}
12000 
12001 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12002                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12003 %{
12004   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12005   match(Set result (AryEq ary1 ary2));
12006   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12007 
12008   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12009   ins_encode %{
12010     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12011                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12012                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12013   %}
12014   ins_pipe( pipe_slow );
12015 %}
12016 
12017 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12018                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12019 %{
12020   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12021   match(Set result (AryEq ary1 ary2));
12022   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12023 
12024   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12025   ins_encode %{
12026     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12027                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12028                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12029   %}
12030   ins_pipe( pipe_slow );
12031 %}
12032 
12033 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12034                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12035 %{
12036   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12037   match(Set result (CountPositives ary1 len));
12038   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12039 
12040   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12041   ins_encode %{
12042     __ count_positives($ary1$$Register, $len$$Register,
12043                        $result$$Register, $tmp3$$Register,
12044                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12045   %}
12046   ins_pipe( pipe_slow );
12047 %}
12048 
12049 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12050                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12051 %{
12052   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12053   match(Set result (CountPositives ary1 len));
12054   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12055 
12056   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12057   ins_encode %{
12058     __ count_positives($ary1$$Register, $len$$Register,
12059                        $result$$Register, $tmp3$$Register,
12060                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12061   %}
12062   ins_pipe( pipe_slow );
12063 %}
12064 
12065 // fast char[] to byte[] compression
12066 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12067                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12068   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12069   match(Set result (StrCompressedCopy src (Binary dst len)));
12070   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12071          USE_KILL len, KILL tmp5, KILL cr);
12072 
12073   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12074   ins_encode %{
12075     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12076                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12077                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12078                            knoreg, knoreg);
12079   %}
12080   ins_pipe( pipe_slow );
12081 %}
12082 
12083 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12084                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12085   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12086   match(Set result (StrCompressedCopy src (Binary dst len)));
12087   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12088          USE_KILL len, KILL tmp5, KILL cr);
12089 
12090   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12091   ins_encode %{
12092     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12093                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12094                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12095                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12096   %}
12097   ins_pipe( pipe_slow );
12098 %}
12099 // fast byte[] to char[] inflation
12100 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12101                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12102   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12103   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12104   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12105 
12106   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12107   ins_encode %{
12108     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12109                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12110   %}
12111   ins_pipe( pipe_slow );
12112 %}
12113 
12114 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12115                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12116   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12117   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12118   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12119 
12120   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12121   ins_encode %{
12122     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12123                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12124   %}
12125   ins_pipe( pipe_slow );
12126 %}
12127 
12128 // encode char[] to byte[] in ISO_8859_1
12129 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12130                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12131                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12132   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12133   match(Set result (EncodeISOArray src (Binary dst len)));
12134   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12135 
12136   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12137   ins_encode %{
12138     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12139                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12140                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12141   %}
12142   ins_pipe( pipe_slow );
12143 %}
12144 
12145 // encode char[] to byte[] in ASCII
12146 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12147                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12148                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12149   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12150   match(Set result (EncodeISOArray src (Binary dst len)));
12151   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12152 
12153   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12154   ins_encode %{
12155     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12156                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12157                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12158   %}
12159   ins_pipe( pipe_slow );
12160 %}
12161 
12162 //----------Overflow Math Instructions-----------------------------------------
12163 
12164 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12165 %{
12166   match(Set cr (OverflowAddI op1 op2));
12167   effect(DEF cr, USE_KILL op1, USE op2);
12168 
12169   format %{ "addl    $op1, $op2\t# overflow check int" %}
12170 
12171   ins_encode %{
12172     __ addl($op1$$Register, $op2$$Register);
12173   %}
12174   ins_pipe(ialu_reg_reg);
12175 %}
12176 
12177 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12178 %{
12179   match(Set cr (OverflowAddI op1 op2));
12180   effect(DEF cr, USE_KILL op1, USE op2);
12181 
12182   format %{ "addl    $op1, $op2\t# overflow check int" %}
12183 
12184   ins_encode %{
12185     __ addl($op1$$Register, $op2$$constant);
12186   %}
12187   ins_pipe(ialu_reg_reg);
12188 %}
12189 
12190 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12191 %{
12192   match(Set cr (OverflowAddL op1 op2));
12193   effect(DEF cr, USE_KILL op1, USE op2);
12194 
12195   format %{ "addq    $op1, $op2\t# overflow check long" %}
12196   ins_encode %{
12197     __ addq($op1$$Register, $op2$$Register);
12198   %}
12199   ins_pipe(ialu_reg_reg);
12200 %}
12201 
12202 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12203 %{
12204   match(Set cr (OverflowAddL op1 op2));
12205   effect(DEF cr, USE_KILL op1, USE op2);
12206 
12207   format %{ "addq    $op1, $op2\t# overflow check long" %}
12208   ins_encode %{
12209     __ addq($op1$$Register, $op2$$constant);
12210   %}
12211   ins_pipe(ialu_reg_reg);
12212 %}
12213 
12214 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12215 %{
12216   match(Set cr (OverflowSubI op1 op2));
12217 
12218   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12219   ins_encode %{
12220     __ cmpl($op1$$Register, $op2$$Register);
12221   %}
12222   ins_pipe(ialu_reg_reg);
12223 %}
12224 
12225 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12226 %{
12227   match(Set cr (OverflowSubI op1 op2));
12228 
12229   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12230   ins_encode %{
12231     __ cmpl($op1$$Register, $op2$$constant);
12232   %}
12233   ins_pipe(ialu_reg_reg);
12234 %}
12235 
12236 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12237 %{
12238   match(Set cr (OverflowSubL op1 op2));
12239 
12240   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12241   ins_encode %{
12242     __ cmpq($op1$$Register, $op2$$Register);
12243   %}
12244   ins_pipe(ialu_reg_reg);
12245 %}
12246 
12247 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12248 %{
12249   match(Set cr (OverflowSubL op1 op2));
12250 
12251   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12252   ins_encode %{
12253     __ cmpq($op1$$Register, $op2$$constant);
12254   %}
12255   ins_pipe(ialu_reg_reg);
12256 %}
12257 
12258 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12259 %{
12260   match(Set cr (OverflowSubI zero op2));
12261   effect(DEF cr, USE_KILL op2);
12262 
12263   format %{ "negl    $op2\t# overflow check int" %}
12264   ins_encode %{
12265     __ negl($op2$$Register);
12266   %}
12267   ins_pipe(ialu_reg_reg);
12268 %}
12269 
12270 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12271 %{
12272   match(Set cr (OverflowSubL zero op2));
12273   effect(DEF cr, USE_KILL op2);
12274 
12275   format %{ "negq    $op2\t# overflow check long" %}
12276   ins_encode %{
12277     __ negq($op2$$Register);
12278   %}
12279   ins_pipe(ialu_reg_reg);
12280 %}
12281 
12282 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12283 %{
12284   match(Set cr (OverflowMulI op1 op2));
12285   effect(DEF cr, USE_KILL op1, USE op2);
12286 
12287   format %{ "imull    $op1, $op2\t# overflow check int" %}
12288   ins_encode %{
12289     __ imull($op1$$Register, $op2$$Register);
12290   %}
12291   ins_pipe(ialu_reg_reg_alu0);
12292 %}
12293 
12294 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12295 %{
12296   match(Set cr (OverflowMulI op1 op2));
12297   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12298 
12299   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12300   ins_encode %{
12301     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12302   %}
12303   ins_pipe(ialu_reg_reg_alu0);
12304 %}
12305 
12306 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12307 %{
12308   match(Set cr (OverflowMulL op1 op2));
12309   effect(DEF cr, USE_KILL op1, USE op2);
12310 
12311   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12312   ins_encode %{
12313     __ imulq($op1$$Register, $op2$$Register);
12314   %}
12315   ins_pipe(ialu_reg_reg_alu0);
12316 %}
12317 
12318 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12319 %{
12320   match(Set cr (OverflowMulL op1 op2));
12321   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12322 
12323   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12324   ins_encode %{
12325     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12326   %}
12327   ins_pipe(ialu_reg_reg_alu0);
12328 %}
12329 
12330 
12331 //----------Control Flow Instructions------------------------------------------
12332 // Signed compare Instructions
12333 
12334 // XXX more variants!!
12335 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12336 %{
12337   match(Set cr (CmpI op1 op2));
12338   effect(DEF cr, USE op1, USE op2);
12339 
12340   format %{ "cmpl    $op1, $op2" %}
12341   ins_encode %{
12342     __ cmpl($op1$$Register, $op2$$Register);
12343   %}
12344   ins_pipe(ialu_cr_reg_reg);
12345 %}
12346 
12347 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12348 %{
12349   match(Set cr (CmpI op1 op2));
12350 
12351   format %{ "cmpl    $op1, $op2" %}
12352   ins_encode %{
12353     __ cmpl($op1$$Register, $op2$$constant);
12354   %}
12355   ins_pipe(ialu_cr_reg_imm);
12356 %}
12357 
12358 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12359 %{
12360   match(Set cr (CmpI op1 (LoadI op2)));
12361 
12362   ins_cost(500); // XXX
12363   format %{ "cmpl    $op1, $op2" %}
12364   ins_encode %{
12365     __ cmpl($op1$$Register, $op2$$Address);
12366   %}
12367   ins_pipe(ialu_cr_reg_mem);
12368 %}
12369 
12370 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12371 %{
12372   match(Set cr (CmpI src zero));
12373 
12374   format %{ "testl   $src, $src" %}
12375   ins_encode %{
12376     __ testl($src$$Register, $src$$Register);
12377   %}
12378   ins_pipe(ialu_cr_reg_imm);
12379 %}
12380 
12381 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12382 %{
12383   match(Set cr (CmpI (AndI src con) zero));
12384 
12385   format %{ "testl   $src, $con" %}
12386   ins_encode %{
12387     __ testl($src$$Register, $con$$constant);
12388   %}
12389   ins_pipe(ialu_cr_reg_imm);
12390 %}
12391 
12392 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12393 %{
12394   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12395 
12396   format %{ "testl   $src, $mem" %}
12397   ins_encode %{
12398     __ testl($src$$Register, $mem$$Address);
12399   %}
12400   ins_pipe(ialu_cr_reg_mem);
12401 %}
12402 
12403 // Unsigned compare Instructions; really, same as signed except they
12404 // produce an rFlagsRegU instead of rFlagsReg.
12405 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12406 %{
12407   match(Set cr (CmpU op1 op2));
12408 
12409   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12410   ins_encode %{
12411     __ cmpl($op1$$Register, $op2$$Register);
12412   %}
12413   ins_pipe(ialu_cr_reg_reg);
12414 %}
12415 
12416 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12417 %{
12418   match(Set cr (CmpU op1 op2));
12419 
12420   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12421   ins_encode %{
12422     __ cmpl($op1$$Register, $op2$$constant);
12423   %}
12424   ins_pipe(ialu_cr_reg_imm);
12425 %}
12426 
12427 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12428 %{
12429   match(Set cr (CmpU op1 (LoadI op2)));
12430 
12431   ins_cost(500); // XXX
12432   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12433   ins_encode %{
12434     __ cmpl($op1$$Register, $op2$$Address);
12435   %}
12436   ins_pipe(ialu_cr_reg_mem);
12437 %}
12438 
12439 // // // Cisc-spilled version of cmpU_rReg
12440 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12441 // //%{
12442 // //  match(Set cr (CmpU (LoadI op1) op2));
12443 // //
12444 // //  format %{ "CMPu   $op1,$op2" %}
12445 // //  ins_cost(500);
12446 // //  opcode(0x39);  /* Opcode 39 /r */
12447 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12448 // //%}
12449 
12450 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12451 %{
12452   match(Set cr (CmpU src zero));
12453 
12454   format %{ "testl   $src, $src\t# unsigned" %}
12455   ins_encode %{
12456     __ testl($src$$Register, $src$$Register);
12457   %}
12458   ins_pipe(ialu_cr_reg_imm);
12459 %}
12460 
12461 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12462 %{
12463   match(Set cr (CmpP op1 op2));
12464 
12465   format %{ "cmpq    $op1, $op2\t# ptr" %}
12466   ins_encode %{
12467     __ cmpq($op1$$Register, $op2$$Register);
12468   %}
12469   ins_pipe(ialu_cr_reg_reg);
12470 %}
12471 
12472 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12473 %{
12474   match(Set cr (CmpP op1 (LoadP op2)));
12475   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12476 
12477   ins_cost(500); // XXX
12478   format %{ "cmpq    $op1, $op2\t# ptr" %}
12479   ins_encode %{
12480     __ cmpq($op1$$Register, $op2$$Address);
12481   %}
12482   ins_pipe(ialu_cr_reg_mem);
12483 %}
12484 
12485 // // // Cisc-spilled version of cmpP_rReg
12486 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12487 // //%{
12488 // //  match(Set cr (CmpP (LoadP op1) op2));
12489 // //
12490 // //  format %{ "CMPu   $op1,$op2" %}
12491 // //  ins_cost(500);
12492 // //  opcode(0x39);  /* Opcode 39 /r */
12493 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12494 // //%}
12495 
12496 // XXX this is generalized by compP_rReg_mem???
12497 // Compare raw pointer (used in out-of-heap check).
12498 // Only works because non-oop pointers must be raw pointers
12499 // and raw pointers have no anti-dependencies.
12500 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12501 %{
12502   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12503             n->in(2)->as_Load()->barrier_data() == 0);
12504   match(Set cr (CmpP op1 (LoadP op2)));
12505 
12506   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12507   ins_encode %{
12508     __ cmpq($op1$$Register, $op2$$Address);
12509   %}
12510   ins_pipe(ialu_cr_reg_mem);
12511 %}
12512 
12513 // This will generate a signed flags result. This should be OK since
12514 // any compare to a zero should be eq/neq.
12515 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12516 %{
12517   match(Set cr (CmpP src zero));
12518 
12519   format %{ "testq   $src, $src\t# ptr" %}
12520   ins_encode %{
12521     __ testq($src$$Register, $src$$Register);
12522   %}
12523   ins_pipe(ialu_cr_reg_imm);
12524 %}
12525 
12526 // This will generate a signed flags result. This should be OK since
12527 // any compare to a zero should be eq/neq.
12528 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12529 %{
12530   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12531             n->in(1)->as_Load()->barrier_data() == 0);
12532   match(Set cr (CmpP (LoadP op) zero));
12533 
12534   ins_cost(500); // XXX
12535   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12536   ins_encode %{
12537     __ testq($op$$Address, 0xFFFFFFFF);
12538   %}
12539   ins_pipe(ialu_cr_reg_imm);
12540 %}
12541 
12542 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12543 %{
12544   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12545             n->in(1)->as_Load()->barrier_data() == 0);
12546   match(Set cr (CmpP (LoadP mem) zero));
12547 
12548   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12549   ins_encode %{
12550     __ cmpq(r12, $mem$$Address);
12551   %}
12552   ins_pipe(ialu_cr_reg_mem);
12553 %}
12554 
12555 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12556 %{
12557   match(Set cr (CmpN op1 op2));
12558 
12559   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12560   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12561   ins_pipe(ialu_cr_reg_reg);
12562 %}
12563 
12564 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12565 %{
12566   match(Set cr (CmpN src (LoadN mem)));
12567 
12568   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12569   ins_encode %{
12570     __ cmpl($src$$Register, $mem$$Address);
12571   %}
12572   ins_pipe(ialu_cr_reg_mem);
12573 %}
12574 
12575 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12576   match(Set cr (CmpN op1 op2));
12577 
12578   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12579   ins_encode %{
12580     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12581   %}
12582   ins_pipe(ialu_cr_reg_imm);
12583 %}
12584 
12585 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12586 %{
12587   match(Set cr (CmpN src (LoadN mem)));
12588 
12589   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12590   ins_encode %{
12591     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12592   %}
12593   ins_pipe(ialu_cr_reg_mem);
12594 %}
12595 
12596 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12597   match(Set cr (CmpN op1 op2));
12598 
12599   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12600   ins_encode %{
12601     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12602   %}
12603   ins_pipe(ialu_cr_reg_imm);
12604 %}
12605 
12606 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12607 %{
12608   match(Set cr (CmpN src (LoadNKlass mem)));
12609 
12610   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12611   ins_encode %{
12612     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12613   %}
12614   ins_pipe(ialu_cr_reg_mem);
12615 %}
12616 
12617 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12618   match(Set cr (CmpN src zero));
12619 
12620   format %{ "testl   $src, $src\t# compressed ptr" %}
12621   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12622   ins_pipe(ialu_cr_reg_imm);
12623 %}
12624 
12625 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12626 %{
12627   predicate(CompressedOops::base() != NULL);
12628   match(Set cr (CmpN (LoadN mem) zero));
12629 
12630   ins_cost(500); // XXX
12631   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12632   ins_encode %{
12633     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12634   %}
12635   ins_pipe(ialu_cr_reg_mem);
12636 %}
12637 
12638 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12639 %{
12640   predicate(CompressedOops::base() == NULL);
12641   match(Set cr (CmpN (LoadN mem) zero));
12642 
12643   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12644   ins_encode %{
12645     __ cmpl(r12, $mem$$Address);
12646   %}
12647   ins_pipe(ialu_cr_reg_mem);
12648 %}
12649 
12650 // Yanked all unsigned pointer compare operations.
12651 // Pointer compares are done with CmpP which is already unsigned.
12652 
12653 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12654 %{
12655   match(Set cr (CmpL op1 op2));
12656 
12657   format %{ "cmpq    $op1, $op2" %}
12658   ins_encode %{
12659     __ cmpq($op1$$Register, $op2$$Register);
12660   %}
12661   ins_pipe(ialu_cr_reg_reg);
12662 %}
12663 
12664 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12665 %{
12666   match(Set cr (CmpL op1 op2));
12667 
12668   format %{ "cmpq    $op1, $op2" %}
12669   ins_encode %{
12670     __ cmpq($op1$$Register, $op2$$constant);
12671   %}
12672   ins_pipe(ialu_cr_reg_imm);
12673 %}
12674 
12675 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12676 %{
12677   match(Set cr (CmpL op1 (LoadL op2)));
12678 
12679   format %{ "cmpq    $op1, $op2" %}
12680   ins_encode %{
12681     __ cmpq($op1$$Register, $op2$$Address);
12682   %}
12683   ins_pipe(ialu_cr_reg_mem);
12684 %}
12685 
12686 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12687 %{
12688   match(Set cr (CmpL src zero));
12689 
12690   format %{ "testq   $src, $src" %}
12691   ins_encode %{
12692     __ testq($src$$Register, $src$$Register);
12693   %}
12694   ins_pipe(ialu_cr_reg_imm);
12695 %}
12696 
12697 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12698 %{
12699   match(Set cr (CmpL (AndL src con) zero));
12700 
12701   format %{ "testq   $src, $con\t# long" %}
12702   ins_encode %{
12703     __ testq($src$$Register, $con$$constant);
12704   %}
12705   ins_pipe(ialu_cr_reg_imm);
12706 %}
12707 
12708 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12709 %{
12710   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12711 
12712   format %{ "testq   $src, $mem" %}
12713   ins_encode %{
12714     __ testq($src$$Register, $mem$$Address);
12715   %}
12716   ins_pipe(ialu_cr_reg_mem);
12717 %}
12718 
12719 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12720 %{
12721   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12722 
12723   format %{ "testq   $src, $mem" %}
12724   ins_encode %{
12725     __ testq($src$$Register, $mem$$Address);
12726   %}
12727   ins_pipe(ialu_cr_reg_mem);
12728 %}
12729 
12730 // Manifest a CmpU result in an integer register.  Very painful.
12731 // This is the test to avoid.
12732 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
12733 %{
12734   match(Set dst (CmpU3 src1 src2));
12735   effect(KILL flags);
12736 
12737   ins_cost(275); // XXX
12738   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
12739             "movl    $dst, -1\n\t"
12740             "jb,u    done\n\t"
12741             "setne   $dst\n\t"
12742             "movzbl  $dst, $dst\n\t"
12743     "done:" %}
12744   ins_encode %{
12745     Label done;
12746     __ cmpl($src1$$Register, $src2$$Register);
12747     __ movl($dst$$Register, -1);
12748     __ jccb(Assembler::below, done);
12749     __ setne($dst$$Register);
12750     __ movzbl($dst$$Register, $dst$$Register);
12751     __ bind(done);
12752   %}
12753   ins_pipe(pipe_slow);
12754 %}
12755 
12756 // Manifest a CmpL result in an integer register.  Very painful.
12757 // This is the test to avoid.
12758 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12759 %{
12760   match(Set dst (CmpL3 src1 src2));
12761   effect(KILL flags);
12762 
12763   ins_cost(275); // XXX
12764   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12765             "movl    $dst, -1\n\t"
12766             "jl,s    done\n\t"
12767             "setne   $dst\n\t"
12768             "movzbl  $dst, $dst\n\t"
12769     "done:" %}
12770   ins_encode %{
12771     Label done;
12772     __ cmpq($src1$$Register, $src2$$Register);
12773     __ movl($dst$$Register, -1);
12774     __ jccb(Assembler::less, done);
12775     __ setne($dst$$Register);
12776     __ movzbl($dst$$Register, $dst$$Register);
12777     __ bind(done);
12778   %}
12779   ins_pipe(pipe_slow);
12780 %}
12781 
12782 // Manifest a CmpUL result in an integer register.  Very painful.
12783 // This is the test to avoid.
12784 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12785 %{
12786   match(Set dst (CmpUL3 src1 src2));
12787   effect(KILL flags);
12788 
12789   ins_cost(275); // XXX
12790   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12791             "movl    $dst, -1\n\t"
12792             "jb,u    done\n\t"
12793             "setne   $dst\n\t"
12794             "movzbl  $dst, $dst\n\t"
12795     "done:" %}
12796   ins_encode %{
12797     Label done;
12798     __ cmpq($src1$$Register, $src2$$Register);
12799     __ movl($dst$$Register, -1);
12800     __ jccb(Assembler::below, done);
12801     __ setne($dst$$Register);
12802     __ movzbl($dst$$Register, $dst$$Register);
12803     __ bind(done);
12804   %}
12805   ins_pipe(pipe_slow);
12806 %}
12807 
12808 // Unsigned long compare Instructions; really, same as signed long except they
12809 // produce an rFlagsRegU instead of rFlagsReg.
12810 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12811 %{
12812   match(Set cr (CmpUL op1 op2));
12813 
12814   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12815   ins_encode %{
12816     __ cmpq($op1$$Register, $op2$$Register);
12817   %}
12818   ins_pipe(ialu_cr_reg_reg);
12819 %}
12820 
12821 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12822 %{
12823   match(Set cr (CmpUL op1 op2));
12824 
12825   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12826   ins_encode %{
12827     __ cmpq($op1$$Register, $op2$$constant);
12828   %}
12829   ins_pipe(ialu_cr_reg_imm);
12830 %}
12831 
12832 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12833 %{
12834   match(Set cr (CmpUL op1 (LoadL op2)));
12835 
12836   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12837   ins_encode %{
12838     __ cmpq($op1$$Register, $op2$$Address);
12839   %}
12840   ins_pipe(ialu_cr_reg_mem);
12841 %}
12842 
12843 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12844 %{
12845   match(Set cr (CmpUL src zero));
12846 
12847   format %{ "testq   $src, $src\t# unsigned" %}
12848   ins_encode %{
12849     __ testq($src$$Register, $src$$Register);
12850   %}
12851   ins_pipe(ialu_cr_reg_imm);
12852 %}
12853 
12854 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12855 %{
12856   match(Set cr (CmpI (LoadB mem) imm));
12857 
12858   ins_cost(125);
12859   format %{ "cmpb    $mem, $imm" %}
12860   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12861   ins_pipe(ialu_cr_reg_mem);
12862 %}
12863 
12864 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12865 %{
12866   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12867 
12868   ins_cost(125);
12869   format %{ "testb   $mem, $imm\t# ubyte" %}
12870   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12871   ins_pipe(ialu_cr_reg_mem);
12872 %}
12873 
12874 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12875 %{
12876   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12877 
12878   ins_cost(125);
12879   format %{ "testb   $mem, $imm\t# byte" %}
12880   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12881   ins_pipe(ialu_cr_reg_mem);
12882 %}
12883 
12884 //----------Max and Min--------------------------------------------------------
12885 // Min Instructions
12886 
12887 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12888 %{
12889   effect(USE_DEF dst, USE src, USE cr);
12890 
12891   format %{ "cmovlgt $dst, $src\t# min" %}
12892   ins_encode %{
12893     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12894   %}
12895   ins_pipe(pipe_cmov_reg);
12896 %}
12897 
12898 
12899 instruct minI_rReg(rRegI dst, rRegI src)
12900 %{
12901   match(Set dst (MinI dst src));
12902 
12903   ins_cost(200);
12904   expand %{
12905     rFlagsReg cr;
12906     compI_rReg(cr, dst, src);
12907     cmovI_reg_g(dst, src, cr);
12908   %}
12909 %}
12910 
12911 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12912 %{
12913   effect(USE_DEF dst, USE src, USE cr);
12914 
12915   format %{ "cmovllt $dst, $src\t# max" %}
12916   ins_encode %{
12917     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12918   %}
12919   ins_pipe(pipe_cmov_reg);
12920 %}
12921 
12922 
12923 instruct maxI_rReg(rRegI dst, rRegI src)
12924 %{
12925   match(Set dst (MaxI dst src));
12926 
12927   ins_cost(200);
12928   expand %{
12929     rFlagsReg cr;
12930     compI_rReg(cr, dst, src);
12931     cmovI_reg_l(dst, src, cr);
12932   %}
12933 %}
12934 
12935 // ============================================================================
12936 // Branch Instructions
12937 
12938 // Jump Direct - Label defines a relative address from JMP+1
12939 instruct jmpDir(label labl)
12940 %{
12941   match(Goto);
12942   effect(USE labl);
12943 
12944   ins_cost(300);
12945   format %{ "jmp     $labl" %}
12946   size(5);
12947   ins_encode %{
12948     Label* L = $labl$$label;
12949     __ jmp(*L, false); // Always long jump
12950   %}
12951   ins_pipe(pipe_jmp);
12952 %}
12953 
12954 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12955 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12956 %{
12957   match(If cop cr);
12958   effect(USE labl);
12959 
12960   ins_cost(300);
12961   format %{ "j$cop     $labl" %}
12962   size(6);
12963   ins_encode %{
12964     Label* L = $labl$$label;
12965     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12966   %}
12967   ins_pipe(pipe_jcc);
12968 %}
12969 
12970 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12971 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12972 %{
12973   match(CountedLoopEnd cop cr);
12974   effect(USE labl);
12975 
12976   ins_cost(300);
12977   format %{ "j$cop     $labl\t# loop end" %}
12978   size(6);
12979   ins_encode %{
12980     Label* L = $labl$$label;
12981     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12982   %}
12983   ins_pipe(pipe_jcc);
12984 %}
12985 
12986 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12987 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12988   match(CountedLoopEnd cop cmp);
12989   effect(USE labl);
12990 
12991   ins_cost(300);
12992   format %{ "j$cop,u   $labl\t# loop end" %}
12993   size(6);
12994   ins_encode %{
12995     Label* L = $labl$$label;
12996     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12997   %}
12998   ins_pipe(pipe_jcc);
12999 %}
13000 
13001 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13002   match(CountedLoopEnd cop cmp);
13003   effect(USE labl);
13004 
13005   ins_cost(200);
13006   format %{ "j$cop,u   $labl\t# loop end" %}
13007   size(6);
13008   ins_encode %{
13009     Label* L = $labl$$label;
13010     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13011   %}
13012   ins_pipe(pipe_jcc);
13013 %}
13014 
13015 // Jump Direct Conditional - using unsigned comparison
13016 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13017   match(If cop cmp);
13018   effect(USE labl);
13019 
13020   ins_cost(300);
13021   format %{ "j$cop,u   $labl" %}
13022   size(6);
13023   ins_encode %{
13024     Label* L = $labl$$label;
13025     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13026   %}
13027   ins_pipe(pipe_jcc);
13028 %}
13029 
13030 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13031   match(If cop cmp);
13032   effect(USE labl);
13033 
13034   ins_cost(200);
13035   format %{ "j$cop,u   $labl" %}
13036   size(6);
13037   ins_encode %{
13038     Label* L = $labl$$label;
13039     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13040   %}
13041   ins_pipe(pipe_jcc);
13042 %}
13043 
13044 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13045   match(If cop cmp);
13046   effect(USE labl);
13047 
13048   ins_cost(200);
13049   format %{ $$template
13050     if ($cop$$cmpcode == Assembler::notEqual) {
13051       $$emit$$"jp,u    $labl\n\t"
13052       $$emit$$"j$cop,u   $labl"
13053     } else {
13054       $$emit$$"jp,u    done\n\t"
13055       $$emit$$"j$cop,u   $labl\n\t"
13056       $$emit$$"done:"
13057     }
13058   %}
13059   ins_encode %{
13060     Label* l = $labl$$label;
13061     if ($cop$$cmpcode == Assembler::notEqual) {
13062       __ jcc(Assembler::parity, *l, false);
13063       __ jcc(Assembler::notEqual, *l, false);
13064     } else if ($cop$$cmpcode == Assembler::equal) {
13065       Label done;
13066       __ jccb(Assembler::parity, done);
13067       __ jcc(Assembler::equal, *l, false);
13068       __ bind(done);
13069     } else {
13070        ShouldNotReachHere();
13071     }
13072   %}
13073   ins_pipe(pipe_jcc);
13074 %}
13075 
13076 // ============================================================================
13077 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13078 // superklass array for an instance of the superklass.  Set a hidden
13079 // internal cache on a hit (cache is checked with exposed code in
13080 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13081 // encoding ALSO sets flags.
13082 
13083 instruct partialSubtypeCheck(rdi_RegP result,
13084                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13085                              rFlagsReg cr)
13086 %{
13087   match(Set result (PartialSubtypeCheck sub super));
13088   effect(KILL rcx, KILL cr);
13089 
13090   ins_cost(1100);  // slightly larger than the next version
13091   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13092             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13093             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13094             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13095             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13096             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13097             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13098     "miss:\t" %}
13099 
13100   opcode(0x1); // Force a XOR of RDI
13101   ins_encode(enc_PartialSubtypeCheck());
13102   ins_pipe(pipe_slow);
13103 %}
13104 
13105 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13106                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13107                                      immP0 zero,
13108                                      rdi_RegP result)
13109 %{
13110   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13111   effect(KILL rcx, KILL result);
13112 
13113   ins_cost(1000);
13114   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13115             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13116             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13117             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13118             "jne,s   miss\t\t# Missed: flags nz\n\t"
13119             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13120     "miss:\t" %}
13121 
13122   opcode(0x0); // No need to XOR RDI
13123   ins_encode(enc_PartialSubtypeCheck());
13124   ins_pipe(pipe_slow);
13125 %}
13126 
13127 // ============================================================================
13128 // Branch Instructions -- short offset versions
13129 //
13130 // These instructions are used to replace jumps of a long offset (the default
13131 // match) with jumps of a shorter offset.  These instructions are all tagged
13132 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13133 // match rules in general matching.  Instead, the ADLC generates a conversion
13134 // method in the MachNode which can be used to do in-place replacement of the
13135 // long variant with the shorter variant.  The compiler will determine if a
13136 // branch can be taken by the is_short_branch_offset() predicate in the machine
13137 // specific code section of the file.
13138 
13139 // Jump Direct - Label defines a relative address from JMP+1
13140 instruct jmpDir_short(label labl) %{
13141   match(Goto);
13142   effect(USE labl);
13143 
13144   ins_cost(300);
13145   format %{ "jmp,s   $labl" %}
13146   size(2);
13147   ins_encode %{
13148     Label* L = $labl$$label;
13149     __ jmpb(*L);
13150   %}
13151   ins_pipe(pipe_jmp);
13152   ins_short_branch(1);
13153 %}
13154 
13155 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13156 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13157   match(If cop cr);
13158   effect(USE labl);
13159 
13160   ins_cost(300);
13161   format %{ "j$cop,s   $labl" %}
13162   size(2);
13163   ins_encode %{
13164     Label* L = $labl$$label;
13165     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13166   %}
13167   ins_pipe(pipe_jcc);
13168   ins_short_branch(1);
13169 %}
13170 
13171 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13172 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13173   match(CountedLoopEnd cop cr);
13174   effect(USE labl);
13175 
13176   ins_cost(300);
13177   format %{ "j$cop,s   $labl\t# loop end" %}
13178   size(2);
13179   ins_encode %{
13180     Label* L = $labl$$label;
13181     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13182   %}
13183   ins_pipe(pipe_jcc);
13184   ins_short_branch(1);
13185 %}
13186 
13187 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13188 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13189   match(CountedLoopEnd cop cmp);
13190   effect(USE labl);
13191 
13192   ins_cost(300);
13193   format %{ "j$cop,us  $labl\t# loop end" %}
13194   size(2);
13195   ins_encode %{
13196     Label* L = $labl$$label;
13197     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13198   %}
13199   ins_pipe(pipe_jcc);
13200   ins_short_branch(1);
13201 %}
13202 
13203 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13204   match(CountedLoopEnd cop cmp);
13205   effect(USE labl);
13206 
13207   ins_cost(300);
13208   format %{ "j$cop,us  $labl\t# loop end" %}
13209   size(2);
13210   ins_encode %{
13211     Label* L = $labl$$label;
13212     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13213   %}
13214   ins_pipe(pipe_jcc);
13215   ins_short_branch(1);
13216 %}
13217 
13218 // Jump Direct Conditional - using unsigned comparison
13219 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13220   match(If cop cmp);
13221   effect(USE labl);
13222 
13223   ins_cost(300);
13224   format %{ "j$cop,us  $labl" %}
13225   size(2);
13226   ins_encode %{
13227     Label* L = $labl$$label;
13228     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13229   %}
13230   ins_pipe(pipe_jcc);
13231   ins_short_branch(1);
13232 %}
13233 
13234 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13235   match(If cop cmp);
13236   effect(USE labl);
13237 
13238   ins_cost(300);
13239   format %{ "j$cop,us  $labl" %}
13240   size(2);
13241   ins_encode %{
13242     Label* L = $labl$$label;
13243     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13244   %}
13245   ins_pipe(pipe_jcc);
13246   ins_short_branch(1);
13247 %}
13248 
13249 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13250   match(If cop cmp);
13251   effect(USE labl);
13252 
13253   ins_cost(300);
13254   format %{ $$template
13255     if ($cop$$cmpcode == Assembler::notEqual) {
13256       $$emit$$"jp,u,s  $labl\n\t"
13257       $$emit$$"j$cop,u,s  $labl"
13258     } else {
13259       $$emit$$"jp,u,s  done\n\t"
13260       $$emit$$"j$cop,u,s  $labl\n\t"
13261       $$emit$$"done:"
13262     }
13263   %}
13264   size(4);
13265   ins_encode %{
13266     Label* l = $labl$$label;
13267     if ($cop$$cmpcode == Assembler::notEqual) {
13268       __ jccb(Assembler::parity, *l);
13269       __ jccb(Assembler::notEqual, *l);
13270     } else if ($cop$$cmpcode == Assembler::equal) {
13271       Label done;
13272       __ jccb(Assembler::parity, done);
13273       __ jccb(Assembler::equal, *l);
13274       __ bind(done);
13275     } else {
13276        ShouldNotReachHere();
13277     }
13278   %}
13279   ins_pipe(pipe_jcc);
13280   ins_short_branch(1);
13281 %}
13282 
13283 // ============================================================================
13284 // inlined locking and unlocking
13285 
13286 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13287   predicate(Compile::current()->use_rtm());
13288   match(Set cr (FastLock object box));
13289   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13290   ins_cost(300);
13291   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13292   ins_encode %{
13293     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13294                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13295                  _rtm_counters, _stack_rtm_counters,
13296                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13297                  true, ra_->C->profile_rtm());
13298   %}
13299   ins_pipe(pipe_slow);
13300 %}
13301 
13302 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13303   predicate(!Compile::current()->use_rtm());
13304   match(Set cr (FastLock object box));
13305   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13306   ins_cost(300);
13307   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13308   ins_encode %{
13309     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13310                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13311   %}
13312   ins_pipe(pipe_slow);
13313 %}
13314 
13315 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13316   match(Set cr (FastUnlock object box));
13317   effect(TEMP tmp, USE_KILL box);
13318   ins_cost(300);
13319   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13320   ins_encode %{
13321     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13322   %}
13323   ins_pipe(pipe_slow);
13324 %}
13325 
13326 
13327 // ============================================================================
13328 // Safepoint Instructions
13329 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13330 %{
13331   match(SafePoint poll);
13332   effect(KILL cr, USE poll);
13333 
13334   format %{ "testl   rax, [$poll]\t"
13335             "# Safepoint: poll for GC" %}
13336   ins_cost(125);
13337   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13338   ins_encode %{
13339     __ relocate(relocInfo::poll_type);
13340     address pre_pc = __ pc();
13341     __ testl(rax, Address($poll$$Register, 0));
13342     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13343   %}
13344   ins_pipe(ialu_reg_mem);
13345 %}
13346 
13347 instruct mask_all_evexL(kReg dst, rRegL src) %{
13348   match(Set dst (MaskAll src));
13349   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13350   ins_encode %{
13351     int mask_len = Matcher::vector_length(this);
13352     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13353   %}
13354   ins_pipe( pipe_slow );
13355 %}
13356 
13357 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13358   predicate(Matcher::vector_length(n) > 32);
13359   match(Set dst (MaskAll src));
13360   effect(TEMP tmp);
13361   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13362   ins_encode %{
13363     int mask_len = Matcher::vector_length(this);
13364     __ movslq($tmp$$Register, $src$$Register);
13365     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13366   %}
13367   ins_pipe( pipe_slow );
13368 %}
13369 
13370 // ============================================================================
13371 // Procedure Call/Return Instructions
13372 // Call Java Static Instruction
13373 // Note: If this code changes, the corresponding ret_addr_offset() and
13374 //       compute_padding() functions will have to be adjusted.
13375 instruct CallStaticJavaDirect(method meth) %{
13376   match(CallStaticJava);
13377   effect(USE meth);
13378 
13379   ins_cost(300);
13380   format %{ "call,static " %}
13381   opcode(0xE8); /* E8 cd */
13382   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13383   ins_pipe(pipe_slow);
13384   ins_alignment(4);
13385 %}
13386 
13387 // Call Java Dynamic Instruction
13388 // Note: If this code changes, the corresponding ret_addr_offset() and
13389 //       compute_padding() functions will have to be adjusted.
13390 instruct CallDynamicJavaDirect(method meth)
13391 %{
13392   match(CallDynamicJava);
13393   effect(USE meth);
13394 
13395   ins_cost(300);
13396   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13397             "call,dynamic " %}
13398   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13399   ins_pipe(pipe_slow);
13400   ins_alignment(4);
13401 %}
13402 
13403 // Call Runtime Instruction
13404 instruct CallRuntimeDirect(method meth)
13405 %{
13406   match(CallRuntime);
13407   effect(USE meth);
13408 
13409   ins_cost(300);
13410   format %{ "call,runtime " %}
13411   ins_encode(clear_avx, Java_To_Runtime(meth));
13412   ins_pipe(pipe_slow);
13413 %}
13414 
13415 // Call runtime without safepoint
13416 instruct CallLeafDirect(method meth)
13417 %{
13418   match(CallLeaf);
13419   effect(USE meth);
13420 
13421   ins_cost(300);
13422   format %{ "call_leaf,runtime " %}
13423   ins_encode(clear_avx, Java_To_Runtime(meth));
13424   ins_pipe(pipe_slow);
13425 %}
13426 
13427 // Call runtime without safepoint and with vector arguments
13428 instruct CallLeafDirectVector(method meth)
13429 %{
13430   match(CallLeafVector);
13431   effect(USE meth);
13432 
13433   ins_cost(300);
13434   format %{ "call_leaf,vector " %}
13435   ins_encode(Java_To_Runtime(meth));
13436   ins_pipe(pipe_slow);
13437 %}
13438 
13439 // Call runtime without safepoint
13440 instruct CallLeafNoFPDirect(method meth)
13441 %{
13442   match(CallLeafNoFP);
13443   effect(USE meth);
13444 
13445   ins_cost(300);
13446   format %{ "call_leaf_nofp,runtime " %}
13447   ins_encode(clear_avx, Java_To_Runtime(meth));
13448   ins_pipe(pipe_slow);
13449 %}
13450 
13451 // Return Instruction
13452 // Remove the return address & jump to it.
13453 // Notice: We always emit a nop after a ret to make sure there is room
13454 // for safepoint patching
13455 instruct Ret()
13456 %{
13457   match(Return);
13458 
13459   format %{ "ret" %}
13460   ins_encode %{
13461     __ ret(0);
13462   %}
13463   ins_pipe(pipe_jmp);
13464 %}
13465 
13466 // Tail Call; Jump from runtime stub to Java code.
13467 // Also known as an 'interprocedural jump'.
13468 // Target of jump will eventually return to caller.
13469 // TailJump below removes the return address.
13470 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13471 %{
13472   match(TailCall jump_target method_ptr);
13473 
13474   ins_cost(300);
13475   format %{ "jmp     $jump_target\t# rbx holds method" %}
13476   ins_encode %{
13477     __ jmp($jump_target$$Register);
13478   %}
13479   ins_pipe(pipe_jmp);
13480 %}
13481 
13482 // Tail Jump; remove the return address; jump to target.
13483 // TailCall above leaves the return address around.
13484 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13485 %{
13486   match(TailJump jump_target ex_oop);
13487 
13488   ins_cost(300);
13489   format %{ "popq    rdx\t# pop return address\n\t"
13490             "jmp     $jump_target" %}
13491   ins_encode %{
13492     __ popq(as_Register(RDX_enc));
13493     __ jmp($jump_target$$Register);
13494   %}
13495   ins_pipe(pipe_jmp);
13496 %}
13497 
13498 // Create exception oop: created by stack-crawling runtime code.
13499 // Created exception is now available to this handler, and is setup
13500 // just prior to jumping to this handler.  No code emitted.
13501 instruct CreateException(rax_RegP ex_oop)
13502 %{
13503   match(Set ex_oop (CreateEx));
13504 
13505   size(0);
13506   // use the following format syntax
13507   format %{ "# exception oop is in rax; no code emitted" %}
13508   ins_encode();
13509   ins_pipe(empty);
13510 %}
13511 
13512 // Rethrow exception:
13513 // The exception oop will come in the first argument position.
13514 // Then JUMP (not call) to the rethrow stub code.
13515 instruct RethrowException()
13516 %{
13517   match(Rethrow);
13518 
13519   // use the following format syntax
13520   format %{ "jmp     rethrow_stub" %}
13521   ins_encode(enc_rethrow);
13522   ins_pipe(pipe_jmp);
13523 %}
13524 
13525 // ============================================================================
13526 // This name is KNOWN by the ADLC and cannot be changed.
13527 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13528 // for this guy.
13529 instruct tlsLoadP(r15_RegP dst) %{
13530   match(Set dst (ThreadLocal));
13531   effect(DEF dst);
13532 
13533   size(0);
13534   format %{ "# TLS is in R15" %}
13535   ins_encode( /*empty encoding*/ );
13536   ins_pipe(ialu_reg_reg);
13537 %}
13538 
13539 
13540 //----------PEEPHOLE RULES-----------------------------------------------------
13541 // These must follow all instruction definitions as they use the names
13542 // defined in the instructions definitions.
13543 //
13544 // peepmatch ( root_instr_name [preceding_instruction]* );
13545 //
13546 // peepconstraint %{
13547 // (instruction_number.operand_name relational_op instruction_number.operand_name
13548 //  [, ...] );
13549 // // instruction numbers are zero-based using left to right order in peepmatch
13550 //
13551 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13552 // // provide an instruction_number.operand_name for each operand that appears
13553 // // in the replacement instruction's match rule
13554 //
13555 // ---------VM FLAGS---------------------------------------------------------
13556 //
13557 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13558 //
13559 // Each peephole rule is given an identifying number starting with zero and
13560 // increasing by one in the order seen by the parser.  An individual peephole
13561 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13562 // on the command-line.
13563 //
13564 // ---------CURRENT LIMITATIONS----------------------------------------------
13565 //
13566 // Only match adjacent instructions in same basic block
13567 // Only equality constraints
13568 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13569 // Only one replacement instruction
13570 //
13571 // ---------EXAMPLE----------------------------------------------------------
13572 //
13573 // // pertinent parts of existing instructions in architecture description
13574 // instruct movI(rRegI dst, rRegI src)
13575 // %{
13576 //   match(Set dst (CopyI src));
13577 // %}
13578 //
13579 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13580 // %{
13581 //   match(Set dst (AddI dst src));
13582 //   effect(KILL cr);
13583 // %}
13584 //
13585 // // Change (inc mov) to lea
13586 // peephole %{
13587 //   // increment preceded by register-register move
13588 //   peepmatch ( incI_rReg movI );
13589 //   // require that the destination register of the increment
13590 //   // match the destination register of the move
13591 //   peepconstraint ( 0.dst == 1.dst );
13592 //   // construct a replacement instruction that sets
13593 //   // the destination to ( move's source register + one )
13594 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13595 // %}
13596 //
13597 
13598 // Implementation no longer uses movX instructions since
13599 // machine-independent system no longer uses CopyX nodes.
13600 //
13601 // peephole
13602 // %{
13603 //   peepmatch (incI_rReg movI);
13604 //   peepconstraint (0.dst == 1.dst);
13605 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13606 // %}
13607 
13608 // peephole
13609 // %{
13610 //   peepmatch (decI_rReg movI);
13611 //   peepconstraint (0.dst == 1.dst);
13612 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13613 // %}
13614 
13615 // peephole
13616 // %{
13617 //   peepmatch (addI_rReg_imm movI);
13618 //   peepconstraint (0.dst == 1.dst);
13619 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13620 // %}
13621 
13622 // peephole
13623 // %{
13624 //   peepmatch (incL_rReg movL);
13625 //   peepconstraint (0.dst == 1.dst);
13626 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13627 // %}
13628 
13629 // peephole
13630 // %{
13631 //   peepmatch (decL_rReg movL);
13632 //   peepconstraint (0.dst == 1.dst);
13633 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13634 // %}
13635 
13636 // peephole
13637 // %{
13638 //   peepmatch (addL_rReg_imm movL);
13639 //   peepconstraint (0.dst == 1.dst);
13640 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13641 // %}
13642 
13643 // peephole
13644 // %{
13645 //   peepmatch (addP_rReg_imm movP);
13646 //   peepconstraint (0.dst == 1.dst);
13647 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13648 // %}
13649 
13650 // // Change load of spilled value to only a spill
13651 // instruct storeI(memory mem, rRegI src)
13652 // %{
13653 //   match(Set mem (StoreI mem src));
13654 // %}
13655 //
13656 // instruct loadI(rRegI dst, memory mem)
13657 // %{
13658 //   match(Set dst (LoadI mem));
13659 // %}
13660 //
13661 
13662 peephole
13663 %{
13664   peepmatch (loadI storeI);
13665   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13666   peepreplace (storeI(1.mem 1.mem 1.src));
13667 %}
13668 
13669 peephole
13670 %{
13671   peepmatch (loadL storeL);
13672   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13673   peepreplace (storeL(1.mem 1.mem 1.src));
13674 %}
13675 
13676 //----------SMARTSPILL RULES---------------------------------------------------
13677 // These must follow all instruction definitions as they use the names
13678 // defined in the instructions definitions.