1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 }
  458 
  459 static bool generate_vzeroupper(Compile* C) {
  460   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  461 }
  462 
  463 static int clear_avx_size() {
  464   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  465 }
  466 
  467 // !!!!! Special hack to get all types of calls to specify the byte offset
  468 //       from the start of the call to the point where the return address
  469 //       will point.
  470 int MachCallStaticJavaNode::ret_addr_offset()
  471 {
  472   int offset = 5; // 5 bytes from start of call to where return address points
  473   offset += clear_avx_size();
  474   return offset;
  475 }
  476 
  477 int MachCallDynamicJavaNode::ret_addr_offset()
  478 {
  479   int offset = 15; // 15 bytes from start of call to where return address points
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 
  484 int MachCallRuntimeNode::ret_addr_offset() {
  485   int offset = 13; // movq r10,#addr; callq (r10)
  486   if (this->ideal_Opcode() != Op_CallLeafVector) {
  487     offset += clear_avx_size();
  488   }
  489   return offset;
  490 }
  491 //
  492 // Compute padding required for nodes which need alignment
  493 //
  494 
  495 // The address of the call instruction needs to be 4-byte aligned to
  496 // ensure that it does not span a cache line so that it can be patched.
  497 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  498 {
  499   current_offset += clear_avx_size(); // skip vzeroupper
  500   current_offset += 1; // skip call opcode byte
  501   return align_up(current_offset, alignment_required()) - current_offset;
  502 }
  503 
  504 // The address of the call instruction needs to be 4-byte aligned to
  505 // ensure that it does not span a cache line so that it can be patched.
  506 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  507 {
  508   current_offset += clear_avx_size(); // skip vzeroupper
  509   current_offset += 11; // skip movq instruction + call opcode byte
  510   return align_up(current_offset, alignment_required()) - current_offset;
  511 }
  512 
  513 // EMIT_RM()
  514 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  515   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  516   cbuf.insts()->emit_int8(c);
  517 }
  518 
  519 // EMIT_CC()
  520 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  521   unsigned char c = (unsigned char) (f1 | f2);
  522   cbuf.insts()->emit_int8(c);
  523 }
  524 
  525 // EMIT_OPCODE()
  526 void emit_opcode(CodeBuffer &cbuf, int code) {
  527   cbuf.insts()->emit_int8((unsigned char) code);
  528 }
  529 
  530 // EMIT_OPCODE() w/ relocation information
  531 void emit_opcode(CodeBuffer &cbuf,
  532                  int code, relocInfo::relocType reloc, int offset, int format)
  533 {
  534   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  535   emit_opcode(cbuf, code);
  536 }
  537 
  538 // EMIT_D8()
  539 void emit_d8(CodeBuffer &cbuf, int d8) {
  540   cbuf.insts()->emit_int8((unsigned char) d8);
  541 }
  542 
  543 // EMIT_D16()
  544 void emit_d16(CodeBuffer &cbuf, int d16) {
  545   cbuf.insts()->emit_int16(d16);
  546 }
  547 
  548 // EMIT_D32()
  549 void emit_d32(CodeBuffer &cbuf, int d32) {
  550   cbuf.insts()->emit_int32(d32);
  551 }
  552 
  553 // EMIT_D64()
  554 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  555   cbuf.insts()->emit_int64(d64);
  556 }
  557 
  558 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  559 void emit_d32_reloc(CodeBuffer& cbuf,
  560                     int d32,
  561                     relocInfo::relocType reloc,
  562                     int format)
  563 {
  564   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  565   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  566   cbuf.insts()->emit_int32(d32);
  567 }
  568 
  569 // emit 32 bit value and construct relocation entry from RelocationHolder
  570 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  571 #ifdef ASSERT
  572   if (rspec.reloc()->type() == relocInfo::oop_type &&
  573       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  574     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  575     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  576   }
  577 #endif
  578   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  579   cbuf.insts()->emit_int32(d32);
  580 }
  581 
  582 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  583   address next_ip = cbuf.insts_end() + 4;
  584   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  585                  external_word_Relocation::spec(addr),
  586                  RELOC_DISP32);
  587 }
  588 
  589 
  590 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  591 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  592   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  593   cbuf.insts()->emit_int64(d64);
  594 }
  595 
  596 // emit 64 bit value and construct relocation entry from RelocationHolder
  597 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  598 #ifdef ASSERT
  599   if (rspec.reloc()->type() == relocInfo::oop_type &&
  600       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  601     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  602     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  603   }
  604 #endif
  605   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  606   cbuf.insts()->emit_int64(d64);
  607 }
  608 
  609 // Access stack slot for load or store
  610 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  611 {
  612   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  613   if (-0x80 <= disp && disp < 0x80) {
  614     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  615     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  616     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  617   } else {
  618     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  619     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  620     emit_d32(cbuf, disp);     // Displacement // R/M byte
  621   }
  622 }
  623 
  624    // rRegI ereg, memory mem) %{    // emit_reg_mem
  625 void encode_RegMem(CodeBuffer &cbuf,
  626                    int reg,
  627                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  628 {
  629   assert(disp_reloc == relocInfo::none, "cannot have disp");
  630   int regenc = reg & 7;
  631   int baseenc = base & 7;
  632   int indexenc = index & 7;
  633 
  634   // There is no index & no scale, use form without SIB byte
  635   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  636     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  637     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  638       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  639     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  640       // If 8-bit displacement, mode 0x1
  641       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  642       emit_d8(cbuf, disp);
  643     } else {
  644       // If 32-bit displacement
  645       if (base == -1) { // Special flag for absolute address
  646         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       } else {
  653         // Normal base + offset
  654         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       }
  661     }
  662   } else {
  663     // Else, encode with the SIB byte
  664     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  665     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  666       // If no displacement
  667       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  668       emit_rm(cbuf, scale, indexenc, baseenc);
  669     } else {
  670       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  671         // If 8-bit displacement, mode 0x1
  672         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  673         emit_rm(cbuf, scale, indexenc, baseenc);
  674         emit_d8(cbuf, disp);
  675       } else {
  676         // If 32-bit displacement
  677         if (base == 0x04 ) {
  678           emit_rm(cbuf, 0x2, regenc, 0x4);
  679           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  680         } else {
  681           emit_rm(cbuf, 0x2, regenc, 0x4);
  682           emit_rm(cbuf, scale, indexenc, baseenc); // *
  683         }
  684         if (disp_reloc != relocInfo::none) {
  685           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  686         } else {
  687           emit_d32(cbuf, disp);
  688         }
  689       }
  690     }
  691   }
  692 }
  693 
  694 // This could be in MacroAssembler but it's fairly C2 specific
  695 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  696   Label exit;
  697   __ jccb(Assembler::noParity, exit);
  698   __ pushf();
  699   //
  700   // comiss/ucomiss instructions set ZF,PF,CF flags and
  701   // zero OF,AF,SF for NaN values.
  702   // Fixup flags by zeroing ZF,PF so that compare of NaN
  703   // values returns 'less than' result (CF is set).
  704   // Leave the rest of flags unchanged.
  705   //
  706   //    7 6 5 4 3 2 1 0
  707   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  708   //    0 0 1 0 1 0 1 1   (0x2B)
  709   //
  710   __ andq(Address(rsp, 0), 0xffffff2b);
  711   __ popf();
  712   __ bind(exit);
  713 }
  714 
  715 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  716   Label done;
  717   __ movl(dst, -1);
  718   __ jcc(Assembler::parity, done);
  719   __ jcc(Assembler::below, done);
  720   __ setb(Assembler::notEqual, dst);
  721   __ movzbl(dst, dst);
  722   __ bind(done);
  723 }
  724 
  725 // Math.min()    # Math.max()
  726 // --------------------------
  727 // ucomis[s/d]   #
  728 // ja   -> b     # a
  729 // jp   -> NaN   # NaN
  730 // jb   -> a     # b
  731 // je            #
  732 // |-jz -> a | b # a & b
  733 // |    -> a     #
  734 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  735                      XMMRegister a, XMMRegister b,
  736                      XMMRegister xmmt, Register rt,
  737                      bool min, bool single) {
  738 
  739   Label nan, zero, below, above, done;
  740 
  741   if (single)
  742     __ ucomiss(a, b);
  743   else
  744     __ ucomisd(a, b);
  745 
  746   if (dst->encoding() != (min ? b : a)->encoding())
  747     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  748   else
  749     __ jccb(Assembler::above, done);
  750 
  751   __ jccb(Assembler::parity, nan);  // PF=1
  752   __ jccb(Assembler::below, below); // CF=1
  753 
  754   // equal
  755   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  756   if (single) {
  757     __ ucomiss(a, xmmt);
  758     __ jccb(Assembler::equal, zero);
  759 
  760     __ movflt(dst, a);
  761     __ jmp(done);
  762   }
  763   else {
  764     __ ucomisd(a, xmmt);
  765     __ jccb(Assembler::equal, zero);
  766 
  767     __ movdbl(dst, a);
  768     __ jmp(done);
  769   }
  770 
  771   __ bind(zero);
  772   if (min)
  773     __ vpor(dst, a, b, Assembler::AVX_128bit);
  774   else
  775     __ vpand(dst, a, b, Assembler::AVX_128bit);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(above);
  780   if (single)
  781     __ movflt(dst, min ? b : a);
  782   else
  783     __ movdbl(dst, min ? b : a);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(nan);
  788   if (single) {
  789     __ movl(rt, 0x7fc00000); // Float.NaN
  790     __ movdl(dst, rt);
  791   }
  792   else {
  793     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  794     __ movdq(dst, rt);
  795   }
  796   __ jmp(done);
  797 
  798   __ bind(below);
  799   if (single)
  800     __ movflt(dst, min ? a : b);
  801   else
  802     __ movdbl(dst, min ? a : b);
  803 
  804   __ bind(done);
  805 }
  806 
  807 //=============================================================================
  808 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  809 
  810 int ConstantTable::calculate_table_base_offset() const {
  811   return 0;  // absolute addressing, no offset
  812 }
  813 
  814 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  815 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  816   ShouldNotReachHere();
  817 }
  818 
  819 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  820   // Empty encoding
  821 }
  822 
  823 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  824   return 0;
  825 }
  826 
  827 #ifndef PRODUCT
  828 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   st->print("# MachConstantBaseNode (empty encoding)");
  830 }
  831 #endif
  832 
  833 
  834 //=============================================================================
  835 #ifndef PRODUCT
  836 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   Compile* C = ra_->C;
  838 
  839   int framesize = C->output()->frame_size_in_bytes();
  840   int bangsize = C->output()->bang_size_in_bytes();
  841   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  842   // Remove wordSize for return addr which is already pushed.
  843   framesize -= wordSize;
  844 
  845   if (C->output()->need_stack_bang(bangsize)) {
  846     framesize -= wordSize;
  847     st->print("# stack bang (%d bytes)", bangsize);
  848     st->print("\n\t");
  849     st->print("pushq   rbp\t# Save rbp");
  850     if (PreserveFramePointer) {
  851         st->print("\n\t");
  852         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  853     }
  854     if (framesize) {
  855       st->print("\n\t");
  856       st->print("subq    rsp, #%d\t# Create frame",framesize);
  857     }
  858   } else {
  859     st->print("subq    rsp, #%d\t# Create frame",framesize);
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  863     if (PreserveFramePointer) {
  864       st->print("\n\t");
  865       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  866       if (framesize > 0) {
  867         st->print("\n\t");
  868         st->print("addq    rbp, #%d", framesize);
  869       }
  870     }
  871   }
  872 
  873   if (VerifyStackAtCalls) {
  874     st->print("\n\t");
  875     framesize -= wordSize;
  876     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  877 #ifdef ASSERT
  878     st->print("\n\t");
  879     st->print("# stack alignment check");
  880 #endif
  881   }
  882   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  883     st->print("\n\t");
  884     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  885     st->print("\n\t");
  886     st->print("je      fast_entry\t");
  887     st->print("\n\t");
  888     st->print("call    #nmethod_entry_barrier_stub\t");
  889     st->print("\n\tfast_entry:");
  890   }
  891   st->cr();
  892 }
  893 #endif
  894 
  895 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  896   Compile* C = ra_->C;
  897   C2_MacroAssembler _masm(&cbuf);
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   int bangsize = C->output()->bang_size_in_bytes();
  901 
  902   if (C->clinit_barrier_on_entry()) {
  903     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  904     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  905 
  906     Label L_skip_barrier;
  907     Register klass = rscratch1;
  908 
  909     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  910     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  911 
  912     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  913 
  914     __ bind(L_skip_barrier);
  915   }
  916 
  917   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  918 
  919   C->output()->set_frame_complete(cbuf.insts_size());
  920 
  921   if (C->has_mach_constant_base_node()) {
  922     // NOTE: We set the table base offset here because users might be
  923     // emitted before MachConstantBaseNode.
  924     ConstantTable& constant_table = C->output()->constant_table();
  925     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  926   }
  927 }
  928 
  929 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  930 {
  931   return MachNode::size(ra_); // too many variables; just compute it
  932                               // the hard way
  933 }
  934 
  935 int MachPrologNode::reloc() const
  936 {
  937   return 0; // a large enough number
  938 }
  939 
  940 //=============================================================================
  941 #ifndef PRODUCT
  942 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  943 {
  944   Compile* C = ra_->C;
  945   if (generate_vzeroupper(C)) {
  946     st->print("vzeroupper");
  947     st->cr(); st->print("\t");
  948   }
  949 
  950   int framesize = C->output()->frame_size_in_bytes();
  951   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  952   // Remove word for return adr already pushed
  953   // and RBP
  954   framesize -= 2*wordSize;
  955 
  956   if (framesize) {
  957     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  958     st->print("\t");
  959   }
  960 
  961   st->print_cr("popq    rbp");
  962   if (do_polling() && C->is_method_compilation()) {
  963     st->print("\t");
  964     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  965                  "ja       #safepoint_stub\t"
  966                  "# Safepoint: poll for GC");
  967   }
  968 }
  969 #endif
  970 
  971 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  972 {
  973   Compile* C = ra_->C;
  974   MacroAssembler _masm(&cbuf);
  975 
  976   if (generate_vzeroupper(C)) {
  977     // Clear upper bits of YMM registers when current compiled code uses
  978     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  979     __ vzeroupper();
  980   }
  981 
  982   int framesize = C->output()->frame_size_in_bytes();
  983   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  984   // Remove word for return adr already pushed
  985   // and RBP
  986   framesize -= 2*wordSize;
  987 
  988   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  989 
  990   if (framesize) {
  991     emit_opcode(cbuf, Assembler::REX_W);
  992     if (framesize < 0x80) {
  993       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  995       emit_d8(cbuf, framesize);
  996     } else {
  997       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  998       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  999       emit_d32(cbuf, framesize);
 1000     }
 1001   }
 1002 
 1003   // popq rbp
 1004   emit_opcode(cbuf, 0x58 | RBP_enc);
 1005 
 1006   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1007     __ reserved_stack_check();
 1008   }
 1009 
 1010   if (do_polling() && C->is_method_compilation()) {
 1011     MacroAssembler _masm(&cbuf);
 1012     Label dummy_label;
 1013     Label* code_stub = &dummy_label;
 1014     if (!C->output()->in_scratch_emit_size()) {
 1015       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1016     }
 1017     __ relocate(relocInfo::poll_return_type);
 1018     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1019   }
 1020 }
 1021 
 1022 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1023 {
 1024   return MachNode::size(ra_); // too many variables; just compute it
 1025                               // the hard way
 1026 }
 1027 
 1028 int MachEpilogNode::reloc() const
 1029 {
 1030   return 2; // a large enough number
 1031 }
 1032 
 1033 const Pipeline* MachEpilogNode::pipeline() const
 1034 {
 1035   return MachNode::pipeline_class();
 1036 }
 1037 
 1038 //=============================================================================
 1039 
 1040 enum RC {
 1041   rc_bad,
 1042   rc_int,
 1043   rc_kreg,
 1044   rc_float,
 1045   rc_stack
 1046 };
 1047 
 1048 static enum RC rc_class(OptoReg::Name reg)
 1049 {
 1050   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1051 
 1052   if (OptoReg::is_stack(reg)) return rc_stack;
 1053 
 1054   VMReg r = OptoReg::as_VMReg(reg);
 1055 
 1056   if (r->is_Register()) return rc_int;
 1057 
 1058   if (r->is_KRegister()) return rc_kreg;
 1059 
 1060   assert(r->is_XMMRegister(), "must be");
 1061   return rc_float;
 1062 }
 1063 
 1064 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1065 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1066                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1067 
 1068 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1069                      int stack_offset, int reg, uint ireg, outputStream* st);
 1070 
 1071 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1072                                       int dst_offset, uint ireg, outputStream* st) {
 1073   if (cbuf) {
 1074     MacroAssembler _masm(cbuf);
 1075     switch (ireg) {
 1076     case Op_VecS:
 1077       __ movq(Address(rsp, -8), rax);
 1078       __ movl(rax, Address(rsp, src_offset));
 1079       __ movl(Address(rsp, dst_offset), rax);
 1080       __ movq(rax, Address(rsp, -8));
 1081       break;
 1082     case Op_VecD:
 1083       __ pushq(Address(rsp, src_offset));
 1084       __ popq (Address(rsp, dst_offset));
 1085       break;
 1086     case Op_VecX:
 1087       __ pushq(Address(rsp, src_offset));
 1088       __ popq (Address(rsp, dst_offset));
 1089       __ pushq(Address(rsp, src_offset+8));
 1090       __ popq (Address(rsp, dst_offset+8));
 1091       break;
 1092     case Op_VecY:
 1093       __ vmovdqu(Address(rsp, -32), xmm0);
 1094       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1095       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1096       __ vmovdqu(xmm0, Address(rsp, -32));
 1097       break;
 1098     case Op_VecZ:
 1099       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1100       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1101       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1102       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1103       break;
 1104     default:
 1105       ShouldNotReachHere();
 1106     }
 1107 #ifndef PRODUCT
 1108   } else {
 1109     switch (ireg) {
 1110     case Op_VecS:
 1111       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1112                 "movl    rax, [rsp + #%d]\n\t"
 1113                 "movl    [rsp + #%d], rax\n\t"
 1114                 "movq    rax, [rsp - #8]",
 1115                 src_offset, dst_offset);
 1116       break;
 1117     case Op_VecD:
 1118       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1119                 "popq    [rsp + #%d]",
 1120                 src_offset, dst_offset);
 1121       break;
 1122      case Op_VecX:
 1123       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1124                 "popq    [rsp + #%d]\n\t"
 1125                 "pushq   [rsp + #%d]\n\t"
 1126                 "popq    [rsp + #%d]",
 1127                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1128       break;
 1129     case Op_VecY:
 1130       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1133                 "vmovdqu xmm0, [rsp - #32]",
 1134                 src_offset, dst_offset);
 1135       break;
 1136     case Op_VecZ:
 1137       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1138                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1139                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1140                 "vmovdqu xmm0, [rsp - #64]",
 1141                 src_offset, dst_offset);
 1142       break;
 1143     default:
 1144       ShouldNotReachHere();
 1145     }
 1146 #endif
 1147   }
 1148 }
 1149 
 1150 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1151                                        PhaseRegAlloc* ra_,
 1152                                        bool do_size,
 1153                                        outputStream* st) const {
 1154   assert(cbuf != NULL || st  != NULL, "sanity");
 1155   // Get registers to move
 1156   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1157   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1158   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1159   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1160 
 1161   enum RC src_second_rc = rc_class(src_second);
 1162   enum RC src_first_rc = rc_class(src_first);
 1163   enum RC dst_second_rc = rc_class(dst_second);
 1164   enum RC dst_first_rc = rc_class(dst_first);
 1165 
 1166   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1167          "must move at least 1 register" );
 1168 
 1169   if (src_first == dst_first && src_second == dst_second) {
 1170     // Self copy, no move
 1171     return 0;
 1172   }
 1173   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1174     uint ireg = ideal_reg();
 1175     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1176     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1177     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1178       // mem -> mem
 1179       int src_offset = ra_->reg2offset(src_first);
 1180       int dst_offset = ra_->reg2offset(dst_first);
 1181       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1182     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1183       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1184     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1185       int stack_offset = ra_->reg2offset(dst_first);
 1186       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1187     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1188       int stack_offset = ra_->reg2offset(src_first);
 1189       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1190     } else {
 1191       ShouldNotReachHere();
 1192     }
 1193     return 0;
 1194   }
 1195   if (src_first_rc == rc_stack) {
 1196     // mem ->
 1197     if (dst_first_rc == rc_stack) {
 1198       // mem -> mem
 1199       assert(src_second != dst_first, "overlap");
 1200       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1201           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1202         // 64-bit
 1203         int src_offset = ra_->reg2offset(src_first);
 1204         int dst_offset = ra_->reg2offset(dst_first);
 1205         if (cbuf) {
 1206           MacroAssembler _masm(cbuf);
 1207           __ pushq(Address(rsp, src_offset));
 1208           __ popq (Address(rsp, dst_offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1212                     "popq    [rsp + #%d]",
 1213                      src_offset, dst_offset);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         // No pushl/popl, so:
 1221         int src_offset = ra_->reg2offset(src_first);
 1222         int dst_offset = ra_->reg2offset(dst_first);
 1223         if (cbuf) {
 1224           MacroAssembler _masm(cbuf);
 1225           __ movq(Address(rsp, -8), rax);
 1226           __ movl(rax, Address(rsp, src_offset));
 1227           __ movl(Address(rsp, dst_offset), rax);
 1228           __ movq(rax, Address(rsp, -8));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1232                     "movl    rax, [rsp + #%d]\n\t"
 1233                     "movl    [rsp + #%d], rax\n\t"
 1234                     "movq    rax, [rsp - #8]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_int) {
 1241       // mem -> gpr
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (cbuf) {
 1247           MacroAssembler _masm(cbuf);
 1248           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1249 #ifndef PRODUCT
 1250         } else {
 1251           st->print("movq    %s, [rsp + #%d]\t# spill",
 1252                      Matcher::regName[dst_first],
 1253                      offset);
 1254 #endif
 1255         }
 1256       } else {
 1257         // 32-bit
 1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1260         int offset = ra_->reg2offset(src_first);
 1261         if (cbuf) {
 1262           MacroAssembler _masm(cbuf);
 1263           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1264 #ifndef PRODUCT
 1265         } else {
 1266           st->print("movl    %s, [rsp + #%d]\t# spill",
 1267                      Matcher::regName[dst_first],
 1268                      offset);
 1269 #endif
 1270         }
 1271       }
 1272       return 0;
 1273     } else if (dst_first_rc == rc_float) {
 1274       // mem-> xmm
 1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1277         // 64-bit
 1278         int offset = ra_->reg2offset(src_first);
 1279         if (cbuf) {
 1280           MacroAssembler _masm(cbuf);
 1281           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("%s  %s, [rsp + #%d]\t# spill",
 1285                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1286                      Matcher::regName[dst_first],
 1287                      offset);
 1288 #endif
 1289         }
 1290       } else {
 1291         // 32-bit
 1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1294         int offset = ra_->reg2offset(src_first);
 1295         if (cbuf) {
 1296           MacroAssembler _masm(cbuf);
 1297           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movss   %s, [rsp + #%d]\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      offset);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_kreg) {
 1308       // mem -> kreg
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         int offset = ra_->reg2offset(src_first);
 1313         if (cbuf) {
 1314           MacroAssembler _masm(cbuf);
 1315           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1316 #ifndef PRODUCT
 1317         } else {
 1318           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1319                      Matcher::regName[dst_first],
 1320                      offset);
 1321 #endif
 1322         }
 1323       }
 1324       return 0;
 1325     }
 1326   } else if (src_first_rc == rc_int) {
 1327     // gpr ->
 1328     if (dst_first_rc == rc_stack) {
 1329       // gpr -> mem
 1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1332         // 64-bit
 1333         int offset = ra_->reg2offset(dst_first);
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("movq    [rsp + #%d], %s\t# spill",
 1340                      offset,
 1341                      Matcher::regName[src_first]);
 1342 #endif
 1343         }
 1344       } else {
 1345         // 32-bit
 1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("movl    [rsp + #%d], %s\t# spill",
 1355                      offset,
 1356                      Matcher::regName[src_first]);
 1357 #endif
 1358         }
 1359       }
 1360       return 0;
 1361     } else if (dst_first_rc == rc_int) {
 1362       // gpr -> gpr
 1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1365         // 64-bit
 1366         if (cbuf) {
 1367           MacroAssembler _masm(cbuf);
 1368           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1369                   as_Register(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movq    %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377         return 0;
 1378       } else {
 1379         // 32-bit
 1380         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1381         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1382         if (cbuf) {
 1383           MacroAssembler _masm(cbuf);
 1384           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1385                   as_Register(Matcher::_regEncode[src_first]));
 1386 #ifndef PRODUCT
 1387         } else {
 1388           st->print("movl    %s, %s\t# spill",
 1389                      Matcher::regName[dst_first],
 1390                      Matcher::regName[src_first]);
 1391 #endif
 1392         }
 1393         return 0;
 1394       }
 1395     } else if (dst_first_rc == rc_float) {
 1396       // gpr -> xmm
 1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1399         // 64-bit
 1400         if (cbuf) {
 1401           MacroAssembler _masm(cbuf);
 1402           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1403 #ifndef PRODUCT
 1404         } else {
 1405           st->print("movdq   %s, %s\t# spill",
 1406                      Matcher::regName[dst_first],
 1407                      Matcher::regName[src_first]);
 1408 #endif
 1409         }
 1410       } else {
 1411         // 32-bit
 1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1414         if (cbuf) {
 1415           MacroAssembler _masm(cbuf);
 1416           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1417 #ifndef PRODUCT
 1418         } else {
 1419           st->print("movdl   %s, %s\t# spill",
 1420                      Matcher::regName[dst_first],
 1421                      Matcher::regName[src_first]);
 1422 #endif
 1423         }
 1424       }
 1425       return 0;
 1426     } else if (dst_first_rc == rc_kreg) {
 1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1429         // 64-bit
 1430         if (cbuf) {
 1431           MacroAssembler _masm(cbuf);
 1432           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1433   #ifndef PRODUCT
 1434         } else {
 1435            st->print("kmovq   %s, %s\t# spill",
 1436                        Matcher::regName[dst_first],
 1437                        Matcher::regName[src_first]);
 1438   #endif
 1439         }
 1440       }
 1441       Unimplemented();
 1442       return 0;
 1443     }
 1444   } else if (src_first_rc == rc_float) {
 1445     // xmm ->
 1446     if (dst_first_rc == rc_stack) {
 1447       // xmm -> mem
 1448       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1449           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1450         // 64-bit
 1451         int offset = ra_->reg2offset(dst_first);
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1455 #ifndef PRODUCT
 1456         } else {
 1457           st->print("movsd   [rsp + #%d], %s\t# spill",
 1458                      offset,
 1459                      Matcher::regName[src_first]);
 1460 #endif
 1461         }
 1462       } else {
 1463         // 32-bit
 1464         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1465         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1466         int offset = ra_->reg2offset(dst_first);
 1467         if (cbuf) {
 1468           MacroAssembler _masm(cbuf);
 1469           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1470 #ifndef PRODUCT
 1471         } else {
 1472           st->print("movss   [rsp + #%d], %s\t# spill",
 1473                      offset,
 1474                      Matcher::regName[src_first]);
 1475 #endif
 1476         }
 1477       }
 1478       return 0;
 1479     } else if (dst_first_rc == rc_int) {
 1480       // xmm -> gpr
 1481       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1482           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1483         // 64-bit
 1484         if (cbuf) {
 1485           MacroAssembler _masm(cbuf);
 1486           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1487 #ifndef PRODUCT
 1488         } else {
 1489           st->print("movdq   %s, %s\t# spill",
 1490                      Matcher::regName[dst_first],
 1491                      Matcher::regName[src_first]);
 1492 #endif
 1493         }
 1494       } else {
 1495         // 32-bit
 1496         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1497         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1498         if (cbuf) {
 1499           MacroAssembler _masm(cbuf);
 1500           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1501 #ifndef PRODUCT
 1502         } else {
 1503           st->print("movdl   %s, %s\t# spill",
 1504                      Matcher::regName[dst_first],
 1505                      Matcher::regName[src_first]);
 1506 #endif
 1507         }
 1508       }
 1509       return 0;
 1510     } else if (dst_first_rc == rc_float) {
 1511       // xmm -> xmm
 1512       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1513           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1514         // 64-bit
 1515         if (cbuf) {
 1516           MacroAssembler _masm(cbuf);
 1517           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1518 #ifndef PRODUCT
 1519         } else {
 1520           st->print("%s  %s, %s\t# spill",
 1521                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1522                      Matcher::regName[dst_first],
 1523                      Matcher::regName[src_first]);
 1524 #endif
 1525         }
 1526       } else {
 1527         // 32-bit
 1528         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1529         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1530         if (cbuf) {
 1531           MacroAssembler _masm(cbuf);
 1532           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1533 #ifndef PRODUCT
 1534         } else {
 1535           st->print("%s  %s, %s\t# spill",
 1536                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1537                      Matcher::regName[dst_first],
 1538                      Matcher::regName[src_first]);
 1539 #endif
 1540         }
 1541       }
 1542       return 0;
 1543     } else if (dst_first_rc == rc_kreg) {
 1544       assert(false, "Illegal spilling");
 1545       return 0;
 1546     }
 1547   } else if (src_first_rc == rc_kreg) {
 1548     if (dst_first_rc == rc_stack) {
 1549       // mem -> kreg
 1550       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1551           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1552         // 64-bit
 1553         int offset = ra_->reg2offset(dst_first);
 1554         if (cbuf) {
 1555           MacroAssembler _masm(cbuf);
 1556           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1557 #ifndef PRODUCT
 1558         } else {
 1559           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1560                      offset,
 1561                      Matcher::regName[src_first]);
 1562 #endif
 1563         }
 1564       }
 1565       return 0;
 1566     } else if (dst_first_rc == rc_int) {
 1567       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1568           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1569         // 64-bit
 1570         if (cbuf) {
 1571           MacroAssembler _masm(cbuf);
 1572           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1573 #ifndef PRODUCT
 1574         } else {
 1575          st->print("kmovq   %s, %s\t# spill",
 1576                      Matcher::regName[dst_first],
 1577                      Matcher::regName[src_first]);
 1578 #endif
 1579         }
 1580       }
 1581       Unimplemented();
 1582       return 0;
 1583     } else if (dst_first_rc == rc_kreg) {
 1584       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1585           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1586         // 64-bit
 1587         if (cbuf) {
 1588           MacroAssembler _masm(cbuf);
 1589           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1590 #ifndef PRODUCT
 1591         } else {
 1592          st->print("kmovq   %s, %s\t# spill",
 1593                      Matcher::regName[dst_first],
 1594                      Matcher::regName[src_first]);
 1595 #endif
 1596         }
 1597       }
 1598       return 0;
 1599     } else if (dst_first_rc == rc_float) {
 1600       assert(false, "Illegal spill");
 1601       return 0;
 1602     }
 1603   }
 1604 
 1605   assert(0," foo ");
 1606   Unimplemented();
 1607   return 0;
 1608 }
 1609 
 1610 #ifndef PRODUCT
 1611 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1612   implementation(NULL, ra_, false, st);
 1613 }
 1614 #endif
 1615 
 1616 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1617   implementation(&cbuf, ra_, false, NULL);
 1618 }
 1619 
 1620 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1621   return MachNode::size(ra_);
 1622 }
 1623 
 1624 //=============================================================================
 1625 #ifndef PRODUCT
 1626 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1627 {
 1628   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1629   int reg = ra_->get_reg_first(this);
 1630   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1631             Matcher::regName[reg], offset);
 1632 }
 1633 #endif
 1634 
 1635 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1636 {
 1637   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1638   int reg = ra_->get_encode(this);
 1639   if (offset >= 0x80) {
 1640     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1641     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1642     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1643     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1644     emit_d32(cbuf, offset);
 1645   } else {
 1646     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1647     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1648     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1649     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1650     emit_d8(cbuf, offset);
 1651   }
 1652 }
 1653 
 1654 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1655 {
 1656   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1657   return (offset < 0x80) ? 5 : 8; // REX
 1658 }
 1659 
 1660 //=============================================================================
 1661 #ifndef PRODUCT
 1662 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1663 {
 1664   if (UseCompressedClassPointers) {
 1665     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1666     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1667     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1668   } else {
 1669     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1670                  "# Inline cache check");
 1671   }
 1672   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1673   st->print_cr("\tnop\t# nops to align entry point");
 1674 }
 1675 #endif
 1676 
 1677 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1678 {
 1679   MacroAssembler masm(&cbuf);
 1680   uint insts_size = cbuf.insts_size();
 1681   if (UseCompressedClassPointers) {
 1682     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1683     masm.cmpptr(rax, rscratch1);
 1684   } else {
 1685     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1686   }
 1687 
 1688   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1689 
 1690   /* WARNING these NOPs are critical so that verified entry point is properly
 1691      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1692   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1693   if (OptoBreakpoint) {
 1694     // Leave space for int3
 1695     nops_cnt -= 1;
 1696   }
 1697   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1698   if (nops_cnt > 0)
 1699     masm.nop(nops_cnt);
 1700 }
 1701 
 1702 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1703 {
 1704   return MachNode::size(ra_); // too many variables; just compute it
 1705                               // the hard way
 1706 }
 1707 
 1708 
 1709 //=============================================================================
 1710 
 1711 const bool Matcher::supports_vector_calling_convention(void) {
 1712   if (EnableVectorSupport && UseVectorStubs) {
 1713     return true;
 1714   }
 1715   return false;
 1716 }
 1717 
 1718 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1719   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1720   int lo = XMM0_num;
 1721   int hi = XMM0b_num;
 1722   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1723   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1724   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1725   return OptoRegPair(hi, lo);
 1726 }
 1727 
 1728 // Is this branch offset short enough that a short branch can be used?
 1729 //
 1730 // NOTE: If the platform does not provide any short branch variants, then
 1731 //       this method should return false for offset 0.
 1732 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1733   // The passed offset is relative to address of the branch.
 1734   // On 86 a branch displacement is calculated relative to address
 1735   // of a next instruction.
 1736   offset -= br_size;
 1737 
 1738   // the short version of jmpConUCF2 contains multiple branches,
 1739   // making the reach slightly less
 1740   if (rule == jmpConUCF2_rule)
 1741     return (-126 <= offset && offset <= 125);
 1742   return (-128 <= offset && offset <= 127);
 1743 }
 1744 
 1745 // Return whether or not this register is ever used as an argument.
 1746 // This function is used on startup to build the trampoline stubs in
 1747 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1748 // call in the trampoline, and arguments in those registers not be
 1749 // available to the callee.
 1750 bool Matcher::can_be_java_arg(int reg)
 1751 {
 1752   return
 1753     reg ==  RDI_num || reg == RDI_H_num ||
 1754     reg ==  RSI_num || reg == RSI_H_num ||
 1755     reg ==  RDX_num || reg == RDX_H_num ||
 1756     reg ==  RCX_num || reg == RCX_H_num ||
 1757     reg ==   R8_num || reg ==  R8_H_num ||
 1758     reg ==   R9_num || reg ==  R9_H_num ||
 1759     reg ==  R12_num || reg == R12_H_num ||
 1760     reg == XMM0_num || reg == XMM0b_num ||
 1761     reg == XMM1_num || reg == XMM1b_num ||
 1762     reg == XMM2_num || reg == XMM2b_num ||
 1763     reg == XMM3_num || reg == XMM3b_num ||
 1764     reg == XMM4_num || reg == XMM4b_num ||
 1765     reg == XMM5_num || reg == XMM5b_num ||
 1766     reg == XMM6_num || reg == XMM6b_num ||
 1767     reg == XMM7_num || reg == XMM7b_num;
 1768 }
 1769 
 1770 bool Matcher::is_spillable_arg(int reg)
 1771 {
 1772   return can_be_java_arg(reg);
 1773 }
 1774 
 1775 uint Matcher::int_pressure_limit()
 1776 {
 1777   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1778 }
 1779 
 1780 uint Matcher::float_pressure_limit()
 1781 {
 1782   // After experiment around with different values, the following default threshold
 1783   // works best for LCM's register pressure scheduling on x64.
 1784   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1785   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1786   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1787 }
 1788 
 1789 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1790   // In 64 bit mode a code which use multiply when
 1791   // devisor is constant is faster than hardware
 1792   // DIV instruction (it uses MulHiL).
 1793   return false;
 1794 }
 1795 
 1796 // Register for DIVI projection of divmodI
 1797 RegMask Matcher::divI_proj_mask() {
 1798   return INT_RAX_REG_mask();
 1799 }
 1800 
 1801 // Register for MODI projection of divmodI
 1802 RegMask Matcher::modI_proj_mask() {
 1803   return INT_RDX_REG_mask();
 1804 }
 1805 
 1806 // Register for DIVL projection of divmodL
 1807 RegMask Matcher::divL_proj_mask() {
 1808   return LONG_RAX_REG_mask();
 1809 }
 1810 
 1811 // Register for MODL projection of divmodL
 1812 RegMask Matcher::modL_proj_mask() {
 1813   return LONG_RDX_REG_mask();
 1814 }
 1815 
 1816 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1817 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1818     return NO_REG_mask();
 1819 }
 1820 
 1821 %}
 1822 
 1823 //----------ENCODING BLOCK-----------------------------------------------------
 1824 // This block specifies the encoding classes used by the compiler to
 1825 // output byte streams.  Encoding classes are parameterized macros
 1826 // used by Machine Instruction Nodes in order to generate the bit
 1827 // encoding of the instruction.  Operands specify their base encoding
 1828 // interface with the interface keyword.  There are currently
 1829 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1830 // COND_INTER.  REG_INTER causes an operand to generate a function
 1831 // which returns its register number when queried.  CONST_INTER causes
 1832 // an operand to generate a function which returns the value of the
 1833 // constant when queried.  MEMORY_INTER causes an operand to generate
 1834 // four functions which return the Base Register, the Index Register,
 1835 // the Scale Value, and the Offset Value of the operand when queried.
 1836 // COND_INTER causes an operand to generate six functions which return
 1837 // the encoding code (ie - encoding bits for the instruction)
 1838 // associated with each basic boolean condition for a conditional
 1839 // instruction.
 1840 //
 1841 // Instructions specify two basic values for encoding.  Again, a
 1842 // function is available to check if the constant displacement is an
 1843 // oop. They use the ins_encode keyword to specify their encoding
 1844 // classes (which must be a sequence of enc_class names, and their
 1845 // parameters, specified in the encoding block), and they use the
 1846 // opcode keyword to specify, in order, their primary, secondary, and
 1847 // tertiary opcode.  Only the opcode sections which a particular
 1848 // instruction needs for encoding need to be specified.
 1849 encode %{
 1850   // Build emit functions for each basic byte or larger field in the
 1851   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1852   // from C++ code in the enc_class source block.  Emit functions will
 1853   // live in the main source block for now.  In future, we can
 1854   // generalize this by adding a syntax that specifies the sizes of
 1855   // fields in an order, so that the adlc can build the emit functions
 1856   // automagically
 1857 
 1858   // Emit primary opcode
 1859   enc_class OpcP
 1860   %{
 1861     emit_opcode(cbuf, $primary);
 1862   %}
 1863 
 1864   // Emit secondary opcode
 1865   enc_class OpcS
 1866   %{
 1867     emit_opcode(cbuf, $secondary);
 1868   %}
 1869 
 1870   // Emit tertiary opcode
 1871   enc_class OpcT
 1872   %{
 1873     emit_opcode(cbuf, $tertiary);
 1874   %}
 1875 
 1876   // Emit opcode directly
 1877   enc_class Opcode(immI d8)
 1878   %{
 1879     emit_opcode(cbuf, $d8$$constant);
 1880   %}
 1881 
 1882   // Emit size prefix
 1883   enc_class SizePrefix
 1884   %{
 1885     emit_opcode(cbuf, 0x66);
 1886   %}
 1887 
 1888   enc_class reg(rRegI reg)
 1889   %{
 1890     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1891   %}
 1892 
 1893   enc_class reg_reg(rRegI dst, rRegI src)
 1894   %{
 1895     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1896   %}
 1897 
 1898   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1899   %{
 1900     emit_opcode(cbuf, $opcode$$constant);
 1901     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1902   %}
 1903 
 1904   enc_class cdql_enc(no_rax_rdx_RegI div)
 1905   %{
 1906     // Full implementation of Java idiv and irem; checks for
 1907     // special case as described in JVM spec., p.243 & p.271.
 1908     //
 1909     //         normal case                           special case
 1910     //
 1911     // input : rax: dividend                         min_int
 1912     //         reg: divisor                          -1
 1913     //
 1914     // output: rax: quotient  (= rax idiv reg)       min_int
 1915     //         rdx: remainder (= rax irem reg)       0
 1916     //
 1917     //  Code sequnce:
 1918     //
 1919     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1920     //    5:   75 07/08                jne    e <normal>
 1921     //    7:   33 d2                   xor    %edx,%edx
 1922     //  [div >= 8 -> offset + 1]
 1923     //  [REX_B]
 1924     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1925     //    c:   74 03/04                je     11 <done>
 1926     // 000000000000000e <normal>:
 1927     //    e:   99                      cltd
 1928     //  [div >= 8 -> offset + 1]
 1929     //  [REX_B]
 1930     //    f:   f7 f9                   idiv   $div
 1931     // 0000000000000011 <done>:
 1932     MacroAssembler _masm(&cbuf);
 1933     Label normal;
 1934     Label done;
 1935 
 1936     // cmp    $0x80000000,%eax
 1937     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1938 
 1939     // jne    e <normal>
 1940     __ jccb(Assembler::notEqual, normal);
 1941 
 1942     // xor    %edx,%edx
 1943     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1944 
 1945     // cmp    $0xffffffffffffffff,%ecx
 1946     __ cmpl($div$$Register, -1);
 1947 
 1948     // je     11 <done>
 1949     __ jccb(Assembler::equal, done);
 1950 
 1951     // <normal>
 1952     // cltd
 1953     __ bind(normal);
 1954     __ cdql();
 1955 
 1956     // idivl
 1957     // <done>
 1958     __ idivl($div$$Register);
 1959     __ bind(done);
 1960   %}
 1961 
 1962   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1963   %{
 1964     // Full implementation of Java ldiv and lrem; checks for
 1965     // special case as described in JVM spec., p.243 & p.271.
 1966     //
 1967     //         normal case                           special case
 1968     //
 1969     // input : rax: dividend                         min_long
 1970     //         reg: divisor                          -1
 1971     //
 1972     // output: rax: quotient  (= rax idiv reg)       min_long
 1973     //         rdx: remainder (= rax irem reg)       0
 1974     //
 1975     //  Code sequnce:
 1976     //
 1977     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1978     //    7:   00 00 80
 1979     //    a:   48 39 d0                cmp    %rdx,%rax
 1980     //    d:   75 08                   jne    17 <normal>
 1981     //    f:   33 d2                   xor    %edx,%edx
 1982     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1983     //   15:   74 05                   je     1c <done>
 1984     // 0000000000000017 <normal>:
 1985     //   17:   48 99                   cqto
 1986     //   19:   48 f7 f9                idiv   $div
 1987     // 000000000000001c <done>:
 1988     MacroAssembler _masm(&cbuf);
 1989     Label normal;
 1990     Label done;
 1991 
 1992     // mov    $0x8000000000000000,%rdx
 1993     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1994 
 1995     // cmp    %rdx,%rax
 1996     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1997 
 1998     // jne    17 <normal>
 1999     __ jccb(Assembler::notEqual, normal);
 2000 
 2001     // xor    %edx,%edx
 2002     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2003 
 2004     // cmp    $0xffffffffffffffff,$div
 2005     __ cmpq($div$$Register, -1);
 2006 
 2007     // je     1e <done>
 2008     __ jccb(Assembler::equal, done);
 2009 
 2010     // <normal>
 2011     // cqto
 2012     __ bind(normal);
 2013     __ cdqq();
 2014 
 2015     // idivq (note: must be emitted by the user of this rule)
 2016     // <done>
 2017     __ idivq($div$$Register);
 2018     __ bind(done);
 2019   %}
 2020 
 2021   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2022   enc_class OpcSE(immI imm)
 2023   %{
 2024     // Emit primary opcode and set sign-extend bit
 2025     // Check for 8-bit immediate, and set sign extend bit in opcode
 2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2027       emit_opcode(cbuf, $primary | 0x02);
 2028     } else {
 2029       // 32-bit immediate
 2030       emit_opcode(cbuf, $primary);
 2031     }
 2032   %}
 2033 
 2034   enc_class OpcSErm(rRegI dst, immI imm)
 2035   %{
 2036     // OpcSEr/m
 2037     int dstenc = $dst$$reg;
 2038     if (dstenc >= 8) {
 2039       emit_opcode(cbuf, Assembler::REX_B);
 2040       dstenc -= 8;
 2041     }
 2042     // Emit primary opcode and set sign-extend bit
 2043     // Check for 8-bit immediate, and set sign extend bit in opcode
 2044     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2045       emit_opcode(cbuf, $primary | 0x02);
 2046     } else {
 2047       // 32-bit immediate
 2048       emit_opcode(cbuf, $primary);
 2049     }
 2050     // Emit r/m byte with secondary opcode, after primary opcode.
 2051     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2052   %}
 2053 
 2054   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2055   %{
 2056     // OpcSEr/m
 2057     int dstenc = $dst$$reg;
 2058     if (dstenc < 8) {
 2059       emit_opcode(cbuf, Assembler::REX_W);
 2060     } else {
 2061       emit_opcode(cbuf, Assembler::REX_WB);
 2062       dstenc -= 8;
 2063     }
 2064     // Emit primary opcode and set sign-extend bit
 2065     // Check for 8-bit immediate, and set sign extend bit in opcode
 2066     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2067       emit_opcode(cbuf, $primary | 0x02);
 2068     } else {
 2069       // 32-bit immediate
 2070       emit_opcode(cbuf, $primary);
 2071     }
 2072     // Emit r/m byte with secondary opcode, after primary opcode.
 2073     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2074   %}
 2075 
 2076   enc_class Con8or32(immI imm)
 2077   %{
 2078     // Check for 8-bit immediate, and set sign extend bit in opcode
 2079     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2080       $$$emit8$imm$$constant;
 2081     } else {
 2082       // 32-bit immediate
 2083       $$$emit32$imm$$constant;
 2084     }
 2085   %}
 2086 
 2087   enc_class opc2_reg(rRegI dst)
 2088   %{
 2089     // BSWAP
 2090     emit_cc(cbuf, $secondary, $dst$$reg);
 2091   %}
 2092 
 2093   enc_class opc3_reg(rRegI dst)
 2094   %{
 2095     // BSWAP
 2096     emit_cc(cbuf, $tertiary, $dst$$reg);
 2097   %}
 2098 
 2099   enc_class reg_opc(rRegI div)
 2100   %{
 2101     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2102     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2103   %}
 2104 
 2105   enc_class enc_cmov(cmpOp cop)
 2106   %{
 2107     // CMOV
 2108     $$$emit8$primary;
 2109     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2110   %}
 2111 
 2112   enc_class enc_PartialSubtypeCheck()
 2113   %{
 2114     Register Rrdi = as_Register(RDI_enc); // result register
 2115     Register Rrax = as_Register(RAX_enc); // super class
 2116     Register Rrcx = as_Register(RCX_enc); // killed
 2117     Register Rrsi = as_Register(RSI_enc); // sub class
 2118     Label miss;
 2119     const bool set_cond_codes = true;
 2120 
 2121     MacroAssembler _masm(&cbuf);
 2122     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2123                                      NULL, &miss,
 2124                                      /*set_cond_codes:*/ true);
 2125     if ($primary) {
 2126       __ xorptr(Rrdi, Rrdi);
 2127     }
 2128     __ bind(miss);
 2129   %}
 2130 
 2131   enc_class clear_avx %{
 2132     debug_only(int off0 = cbuf.insts_size());
 2133     if (generate_vzeroupper(Compile::current())) {
 2134       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2135       // Clear upper bits of YMM registers when current compiled code uses
 2136       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2137       MacroAssembler _masm(&cbuf);
 2138       __ vzeroupper();
 2139     }
 2140     debug_only(int off1 = cbuf.insts_size());
 2141     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2142   %}
 2143 
 2144   enc_class Java_To_Runtime(method meth) %{
 2145     // No relocation needed
 2146     MacroAssembler _masm(&cbuf);
 2147     __ mov64(r10, (int64_t) $meth$$method);
 2148     __ call(r10);
 2149     __ post_call_nop();
 2150   %}
 2151 
 2152   enc_class Java_Static_Call(method meth)
 2153   %{
 2154     // JAVA STATIC CALL
 2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2156     // determine who we intended to call.
 2157     MacroAssembler _masm(&cbuf);
 2158     cbuf.set_insts_mark();
 2159     $$$emit8$primary;
 2160 
 2161     if (!_method) {
 2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2163                      runtime_call_Relocation::spec(),
 2164                      RELOC_DISP32);
 2165     } else {
 2166       int method_index = resolved_method_index(cbuf);
 2167       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2168                                                   : static_call_Relocation::spec(method_index);
 2169       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2170                      rspec, RELOC_DISP32);
 2171       address mark = cbuf.insts_mark();
 2172       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2173         // Calls of the same statically bound method can share
 2174         // a stub to the interpreter.
 2175         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2176       } else {
 2177         // Emit stubs for static call.
 2178         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2179         if (stub == NULL) {
 2180           ciEnv::current()->record_failure("CodeCache is full");
 2181           return;
 2182         }
 2183       }
 2184     }
 2185     _masm.clear_inst_mark();
 2186     __ post_call_nop();
 2187   %}
 2188 
 2189   enc_class Java_Dynamic_Call(method meth) %{
 2190     MacroAssembler _masm(&cbuf);
 2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2192     __ post_call_nop();
 2193   %}
 2194 
 2195   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2196   %{
 2197     // SAL, SAR, SHR
 2198     int dstenc = $dst$$reg;
 2199     if (dstenc >= 8) {
 2200       emit_opcode(cbuf, Assembler::REX_B);
 2201       dstenc -= 8;
 2202     }
 2203     $$$emit8$primary;
 2204     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2205     $$$emit8$shift$$constant;
 2206   %}
 2207 
 2208   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2209   %{
 2210     // SAL, SAR, SHR
 2211     int dstenc = $dst$$reg;
 2212     if (dstenc < 8) {
 2213       emit_opcode(cbuf, Assembler::REX_W);
 2214     } else {
 2215       emit_opcode(cbuf, Assembler::REX_WB);
 2216       dstenc -= 8;
 2217     }
 2218     $$$emit8$primary;
 2219     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2220     $$$emit8$shift$$constant;
 2221   %}
 2222 
 2223   enc_class load_immI(rRegI dst, immI src)
 2224   %{
 2225     int dstenc = $dst$$reg;
 2226     if (dstenc >= 8) {
 2227       emit_opcode(cbuf, Assembler::REX_B);
 2228       dstenc -= 8;
 2229     }
 2230     emit_opcode(cbuf, 0xB8 | dstenc);
 2231     $$$emit32$src$$constant;
 2232   %}
 2233 
 2234   enc_class load_immL(rRegL dst, immL src)
 2235   %{
 2236     int dstenc = $dst$$reg;
 2237     if (dstenc < 8) {
 2238       emit_opcode(cbuf, Assembler::REX_W);
 2239     } else {
 2240       emit_opcode(cbuf, Assembler::REX_WB);
 2241       dstenc -= 8;
 2242     }
 2243     emit_opcode(cbuf, 0xB8 | dstenc);
 2244     emit_d64(cbuf, $src$$constant);
 2245   %}
 2246 
 2247   enc_class load_immUL32(rRegL dst, immUL32 src)
 2248   %{
 2249     // same as load_immI, but this time we care about zeroes in the high word
 2250     int dstenc = $dst$$reg;
 2251     if (dstenc >= 8) {
 2252       emit_opcode(cbuf, Assembler::REX_B);
 2253       dstenc -= 8;
 2254     }
 2255     emit_opcode(cbuf, 0xB8 | dstenc);
 2256     $$$emit32$src$$constant;
 2257   %}
 2258 
 2259   enc_class load_immL32(rRegL dst, immL32 src)
 2260   %{
 2261     int dstenc = $dst$$reg;
 2262     if (dstenc < 8) {
 2263       emit_opcode(cbuf, Assembler::REX_W);
 2264     } else {
 2265       emit_opcode(cbuf, Assembler::REX_WB);
 2266       dstenc -= 8;
 2267     }
 2268     emit_opcode(cbuf, 0xC7);
 2269     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2270     $$$emit32$src$$constant;
 2271   %}
 2272 
 2273   enc_class load_immP31(rRegP dst, immP32 src)
 2274   %{
 2275     // same as load_immI, but this time we care about zeroes in the high word
 2276     int dstenc = $dst$$reg;
 2277     if (dstenc >= 8) {
 2278       emit_opcode(cbuf, Assembler::REX_B);
 2279       dstenc -= 8;
 2280     }
 2281     emit_opcode(cbuf, 0xB8 | dstenc);
 2282     $$$emit32$src$$constant;
 2283   %}
 2284 
 2285   enc_class load_immP(rRegP dst, immP src)
 2286   %{
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc < 8) {
 2289       emit_opcode(cbuf, Assembler::REX_W);
 2290     } else {
 2291       emit_opcode(cbuf, Assembler::REX_WB);
 2292       dstenc -= 8;
 2293     }
 2294     emit_opcode(cbuf, 0xB8 | dstenc);
 2295     // This next line should be generated from ADLC
 2296     if ($src->constant_reloc() != relocInfo::none) {
 2297       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2298     } else {
 2299       emit_d64(cbuf, $src$$constant);
 2300     }
 2301   %}
 2302 
 2303   enc_class Con32(immI src)
 2304   %{
 2305     // Output immediate
 2306     $$$emit32$src$$constant;
 2307   %}
 2308 
 2309   enc_class Con32F_as_bits(immF src)
 2310   %{
 2311     // Output Float immediate bits
 2312     jfloat jf = $src$$constant;
 2313     jint jf_as_bits = jint_cast(jf);
 2314     emit_d32(cbuf, jf_as_bits);
 2315   %}
 2316 
 2317   enc_class Con16(immI src)
 2318   %{
 2319     // Output immediate
 2320     $$$emit16$src$$constant;
 2321   %}
 2322 
 2323   // How is this different from Con32??? XXX
 2324   enc_class Con_d32(immI src)
 2325   %{
 2326     emit_d32(cbuf,$src$$constant);
 2327   %}
 2328 
 2329   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2330     // Output immediate memory reference
 2331     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2332     emit_d32(cbuf, 0x00);
 2333   %}
 2334 
 2335   enc_class lock_prefix()
 2336   %{
 2337     emit_opcode(cbuf, 0xF0); // lock
 2338   %}
 2339 
 2340   enc_class REX_mem(memory mem)
 2341   %{
 2342     if ($mem$$base >= 8) {
 2343       if ($mem$$index < 8) {
 2344         emit_opcode(cbuf, Assembler::REX_B);
 2345       } else {
 2346         emit_opcode(cbuf, Assembler::REX_XB);
 2347       }
 2348     } else {
 2349       if ($mem$$index >= 8) {
 2350         emit_opcode(cbuf, Assembler::REX_X);
 2351       }
 2352     }
 2353   %}
 2354 
 2355   enc_class REX_mem_wide(memory mem)
 2356   %{
 2357     if ($mem$$base >= 8) {
 2358       if ($mem$$index < 8) {
 2359         emit_opcode(cbuf, Assembler::REX_WB);
 2360       } else {
 2361         emit_opcode(cbuf, Assembler::REX_WXB);
 2362       }
 2363     } else {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_W);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_WX);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   // for byte regs
 2373   enc_class REX_breg(rRegI reg)
 2374   %{
 2375     if ($reg$$reg >= 4) {
 2376       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2382   %{
 2383     if ($dst$$reg < 8) {
 2384       if ($src$$reg >= 4) {
 2385         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2386       }
 2387     } else {
 2388       if ($src$$reg < 8) {
 2389         emit_opcode(cbuf, Assembler::REX_R);
 2390       } else {
 2391         emit_opcode(cbuf, Assembler::REX_RB);
 2392       }
 2393     }
 2394   %}
 2395 
 2396   // for byte regs
 2397   enc_class REX_breg_mem(rRegI reg, memory mem)
 2398   %{
 2399     if ($reg$$reg < 8) {
 2400       if ($mem$$base < 8) {
 2401         if ($mem$$index >= 8) {
 2402           emit_opcode(cbuf, Assembler::REX_X);
 2403         } else if ($reg$$reg >= 4) {
 2404           emit_opcode(cbuf, Assembler::REX);
 2405         }
 2406       } else {
 2407         if ($mem$$index < 8) {
 2408           emit_opcode(cbuf, Assembler::REX_B);
 2409         } else {
 2410           emit_opcode(cbuf, Assembler::REX_XB);
 2411         }
 2412       }
 2413     } else {
 2414       if ($mem$$base < 8) {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_R);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_RX);
 2419         }
 2420       } else {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_RB);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RXB);
 2425         }
 2426       }
 2427     }
 2428   %}
 2429 
 2430   enc_class REX_reg(rRegI reg)
 2431   %{
 2432     if ($reg$$reg >= 8) {
 2433       emit_opcode(cbuf, Assembler::REX_B);
 2434     }
 2435   %}
 2436 
 2437   enc_class REX_reg_wide(rRegI reg)
 2438   %{
 2439     if ($reg$$reg < 8) {
 2440       emit_opcode(cbuf, Assembler::REX_W);
 2441     } else {
 2442       emit_opcode(cbuf, Assembler::REX_WB);
 2443     }
 2444   %}
 2445 
 2446   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2447   %{
 2448     if ($dst$$reg < 8) {
 2449       if ($src$$reg >= 8) {
 2450         emit_opcode(cbuf, Assembler::REX_B);
 2451       }
 2452     } else {
 2453       if ($src$$reg < 8) {
 2454         emit_opcode(cbuf, Assembler::REX_R);
 2455       } else {
 2456         emit_opcode(cbuf, Assembler::REX_RB);
 2457       }
 2458     }
 2459   %}
 2460 
 2461   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2462   %{
 2463     if ($dst$$reg < 8) {
 2464       if ($src$$reg < 8) {
 2465         emit_opcode(cbuf, Assembler::REX_W);
 2466       } else {
 2467         emit_opcode(cbuf, Assembler::REX_WB);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_WR);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_WRB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_mem(rRegI reg, memory mem)
 2479   %{
 2480     if ($reg$$reg < 8) {
 2481       if ($mem$$base < 8) {
 2482         if ($mem$$index >= 8) {
 2483           emit_opcode(cbuf, Assembler::REX_X);
 2484         }
 2485       } else {
 2486         if ($mem$$index < 8) {
 2487           emit_opcode(cbuf, Assembler::REX_B);
 2488         } else {
 2489           emit_opcode(cbuf, Assembler::REX_XB);
 2490         }
 2491       }
 2492     } else {
 2493       if ($mem$$base < 8) {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_R);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_RX);
 2498         }
 2499       } else {
 2500         if ($mem$$index < 8) {
 2501           emit_opcode(cbuf, Assembler::REX_RB);
 2502         } else {
 2503           emit_opcode(cbuf, Assembler::REX_RXB);
 2504         }
 2505       }
 2506     }
 2507   %}
 2508 
 2509   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2510   %{
 2511     if ($reg$$reg < 8) {
 2512       if ($mem$$base < 8) {
 2513         if ($mem$$index < 8) {
 2514           emit_opcode(cbuf, Assembler::REX_W);
 2515         } else {
 2516           emit_opcode(cbuf, Assembler::REX_WX);
 2517         }
 2518       } else {
 2519         if ($mem$$index < 8) {
 2520           emit_opcode(cbuf, Assembler::REX_WB);
 2521         } else {
 2522           emit_opcode(cbuf, Assembler::REX_WXB);
 2523         }
 2524       }
 2525     } else {
 2526       if ($mem$$base < 8) {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WR);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WRX);
 2531         }
 2532       } else {
 2533         if ($mem$$index < 8) {
 2534           emit_opcode(cbuf, Assembler::REX_WRB);
 2535         } else {
 2536           emit_opcode(cbuf, Assembler::REX_WRXB);
 2537         }
 2538       }
 2539     }
 2540   %}
 2541 
 2542   enc_class reg_mem(rRegI ereg, memory mem)
 2543   %{
 2544     // High registers handle in encode_RegMem
 2545     int reg = $ereg$$reg;
 2546     int base = $mem$$base;
 2547     int index = $mem$$index;
 2548     int scale = $mem$$scale;
 2549     int disp = $mem$$disp;
 2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2551 
 2552     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2553   %}
 2554 
 2555   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2556   %{
 2557     int rm_byte_opcode = $rm_opcode$$constant;
 2558 
 2559     // High registers handle in encode_RegMem
 2560     int base = $mem$$base;
 2561     int index = $mem$$index;
 2562     int scale = $mem$$scale;
 2563     int displace = $mem$$disp;
 2564 
 2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2566                                             // working with static
 2567                                             // globals
 2568     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2569                   disp_reloc);
 2570   %}
 2571 
 2572   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2573   %{
 2574     int reg_encoding = $dst$$reg;
 2575     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2576     int index        = 0x04;            // 0x04 indicates no index
 2577     int scale        = 0x00;            // 0x00 indicates no scale
 2578     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2579     relocInfo::relocType disp_reloc = relocInfo::none;
 2580     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2581                   disp_reloc);
 2582   %}
 2583 
 2584   enc_class neg_reg(rRegI dst)
 2585   %{
 2586     int dstenc = $dst$$reg;
 2587     if (dstenc >= 8) {
 2588       emit_opcode(cbuf, Assembler::REX_B);
 2589       dstenc -= 8;
 2590     }
 2591     // NEG $dst
 2592     emit_opcode(cbuf, 0xF7);
 2593     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2594   %}
 2595 
 2596   enc_class neg_reg_wide(rRegI dst)
 2597   %{
 2598     int dstenc = $dst$$reg;
 2599     if (dstenc < 8) {
 2600       emit_opcode(cbuf, Assembler::REX_W);
 2601     } else {
 2602       emit_opcode(cbuf, Assembler::REX_WB);
 2603       dstenc -= 8;
 2604     }
 2605     // NEG $dst
 2606     emit_opcode(cbuf, 0xF7);
 2607     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2608   %}
 2609 
 2610   enc_class setLT_reg(rRegI dst)
 2611   %{
 2612     int dstenc = $dst$$reg;
 2613     if (dstenc >= 8) {
 2614       emit_opcode(cbuf, Assembler::REX_B);
 2615       dstenc -= 8;
 2616     } else if (dstenc >= 4) {
 2617       emit_opcode(cbuf, Assembler::REX);
 2618     }
 2619     // SETLT $dst
 2620     emit_opcode(cbuf, 0x0F);
 2621     emit_opcode(cbuf, 0x9C);
 2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2623   %}
 2624 
 2625   enc_class setNZ_reg(rRegI dst)
 2626   %{
 2627     int dstenc = $dst$$reg;
 2628     if (dstenc >= 8) {
 2629       emit_opcode(cbuf, Assembler::REX_B);
 2630       dstenc -= 8;
 2631     } else if (dstenc >= 4) {
 2632       emit_opcode(cbuf, Assembler::REX);
 2633     }
 2634     // SETNZ $dst
 2635     emit_opcode(cbuf, 0x0F);
 2636     emit_opcode(cbuf, 0x95);
 2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2638   %}
 2639 
 2640 
 2641   // Compare the lonogs and set -1, 0, or 1 into dst
 2642   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2643   %{
 2644     int src1enc = $src1$$reg;
 2645     int src2enc = $src2$$reg;
 2646     int dstenc = $dst$$reg;
 2647 
 2648     // cmpq $src1, $src2
 2649     if (src1enc < 8) {
 2650       if (src2enc < 8) {
 2651         emit_opcode(cbuf, Assembler::REX_W);
 2652       } else {
 2653         emit_opcode(cbuf, Assembler::REX_WB);
 2654       }
 2655     } else {
 2656       if (src2enc < 8) {
 2657         emit_opcode(cbuf, Assembler::REX_WR);
 2658       } else {
 2659         emit_opcode(cbuf, Assembler::REX_WRB);
 2660       }
 2661     }
 2662     emit_opcode(cbuf, 0x3B);
 2663     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2664 
 2665     // movl $dst, -1
 2666     if (dstenc >= 8) {
 2667       emit_opcode(cbuf, Assembler::REX_B);
 2668     }
 2669     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2670     emit_d32(cbuf, -1);
 2671 
 2672     // jl,s done
 2673     emit_opcode(cbuf, 0x7C);
 2674     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2675 
 2676     // setne $dst
 2677     if (dstenc >= 4) {
 2678       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2679     }
 2680     emit_opcode(cbuf, 0x0F);
 2681     emit_opcode(cbuf, 0x95);
 2682     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2683 
 2684     // movzbl $dst, $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0xB6);
 2690     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2691   %}
 2692 
 2693   enc_class Push_ResultXD(regD dst) %{
 2694     MacroAssembler _masm(&cbuf);
 2695     __ fstp_d(Address(rsp, 0));
 2696     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2697     __ addptr(rsp, 8);
 2698   %}
 2699 
 2700   enc_class Push_SrcXD(regD src) %{
 2701     MacroAssembler _masm(&cbuf);
 2702     __ subptr(rsp, 8);
 2703     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2704     __ fld_d(Address(rsp, 0));
 2705   %}
 2706 
 2707 
 2708   enc_class enc_rethrow()
 2709   %{
 2710     cbuf.set_insts_mark();
 2711     emit_opcode(cbuf, 0xE9); // jmp entry
 2712     emit_d32_reloc(cbuf,
 2713                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2714                    runtime_call_Relocation::spec(),
 2715                    RELOC_DISP32);
 2716   %}
 2717 
 2718 %}
 2719 
 2720 
 2721 
 2722 //----------FRAME--------------------------------------------------------------
 2723 // Definition of frame structure and management information.
 2724 //
 2725 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2726 //                             |   (to get allocators register number
 2727 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2728 //  r   CALLER     |        |
 2729 //  o     |        +--------+      pad to even-align allocators stack-slot
 2730 //  w     V        |  pad0  |        numbers; owned by CALLER
 2731 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2732 //  h     ^        |   in   |  5
 2733 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2734 //  |     |        |        |  3
 2735 //  |     |        +--------+
 2736 //  V     |        | old out|      Empty on Intel, window on Sparc
 2737 //        |    old |preserve|      Must be even aligned.
 2738 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2739 //        |        |   in   |  3   area for Intel ret address
 2740 //     Owned by    |preserve|      Empty on Sparc.
 2741 //       SELF      +--------+
 2742 //        |        |  pad2  |  2   pad to align old SP
 2743 //        |        +--------+  1
 2744 //        |        | locks  |  0
 2745 //        |        +--------+----> OptoReg::stack0(), even aligned
 2746 //        |        |  pad1  | 11   pad to align new SP
 2747 //        |        +--------+
 2748 //        |        |        | 10
 2749 //        |        | spills |  9   spills
 2750 //        V        |        |  8   (pad0 slot for callee)
 2751 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2752 //        ^        |  out   |  7
 2753 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2754 //     Owned by    +--------+
 2755 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2756 //        |    new |preserve|      Must be even-aligned.
 2757 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2758 //        |        |        |
 2759 //
 2760 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2761 //         known from SELF's arguments and the Java calling convention.
 2762 //         Region 6-7 is determined per call site.
 2763 // Note 2: If the calling convention leaves holes in the incoming argument
 2764 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2765 //         are owned by the CALLEE.  Holes should not be necessary in the
 2766 //         incoming area, as the Java calling convention is completely under
 2767 //         the control of the AD file.  Doubles can be sorted and packed to
 2768 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2769 //         varargs C calling conventions.
 2770 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2771 //         even aligned with pad0 as needed.
 2772 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2773 //         region 6-11 is even aligned; it may be padded out more so that
 2774 //         the region from SP to FP meets the minimum stack alignment.
 2775 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2776 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2777 //         SP meets the minimum alignment.
 2778 
 2779 frame
 2780 %{
 2781   // These three registers define part of the calling convention
 2782   // between compiled code and the interpreter.
 2783   inline_cache_reg(RAX);                // Inline Cache Register
 2784 
 2785   // Optional: name the operand used by cisc-spilling to access
 2786   // [stack_pointer + offset]
 2787   cisc_spilling_operand_name(indOffset32);
 2788 
 2789   // Number of stack slots consumed by locking an object
 2790   sync_stack_slots(2);
 2791 
 2792   // Compiled code's Frame Pointer
 2793   frame_pointer(RSP);
 2794 
 2795   // Interpreter stores its frame pointer in a register which is
 2796   // stored to the stack by I2CAdaptors.
 2797   // I2CAdaptors convert from interpreted java to compiled java.
 2798   interpreter_frame_pointer(RBP);
 2799 
 2800   // Stack alignment requirement
 2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2802 
 2803   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2804   // for calls to C.  Supports the var-args backing area for register parms.
 2805   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2806 
 2807   // The after-PROLOG location of the return address.  Location of
 2808   // return address specifies a type (REG or STACK) and a number
 2809   // representing the register number (i.e. - use a register name) or
 2810   // stack slot.
 2811   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2812   // Otherwise, it is above the locks and verification slot and alignment word
 2813   return_addr(STACK - 2 +
 2814               align_up((Compile::current()->in_preserve_stack_slots() +
 2815                         Compile::current()->fixed_slots()),
 2816                        stack_alignment_in_slots()));
 2817 
 2818   // Location of compiled Java return values.  Same as C for now.
 2819   return_value
 2820   %{
 2821     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2822            "only return normal values");
 2823 
 2824     static const int lo[Op_RegL + 1] = {
 2825       0,
 2826       0,
 2827       RAX_num,  // Op_RegN
 2828       RAX_num,  // Op_RegI
 2829       RAX_num,  // Op_RegP
 2830       XMM0_num, // Op_RegF
 2831       XMM0_num, // Op_RegD
 2832       RAX_num   // Op_RegL
 2833     };
 2834     static const int hi[Op_RegL + 1] = {
 2835       0,
 2836       0,
 2837       OptoReg::Bad, // Op_RegN
 2838       OptoReg::Bad, // Op_RegI
 2839       RAX_H_num,    // Op_RegP
 2840       OptoReg::Bad, // Op_RegF
 2841       XMM0b_num,    // Op_RegD
 2842       RAX_H_num     // Op_RegL
 2843     };
 2844     // Excluded flags and vector registers.
 2845     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2846     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2847   %}
 2848 %}
 2849 
 2850 //----------ATTRIBUTES---------------------------------------------------------
 2851 //----------Operand Attributes-------------------------------------------------
 2852 op_attrib op_cost(0);        // Required cost attribute
 2853 
 2854 //----------Instruction Attributes---------------------------------------------
 2855 ins_attrib ins_cost(100);       // Required cost attribute
 2856 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2857 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2858                                 // a non-matching short branch variant
 2859                                 // of some long branch?
 2860 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2861                                 // be a power of 2) specifies the
 2862                                 // alignment that some part of the
 2863                                 // instruction (not necessarily the
 2864                                 // start) requires.  If > 1, a
 2865                                 // compute_padding() function must be
 2866                                 // provided for the instruction
 2867 
 2868 //----------OPERANDS-----------------------------------------------------------
 2869 // Operand definitions must precede instruction definitions for correct parsing
 2870 // in the ADLC because operands constitute user defined types which are used in
 2871 // instruction definitions.
 2872 
 2873 //----------Simple Operands----------------------------------------------------
 2874 // Immediate Operands
 2875 // Integer Immediate
 2876 operand immI()
 2877 %{
 2878   match(ConI);
 2879 
 2880   op_cost(10);
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 // Constant for test vs zero
 2886 operand immI_0()
 2887 %{
 2888   predicate(n->get_int() == 0);
 2889   match(ConI);
 2890 
 2891   op_cost(0);
 2892   format %{ %}
 2893   interface(CONST_INTER);
 2894 %}
 2895 
 2896 // Constant for increment
 2897 operand immI_1()
 2898 %{
 2899   predicate(n->get_int() == 1);
 2900   match(ConI);
 2901 
 2902   op_cost(0);
 2903   format %{ %}
 2904   interface(CONST_INTER);
 2905 %}
 2906 
 2907 // Constant for decrement
 2908 operand immI_M1()
 2909 %{
 2910   predicate(n->get_int() == -1);
 2911   match(ConI);
 2912 
 2913   op_cost(0);
 2914   format %{ %}
 2915   interface(CONST_INTER);
 2916 %}
 2917 
 2918 operand immI_2()
 2919 %{
 2920   predicate(n->get_int() == 2);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 operand immI_4()
 2929 %{
 2930   predicate(n->get_int() == 4);
 2931   match(ConI);
 2932 
 2933   op_cost(0);
 2934   format %{ %}
 2935   interface(CONST_INTER);
 2936 %}
 2937 
 2938 operand immI_8()
 2939 %{
 2940   predicate(n->get_int() == 8);
 2941   match(ConI);
 2942 
 2943   op_cost(0);
 2944   format %{ %}
 2945   interface(CONST_INTER);
 2946 %}
 2947 
 2948 // Valid scale values for addressing modes
 2949 operand immI2()
 2950 %{
 2951   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2952   match(ConI);
 2953 
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 operand immU7()
 2959 %{
 2960   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2961   match(ConI);
 2962 
 2963   op_cost(5);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 operand immI8()
 2969 %{
 2970   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2971   match(ConI);
 2972 
 2973   op_cost(5);
 2974   format %{ %}
 2975   interface(CONST_INTER);
 2976 %}
 2977 
 2978 operand immU8()
 2979 %{
 2980   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2981   match(ConI);
 2982 
 2983   op_cost(5);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immI16()
 2989 %{
 2990   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2991   match(ConI);
 2992 
 2993   op_cost(10);
 2994   format %{ %}
 2995   interface(CONST_INTER);
 2996 %}
 2997 
 2998 // Int Immediate non-negative
 2999 operand immU31()
 3000 %{
 3001   predicate(n->get_int() >= 0);
 3002   match(ConI);
 3003 
 3004   op_cost(0);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 // Constant for long shifts
 3010 operand immI_32()
 3011 %{
 3012   predicate( n->get_int() == 32 );
 3013   match(ConI);
 3014 
 3015   op_cost(0);
 3016   format %{ %}
 3017   interface(CONST_INTER);
 3018 %}
 3019 
 3020 // Constant for long shifts
 3021 operand immI_64()
 3022 %{
 3023   predicate( n->get_int() == 64 );
 3024   match(ConI);
 3025 
 3026   op_cost(0);
 3027   format %{ %}
 3028   interface(CONST_INTER);
 3029 %}
 3030 
 3031 // Pointer Immediate
 3032 operand immP()
 3033 %{
 3034   match(ConP);
 3035 
 3036   op_cost(10);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // NULL Pointer Immediate
 3042 operand immP0()
 3043 %{
 3044   predicate(n->get_ptr() == 0);
 3045   match(ConP);
 3046 
 3047   op_cost(5);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immN() %{
 3054   match(ConN);
 3055 
 3056   op_cost(10);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immNKlass() %{
 3062   match(ConNKlass);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // NULL Pointer Immediate
 3070 operand immN0() %{
 3071   predicate(n->get_narrowcon() == 0);
 3072   match(ConN);
 3073 
 3074   op_cost(5);
 3075   format %{ %}
 3076   interface(CONST_INTER);
 3077 %}
 3078 
 3079 operand immP31()
 3080 %{
 3081   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3082             && (n->get_ptr() >> 31) == 0);
 3083   match(ConP);
 3084 
 3085   op_cost(5);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 
 3091 // Long Immediate
 3092 operand immL()
 3093 %{
 3094   match(ConL);
 3095 
 3096   op_cost(20);
 3097   format %{ %}
 3098   interface(CONST_INTER);
 3099 %}
 3100 
 3101 // Long Immediate 8-bit
 3102 operand immL8()
 3103 %{
 3104   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3105   match(ConL);
 3106 
 3107   op_cost(5);
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate 32-bit unsigned
 3113 operand immUL32()
 3114 %{
 3115   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3116   match(ConL);
 3117 
 3118   op_cost(10);
 3119   format %{ %}
 3120   interface(CONST_INTER);
 3121 %}
 3122 
 3123 // Long Immediate 32-bit signed
 3124 operand immL32()
 3125 %{
 3126   predicate(n->get_long() == (int) (n->get_long()));
 3127   match(ConL);
 3128 
 3129   op_cost(15);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 operand immL_Pow2()
 3135 %{
 3136   predicate(is_power_of_2((julong)n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(15);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 operand immL_NotPow2()
 3145 %{
 3146   predicate(is_power_of_2((julong)~n->get_long()));
 3147   match(ConL);
 3148 
 3149   op_cost(15);
 3150   format %{ %}
 3151   interface(CONST_INTER);
 3152 %}
 3153 
 3154 // Long Immediate zero
 3155 operand immL0()
 3156 %{
 3157   predicate(n->get_long() == 0L);
 3158   match(ConL);
 3159 
 3160   op_cost(10);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 // Constant for increment
 3166 operand immL1()
 3167 %{
 3168   predicate(n->get_long() == 1);
 3169   match(ConL);
 3170 
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Constant for decrement
 3176 operand immL_M1()
 3177 %{
 3178   predicate(n->get_long() == -1);
 3179   match(ConL);
 3180 
 3181   format %{ %}
 3182   interface(CONST_INTER);
 3183 %}
 3184 
 3185 // Long Immediate: the value 10
 3186 operand immL10()
 3187 %{
 3188   predicate(n->get_long() == 10);
 3189   match(ConL);
 3190 
 3191   format %{ %}
 3192   interface(CONST_INTER);
 3193 %}
 3194 
 3195 // Long immediate from 0 to 127.
 3196 // Used for a shorter form of long mul by 10.
 3197 operand immL_127()
 3198 %{
 3199   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3200   match(ConL);
 3201 
 3202   op_cost(10);
 3203   format %{ %}
 3204   interface(CONST_INTER);
 3205 %}
 3206 
 3207 // Long Immediate: low 32-bit mask
 3208 operand immL_32bits()
 3209 %{
 3210   predicate(n->get_long() == 0xFFFFFFFFL);
 3211   match(ConL);
 3212   op_cost(20);
 3213 
 3214   format %{ %}
 3215   interface(CONST_INTER);
 3216 %}
 3217 
 3218 // Int Immediate: 2^n-1, positive
 3219 operand immI_Pow2M1()
 3220 %{
 3221   predicate((n->get_int() > 0)
 3222             && is_power_of_2(n->get_int() + 1));
 3223   match(ConI);
 3224 
 3225   op_cost(20);
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Float Immediate zero
 3231 operand immF0()
 3232 %{
 3233   predicate(jint_cast(n->getf()) == 0);
 3234   match(ConF);
 3235 
 3236   op_cost(5);
 3237   format %{ %}
 3238   interface(CONST_INTER);
 3239 %}
 3240 
 3241 // Float Immediate
 3242 operand immF()
 3243 %{
 3244   match(ConF);
 3245 
 3246   op_cost(15);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Double Immediate zero
 3252 operand immD0()
 3253 %{
 3254   predicate(jlong_cast(n->getd()) == 0);
 3255   match(ConD);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Double Immediate
 3263 operand immD()
 3264 %{
 3265   match(ConD);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Immediates for special shifts (sign extend)
 3273 
 3274 // Constants for increment
 3275 operand immI_16()
 3276 %{
 3277   predicate(n->get_int() == 16);
 3278   match(ConI);
 3279 
 3280   format %{ %}
 3281   interface(CONST_INTER);
 3282 %}
 3283 
 3284 operand immI_24()
 3285 %{
 3286   predicate(n->get_int() == 24);
 3287   match(ConI);
 3288 
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Constant for byte-wide masking
 3294 operand immI_255()
 3295 %{
 3296   predicate(n->get_int() == 255);
 3297   match(ConI);
 3298 
 3299   format %{ %}
 3300   interface(CONST_INTER);
 3301 %}
 3302 
 3303 // Constant for short-wide masking
 3304 operand immI_65535()
 3305 %{
 3306   predicate(n->get_int() == 65535);
 3307   match(ConI);
 3308 
 3309   format %{ %}
 3310   interface(CONST_INTER);
 3311 %}
 3312 
 3313 // Constant for byte-wide masking
 3314 operand immL_255()
 3315 %{
 3316   predicate(n->get_long() == 255);
 3317   match(ConL);
 3318 
 3319   format %{ %}
 3320   interface(CONST_INTER);
 3321 %}
 3322 
 3323 // Constant for short-wide masking
 3324 operand immL_65535()
 3325 %{
 3326   predicate(n->get_long() == 65535);
 3327   match(ConL);
 3328 
 3329   format %{ %}
 3330   interface(CONST_INTER);
 3331 %}
 3332 
 3333 operand kReg()
 3334 %{
 3335   constraint(ALLOC_IN_RC(vectmask_reg));
 3336   match(RegVectMask);
 3337   format %{%}
 3338   interface(REG_INTER);
 3339 %}
 3340 
 3341 operand kReg_K1()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K2()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 // Special Registers
 3358 operand kReg_K3()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K4()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K5()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K6()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 // Special Registers
 3391 operand kReg_K7()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 // Register Operands
 3400 // Integer Register
 3401 operand rRegI()
 3402 %{
 3403   constraint(ALLOC_IN_RC(int_reg));
 3404   match(RegI);
 3405 
 3406   match(rax_RegI);
 3407   match(rbx_RegI);
 3408   match(rcx_RegI);
 3409   match(rdx_RegI);
 3410   match(rdi_RegI);
 3411 
 3412   format %{ %}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Special Registers
 3417 operand rax_RegI()
 3418 %{
 3419   constraint(ALLOC_IN_RC(int_rax_reg));
 3420   match(RegI);
 3421   match(rRegI);
 3422 
 3423   format %{ "RAX" %}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand rbx_RegI()
 3429 %{
 3430   constraint(ALLOC_IN_RC(int_rbx_reg));
 3431   match(RegI);
 3432   match(rRegI);
 3433 
 3434   format %{ "RBX" %}
 3435   interface(REG_INTER);
 3436 %}
 3437 
 3438 operand rcx_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rcx_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RCX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 operand rdx_RegI()
 3449 %{
 3450   constraint(ALLOC_IN_RC(int_rdx_reg));
 3451   match(RegI);
 3452   match(rRegI);
 3453 
 3454   format %{ "RDX" %}
 3455   interface(REG_INTER);
 3456 %}
 3457 
 3458 operand rdi_RegI()
 3459 %{
 3460   constraint(ALLOC_IN_RC(int_rdi_reg));
 3461   match(RegI);
 3462   match(rRegI);
 3463 
 3464   format %{ "RDI" %}
 3465   interface(REG_INTER);
 3466 %}
 3467 
 3468 operand no_rax_rdx_RegI()
 3469 %{
 3470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3471   match(RegI);
 3472   match(rbx_RegI);
 3473   match(rcx_RegI);
 3474   match(rdi_RegI);
 3475 
 3476   format %{ %}
 3477   interface(REG_INTER);
 3478 %}
 3479 
 3480 operand no_rbp_r13_RegI()
 3481 %{
 3482   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3483   match(RegI);
 3484   match(rRegI);
 3485   match(rax_RegI);
 3486   match(rbx_RegI);
 3487   match(rcx_RegI);
 3488   match(rdx_RegI);
 3489   match(rdi_RegI);
 3490 
 3491   format %{ %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 // Pointer Register
 3496 operand any_RegP()
 3497 %{
 3498   constraint(ALLOC_IN_RC(any_reg));
 3499   match(RegP);
 3500   match(rax_RegP);
 3501   match(rbx_RegP);
 3502   match(rdi_RegP);
 3503   match(rsi_RegP);
 3504   match(rbp_RegP);
 3505   match(r15_RegP);
 3506   match(rRegP);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 operand rRegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_reg));
 3515   match(RegP);
 3516   match(rax_RegP);
 3517   match(rbx_RegP);
 3518   match(rdi_RegP);
 3519   match(rsi_RegP);
 3520   match(rbp_RegP);  // See Q&A below about
 3521   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3522 
 3523   format %{ %}
 3524   interface(REG_INTER);
 3525 %}
 3526 
 3527 operand rRegN() %{
 3528   constraint(ALLOC_IN_RC(int_reg));
 3529   match(RegN);
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3536 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3537 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3538 // The output of an instruction is controlled by the allocator, which respects
 3539 // register class masks, not match rules.  Unless an instruction mentions
 3540 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3541 // by the allocator as an input.
 3542 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3543 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3544 // result, RBP is not included in the output of the instruction either.
 3545 
 3546 operand no_rax_RegP()
 3547 %{
 3548   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3549   match(RegP);
 3550   match(rbx_RegP);
 3551   match(rsi_RegP);
 3552   match(rdi_RegP);
 3553 
 3554   format %{ %}
 3555   interface(REG_INTER);
 3556 %}
 3557 
 3558 // This operand is not allowed to use RBP even if
 3559 // RBP is not used to hold the frame pointer.
 3560 operand no_rbp_RegP()
 3561 %{
 3562   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3563   match(RegP);
 3564   match(rbx_RegP);
 3565   match(rsi_RegP);
 3566   match(rdi_RegP);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 operand no_rax_rbx_RegP()
 3573 %{
 3574   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3575   match(RegP);
 3576   match(rsi_RegP);
 3577   match(rdi_RegP);
 3578 
 3579   format %{ %}
 3580   interface(REG_INTER);
 3581 %}
 3582 
 3583 // Special Registers
 3584 // Return a pointer value
 3585 operand rax_RegP()
 3586 %{
 3587   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3588   match(RegP);
 3589   match(rRegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // Special Registers
 3596 // Return a compressed pointer value
 3597 operand rax_RegN()
 3598 %{
 3599   constraint(ALLOC_IN_RC(int_rax_reg));
 3600   match(RegN);
 3601   match(rRegN);
 3602 
 3603   format %{ %}
 3604   interface(REG_INTER);
 3605 %}
 3606 
 3607 // Used in AtomicAdd
 3608 operand rbx_RegP()
 3609 %{
 3610   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3611   match(RegP);
 3612   match(rRegP);
 3613 
 3614   format %{ %}
 3615   interface(REG_INTER);
 3616 %}
 3617 
 3618 operand rsi_RegP()
 3619 %{
 3620   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3621   match(RegP);
 3622   match(rRegP);
 3623 
 3624   format %{ %}
 3625   interface(REG_INTER);
 3626 %}
 3627 
 3628 operand rbp_RegP()
 3629 %{
 3630   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3631   match(RegP);
 3632   match(rRegP);
 3633 
 3634   format %{ %}
 3635   interface(REG_INTER);
 3636 %}
 3637 
 3638 // Used in rep stosq
 3639 operand rdi_RegP()
 3640 %{
 3641   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3642   match(RegP);
 3643   match(rRegP);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand r15_RegP()
 3650 %{
 3651   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3652   match(RegP);
 3653   match(rRegP);
 3654 
 3655   format %{ %}
 3656   interface(REG_INTER);
 3657 %}
 3658 
 3659 operand rRegL()
 3660 %{
 3661   constraint(ALLOC_IN_RC(long_reg));
 3662   match(RegL);
 3663   match(rax_RegL);
 3664   match(rdx_RegL);
 3665 
 3666   format %{ %}
 3667   interface(REG_INTER);
 3668 %}
 3669 
 3670 // Special Registers
 3671 operand no_rax_rdx_RegL()
 3672 %{
 3673   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3674   match(RegL);
 3675   match(rRegL);
 3676 
 3677   format %{ %}
 3678   interface(REG_INTER);
 3679 %}
 3680 
 3681 operand no_rax_RegL()
 3682 %{
 3683   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3684   match(RegL);
 3685   match(rRegL);
 3686   match(rdx_RegL);
 3687 
 3688   format %{ %}
 3689   interface(REG_INTER);
 3690 %}
 3691 
 3692 operand rax_RegL()
 3693 %{
 3694   constraint(ALLOC_IN_RC(long_rax_reg));
 3695   match(RegL);
 3696   match(rRegL);
 3697 
 3698   format %{ "RAX" %}
 3699   interface(REG_INTER);
 3700 %}
 3701 
 3702 operand rcx_RegL()
 3703 %{
 3704   constraint(ALLOC_IN_RC(long_rcx_reg));
 3705   match(RegL);
 3706   match(rRegL);
 3707 
 3708   format %{ %}
 3709   interface(REG_INTER);
 3710 %}
 3711 
 3712 operand rdx_RegL()
 3713 %{
 3714   constraint(ALLOC_IN_RC(long_rdx_reg));
 3715   match(RegL);
 3716   match(rRegL);
 3717 
 3718   format %{ %}
 3719   interface(REG_INTER);
 3720 %}
 3721 
 3722 operand no_rbp_r13_RegL()
 3723 %{
 3724   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3725   match(RegL);
 3726   match(rRegL);
 3727   match(rax_RegL);
 3728   match(rcx_RegL);
 3729   match(rdx_RegL);
 3730 
 3731   format %{ %}
 3732   interface(REG_INTER);
 3733 %}
 3734 
 3735 // Flags register, used as output of compare instructions
 3736 operand rFlagsReg()
 3737 %{
 3738   constraint(ALLOC_IN_RC(int_flags));
 3739   match(RegFlags);
 3740 
 3741   format %{ "RFLAGS" %}
 3742   interface(REG_INTER);
 3743 %}
 3744 
 3745 // Flags register, used as output of FLOATING POINT compare instructions
 3746 operand rFlagsRegU()
 3747 %{
 3748   constraint(ALLOC_IN_RC(int_flags));
 3749   match(RegFlags);
 3750 
 3751   format %{ "RFLAGS_U" %}
 3752   interface(REG_INTER);
 3753 %}
 3754 
 3755 operand rFlagsRegUCF() %{
 3756   constraint(ALLOC_IN_RC(int_flags));
 3757   match(RegFlags);
 3758   predicate(false);
 3759 
 3760   format %{ "RFLAGS_U_CF" %}
 3761   interface(REG_INTER);
 3762 %}
 3763 
 3764 // Float register operands
 3765 operand regF() %{
 3766    constraint(ALLOC_IN_RC(float_reg));
 3767    match(RegF);
 3768 
 3769    format %{ %}
 3770    interface(REG_INTER);
 3771 %}
 3772 
 3773 // Float register operands
 3774 operand legRegF() %{
 3775    constraint(ALLOC_IN_RC(float_reg_legacy));
 3776    match(RegF);
 3777 
 3778    format %{ %}
 3779    interface(REG_INTER);
 3780 %}
 3781 
 3782 // Float register operands
 3783 operand vlRegF() %{
 3784    constraint(ALLOC_IN_RC(float_reg_vl));
 3785    match(RegF);
 3786 
 3787    format %{ %}
 3788    interface(REG_INTER);
 3789 %}
 3790 
 3791 // Double register operands
 3792 operand regD() %{
 3793    constraint(ALLOC_IN_RC(double_reg));
 3794    match(RegD);
 3795 
 3796    format %{ %}
 3797    interface(REG_INTER);
 3798 %}
 3799 
 3800 // Double register operands
 3801 operand legRegD() %{
 3802    constraint(ALLOC_IN_RC(double_reg_legacy));
 3803    match(RegD);
 3804 
 3805    format %{ %}
 3806    interface(REG_INTER);
 3807 %}
 3808 
 3809 // Double register operands
 3810 operand vlRegD() %{
 3811    constraint(ALLOC_IN_RC(double_reg_vl));
 3812    match(RegD);
 3813 
 3814    format %{ %}
 3815    interface(REG_INTER);
 3816 %}
 3817 
 3818 //----------Memory Operands----------------------------------------------------
 3819 // Direct Memory Operand
 3820 // operand direct(immP addr)
 3821 // %{
 3822 //   match(addr);
 3823 
 3824 //   format %{ "[$addr]" %}
 3825 //   interface(MEMORY_INTER) %{
 3826 //     base(0xFFFFFFFF);
 3827 //     index(0x4);
 3828 //     scale(0x0);
 3829 //     disp($addr);
 3830 //   %}
 3831 // %}
 3832 
 3833 // Indirect Memory Operand
 3834 operand indirect(any_RegP reg)
 3835 %{
 3836   constraint(ALLOC_IN_RC(ptr_reg));
 3837   match(reg);
 3838 
 3839   format %{ "[$reg]" %}
 3840   interface(MEMORY_INTER) %{
 3841     base($reg);
 3842     index(0x4);
 3843     scale(0x0);
 3844     disp(0x0);
 3845   %}
 3846 %}
 3847 
 3848 // Indirect Memory Plus Short Offset Operand
 3849 operand indOffset8(any_RegP reg, immL8 off)
 3850 %{
 3851   constraint(ALLOC_IN_RC(ptr_reg));
 3852   match(AddP reg off);
 3853 
 3854   format %{ "[$reg + $off (8-bit)]" %}
 3855   interface(MEMORY_INTER) %{
 3856     base($reg);
 3857     index(0x4);
 3858     scale(0x0);
 3859     disp($off);
 3860   %}
 3861 %}
 3862 
 3863 // Indirect Memory Plus Long Offset Operand
 3864 operand indOffset32(any_RegP reg, immL32 off)
 3865 %{
 3866   constraint(ALLOC_IN_RC(ptr_reg));
 3867   match(AddP reg off);
 3868 
 3869   format %{ "[$reg + $off (32-bit)]" %}
 3870   interface(MEMORY_INTER) %{
 3871     base($reg);
 3872     index(0x4);
 3873     scale(0x0);
 3874     disp($off);
 3875   %}
 3876 %}
 3877 
 3878 // Indirect Memory Plus Index Register Plus Offset Operand
 3879 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3880 %{
 3881   constraint(ALLOC_IN_RC(ptr_reg));
 3882   match(AddP (AddP reg lreg) off);
 3883 
 3884   op_cost(10);
 3885   format %{"[$reg + $off + $lreg]" %}
 3886   interface(MEMORY_INTER) %{
 3887     base($reg);
 3888     index($lreg);
 3889     scale(0x0);
 3890     disp($off);
 3891   %}
 3892 %}
 3893 
 3894 // Indirect Memory Plus Index Register Plus Offset Operand
 3895 operand indIndex(any_RegP reg, rRegL lreg)
 3896 %{
 3897   constraint(ALLOC_IN_RC(ptr_reg));
 3898   match(AddP reg lreg);
 3899 
 3900   op_cost(10);
 3901   format %{"[$reg + $lreg]" %}
 3902   interface(MEMORY_INTER) %{
 3903     base($reg);
 3904     index($lreg);
 3905     scale(0x0);
 3906     disp(0x0);
 3907   %}
 3908 %}
 3909 
 3910 // Indirect Memory Times Scale Plus Index Register
 3911 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3912 %{
 3913   constraint(ALLOC_IN_RC(ptr_reg));
 3914   match(AddP reg (LShiftL lreg scale));
 3915 
 3916   op_cost(10);
 3917   format %{"[$reg + $lreg << $scale]" %}
 3918   interface(MEMORY_INTER) %{
 3919     base($reg);
 3920     index($lreg);
 3921     scale($scale);
 3922     disp(0x0);
 3923   %}
 3924 %}
 3925 
 3926 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3927 %{
 3928   constraint(ALLOC_IN_RC(ptr_reg));
 3929   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3930   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3931 
 3932   op_cost(10);
 3933   format %{"[$reg + pos $idx << $scale]" %}
 3934   interface(MEMORY_INTER) %{
 3935     base($reg);
 3936     index($idx);
 3937     scale($scale);
 3938     disp(0x0);
 3939   %}
 3940 %}
 3941 
 3942 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3943 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3944 %{
 3945   constraint(ALLOC_IN_RC(ptr_reg));
 3946   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3947 
 3948   op_cost(10);
 3949   format %{"[$reg + $off + $lreg << $scale]" %}
 3950   interface(MEMORY_INTER) %{
 3951     base($reg);
 3952     index($lreg);
 3953     scale($scale);
 3954     disp($off);
 3955   %}
 3956 %}
 3957 
 3958 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3959 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3960 %{
 3961   constraint(ALLOC_IN_RC(ptr_reg));
 3962   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3963   match(AddP (AddP reg (ConvI2L idx)) off);
 3964 
 3965   op_cost(10);
 3966   format %{"[$reg + $off + $idx]" %}
 3967   interface(MEMORY_INTER) %{
 3968     base($reg);
 3969     index($idx);
 3970     scale(0x0);
 3971     disp($off);
 3972   %}
 3973 %}
 3974 
 3975 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3976 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3977 %{
 3978   constraint(ALLOC_IN_RC(ptr_reg));
 3979   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3980   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3981 
 3982   op_cost(10);
 3983   format %{"[$reg + $off + $idx << $scale]" %}
 3984   interface(MEMORY_INTER) %{
 3985     base($reg);
 3986     index($idx);
 3987     scale($scale);
 3988     disp($off);
 3989   %}
 3990 %}
 3991 
 3992 // Indirect Narrow Oop Plus Offset Operand
 3993 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3994 // we can't free r12 even with CompressedOops::base() == NULL.
 3995 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3996   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3997   constraint(ALLOC_IN_RC(ptr_reg));
 3998   match(AddP (DecodeN reg) off);
 3999 
 4000   op_cost(10);
 4001   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 4002   interface(MEMORY_INTER) %{
 4003     base(0xc); // R12
 4004     index($reg);
 4005     scale(0x3);
 4006     disp($off);
 4007   %}
 4008 %}
 4009 
 4010 // Indirect Memory Operand
 4011 operand indirectNarrow(rRegN reg)
 4012 %{
 4013   predicate(CompressedOops::shift() == 0);
 4014   constraint(ALLOC_IN_RC(ptr_reg));
 4015   match(DecodeN reg);
 4016 
 4017   format %{ "[$reg]" %}
 4018   interface(MEMORY_INTER) %{
 4019     base($reg);
 4020     index(0x4);
 4021     scale(0x0);
 4022     disp(0x0);
 4023   %}
 4024 %}
 4025 
 4026 // Indirect Memory Plus Short Offset Operand
 4027 operand indOffset8Narrow(rRegN reg, immL8 off)
 4028 %{
 4029   predicate(CompressedOops::shift() == 0);
 4030   constraint(ALLOC_IN_RC(ptr_reg));
 4031   match(AddP (DecodeN reg) off);
 4032 
 4033   format %{ "[$reg + $off (8-bit)]" %}
 4034   interface(MEMORY_INTER) %{
 4035     base($reg);
 4036     index(0x4);
 4037     scale(0x0);
 4038     disp($off);
 4039   %}
 4040 %}
 4041 
 4042 // Indirect Memory Plus Long Offset Operand
 4043 operand indOffset32Narrow(rRegN reg, immL32 off)
 4044 %{
 4045   predicate(CompressedOops::shift() == 0);
 4046   constraint(ALLOC_IN_RC(ptr_reg));
 4047   match(AddP (DecodeN reg) off);
 4048 
 4049   format %{ "[$reg + $off (32-bit)]" %}
 4050   interface(MEMORY_INTER) %{
 4051     base($reg);
 4052     index(0x4);
 4053     scale(0x0);
 4054     disp($off);
 4055   %}
 4056 %}
 4057 
 4058 // Indirect Memory Plus Index Register Plus Offset Operand
 4059 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4060 %{
 4061   predicate(CompressedOops::shift() == 0);
 4062   constraint(ALLOC_IN_RC(ptr_reg));
 4063   match(AddP (AddP (DecodeN reg) lreg) off);
 4064 
 4065   op_cost(10);
 4066   format %{"[$reg + $off + $lreg]" %}
 4067   interface(MEMORY_INTER) %{
 4068     base($reg);
 4069     index($lreg);
 4070     scale(0x0);
 4071     disp($off);
 4072   %}
 4073 %}
 4074 
 4075 // Indirect Memory Plus Index Register Plus Offset Operand
 4076 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4077 %{
 4078   predicate(CompressedOops::shift() == 0);
 4079   constraint(ALLOC_IN_RC(ptr_reg));
 4080   match(AddP (DecodeN reg) lreg);
 4081 
 4082   op_cost(10);
 4083   format %{"[$reg + $lreg]" %}
 4084   interface(MEMORY_INTER) %{
 4085     base($reg);
 4086     index($lreg);
 4087     scale(0x0);
 4088     disp(0x0);
 4089   %}
 4090 %}
 4091 
 4092 // Indirect Memory Times Scale Plus Index Register
 4093 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4094 %{
 4095   predicate(CompressedOops::shift() == 0);
 4096   constraint(ALLOC_IN_RC(ptr_reg));
 4097   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4098 
 4099   op_cost(10);
 4100   format %{"[$reg + $lreg << $scale]" %}
 4101   interface(MEMORY_INTER) %{
 4102     base($reg);
 4103     index($lreg);
 4104     scale($scale);
 4105     disp(0x0);
 4106   %}
 4107 %}
 4108 
 4109 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4110 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4111 %{
 4112   predicate(CompressedOops::shift() == 0);
 4113   constraint(ALLOC_IN_RC(ptr_reg));
 4114   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4115 
 4116   op_cost(10);
 4117   format %{"[$reg + $off + $lreg << $scale]" %}
 4118   interface(MEMORY_INTER) %{
 4119     base($reg);
 4120     index($lreg);
 4121     scale($scale);
 4122     disp($off);
 4123   %}
 4124 %}
 4125 
 4126 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4127 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4128 %{
 4129   constraint(ALLOC_IN_RC(ptr_reg));
 4130   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4131   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4132 
 4133   op_cost(10);
 4134   format %{"[$reg + $off + $idx]" %}
 4135   interface(MEMORY_INTER) %{
 4136     base($reg);
 4137     index($idx);
 4138     scale(0x0);
 4139     disp($off);
 4140   %}
 4141 %}
 4142 
 4143 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4144 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4145 %{
 4146   constraint(ALLOC_IN_RC(ptr_reg));
 4147   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4148   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4149 
 4150   op_cost(10);
 4151   format %{"[$reg + $off + $idx << $scale]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base($reg);
 4154     index($idx);
 4155     scale($scale);
 4156     disp($off);
 4157   %}
 4158 %}
 4159 
 4160 //----------Special Memory Operands--------------------------------------------
 4161 // Stack Slot Operand - This operand is used for loading and storing temporary
 4162 //                      values on the stack where a match requires a value to
 4163 //                      flow through memory.
 4164 operand stackSlotP(sRegP reg)
 4165 %{
 4166   constraint(ALLOC_IN_RC(stack_slots));
 4167   // No match rule because this operand is only generated in matching
 4168 
 4169   format %{ "[$reg]" %}
 4170   interface(MEMORY_INTER) %{
 4171     base(0x4);   // RSP
 4172     index(0x4);  // No Index
 4173     scale(0x0);  // No Scale
 4174     disp($reg);  // Stack Offset
 4175   %}
 4176 %}
 4177 
 4178 operand stackSlotI(sRegI reg)
 4179 %{
 4180   constraint(ALLOC_IN_RC(stack_slots));
 4181   // No match rule because this operand is only generated in matching
 4182 
 4183   format %{ "[$reg]" %}
 4184   interface(MEMORY_INTER) %{
 4185     base(0x4);   // RSP
 4186     index(0x4);  // No Index
 4187     scale(0x0);  // No Scale
 4188     disp($reg);  // Stack Offset
 4189   %}
 4190 %}
 4191 
 4192 operand stackSlotF(sRegF reg)
 4193 %{
 4194   constraint(ALLOC_IN_RC(stack_slots));
 4195   // No match rule because this operand is only generated in matching
 4196 
 4197   format %{ "[$reg]" %}
 4198   interface(MEMORY_INTER) %{
 4199     base(0x4);   // RSP
 4200     index(0x4);  // No Index
 4201     scale(0x0);  // No Scale
 4202     disp($reg);  // Stack Offset
 4203   %}
 4204 %}
 4205 
 4206 operand stackSlotD(sRegD reg)
 4207 %{
 4208   constraint(ALLOC_IN_RC(stack_slots));
 4209   // No match rule because this operand is only generated in matching
 4210 
 4211   format %{ "[$reg]" %}
 4212   interface(MEMORY_INTER) %{
 4213     base(0x4);   // RSP
 4214     index(0x4);  // No Index
 4215     scale(0x0);  // No Scale
 4216     disp($reg);  // Stack Offset
 4217   %}
 4218 %}
 4219 operand stackSlotL(sRegL reg)
 4220 %{
 4221   constraint(ALLOC_IN_RC(stack_slots));
 4222   // No match rule because this operand is only generated in matching
 4223 
 4224   format %{ "[$reg]" %}
 4225   interface(MEMORY_INTER) %{
 4226     base(0x4);   // RSP
 4227     index(0x4);  // No Index
 4228     scale(0x0);  // No Scale
 4229     disp($reg);  // Stack Offset
 4230   %}
 4231 %}
 4232 
 4233 //----------Conditional Branch Operands----------------------------------------
 4234 // Comparison Op  - This is the operation of the comparison, and is limited to
 4235 //                  the following set of codes:
 4236 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4237 //
 4238 // Other attributes of the comparison, such as unsignedness, are specified
 4239 // by the comparison instruction that sets a condition code flags register.
 4240 // That result is represented by a flags operand whose subtype is appropriate
 4241 // to the unsignedness (etc.) of the comparison.
 4242 //
 4243 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4244 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4245 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4246 
 4247 // Comparison Code
 4248 operand cmpOp()
 4249 %{
 4250   match(Bool);
 4251 
 4252   format %{ "" %}
 4253   interface(COND_INTER) %{
 4254     equal(0x4, "e");
 4255     not_equal(0x5, "ne");
 4256     less(0xC, "l");
 4257     greater_equal(0xD, "ge");
 4258     less_equal(0xE, "le");
 4259     greater(0xF, "g");
 4260     overflow(0x0, "o");
 4261     no_overflow(0x1, "no");
 4262   %}
 4263 %}
 4264 
 4265 // Comparison Code, unsigned compare.  Used by FP also, with
 4266 // C2 (unordered) turned into GT or LT already.  The other bits
 4267 // C0 and C3 are turned into Carry & Zero flags.
 4268 operand cmpOpU()
 4269 %{
 4270   match(Bool);
 4271 
 4272   format %{ "" %}
 4273   interface(COND_INTER) %{
 4274     equal(0x4, "e");
 4275     not_equal(0x5, "ne");
 4276     less(0x2, "b");
 4277     greater_equal(0x3, "ae");
 4278     less_equal(0x6, "be");
 4279     greater(0x7, "a");
 4280     overflow(0x0, "o");
 4281     no_overflow(0x1, "no");
 4282   %}
 4283 %}
 4284 
 4285 
 4286 // Floating comparisons that don't require any fixup for the unordered case,
 4287 // If both inputs of the comparison are the same, ZF is always set so we
 4288 // don't need to use cmpOpUCF2 for eq/ne
 4289 operand cmpOpUCF() %{
 4290   match(Bool);
 4291   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4292             n->as_Bool()->_test._test == BoolTest::ge ||
 4293             n->as_Bool()->_test._test == BoolTest::le ||
 4294             n->as_Bool()->_test._test == BoolTest::gt ||
 4295             n->in(1)->in(1) == n->in(1)->in(2));
 4296   format %{ "" %}
 4297   interface(COND_INTER) %{
 4298     equal(0xb, "np");
 4299     not_equal(0xa, "p");
 4300     less(0x2, "b");
 4301     greater_equal(0x3, "ae");
 4302     less_equal(0x6, "be");
 4303     greater(0x7, "a");
 4304     overflow(0x0, "o");
 4305     no_overflow(0x1, "no");
 4306   %}
 4307 %}
 4308 
 4309 
 4310 // Floating comparisons that can be fixed up with extra conditional jumps
 4311 operand cmpOpUCF2() %{
 4312   match(Bool);
 4313   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4314              n->as_Bool()->_test._test == BoolTest::eq) &&
 4315             n->in(1)->in(1) != n->in(1)->in(2));
 4316   format %{ "" %}
 4317   interface(COND_INTER) %{
 4318     equal(0x4, "e");
 4319     not_equal(0x5, "ne");
 4320     less(0x2, "b");
 4321     greater_equal(0x3, "ae");
 4322     less_equal(0x6, "be");
 4323     greater(0x7, "a");
 4324     overflow(0x0, "o");
 4325     no_overflow(0x1, "no");
 4326   %}
 4327 %}
 4328 
 4329 //----------OPERAND CLASSES----------------------------------------------------
 4330 // Operand Classes are groups of operands that are used as to simplify
 4331 // instruction definitions by not requiring the AD writer to specify separate
 4332 // instructions for every form of operand when the instruction accepts
 4333 // multiple operand types with the same basic encoding and format.  The classic
 4334 // case of this is memory operands.
 4335 
 4336 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4337                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4338                indCompressedOopOffset,
 4339                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4340                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4341                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4342 
 4343 //----------PIPELINE-----------------------------------------------------------
 4344 // Rules which define the behavior of the target architectures pipeline.
 4345 pipeline %{
 4346 
 4347 //----------ATTRIBUTES---------------------------------------------------------
 4348 attributes %{
 4349   variable_size_instructions;        // Fixed size instructions
 4350   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4351   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4352   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4353   instruction_fetch_units = 1;       // of 16 bytes
 4354 
 4355   // List of nop instructions
 4356   nops( MachNop );
 4357 %}
 4358 
 4359 //----------RESOURCES----------------------------------------------------------
 4360 // Resources are the functional units available to the machine
 4361 
 4362 // Generic P2/P3 pipeline
 4363 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4364 // 3 instructions decoded per cycle.
 4365 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4366 // 3 ALU op, only ALU0 handles mul instructions.
 4367 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4368            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4369            BR, FPU,
 4370            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4371 
 4372 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4373 // Pipeline Description specifies the stages in the machine's pipeline
 4374 
 4375 // Generic P2/P3 pipeline
 4376 pipe_desc(S0, S1, S2, S3, S4, S5);
 4377 
 4378 //----------PIPELINE CLASSES---------------------------------------------------
 4379 // Pipeline Classes describe the stages in which input and output are
 4380 // referenced by the hardware pipeline.
 4381 
 4382 // Naming convention: ialu or fpu
 4383 // Then: _reg
 4384 // Then: _reg if there is a 2nd register
 4385 // Then: _long if it's a pair of instructions implementing a long
 4386 // Then: _fat if it requires the big decoder
 4387 //   Or: _mem if it requires the big decoder and a memory unit.
 4388 
 4389 // Integer ALU reg operation
 4390 pipe_class ialu_reg(rRegI dst)
 4391 %{
 4392     single_instruction;
 4393     dst    : S4(write);
 4394     dst    : S3(read);
 4395     DECODE : S0;        // any decoder
 4396     ALU    : S3;        // any alu
 4397 %}
 4398 
 4399 // Long ALU reg operation
 4400 pipe_class ialu_reg_long(rRegL dst)
 4401 %{
 4402     instruction_count(2);
 4403     dst    : S4(write);
 4404     dst    : S3(read);
 4405     DECODE : S0(2);     // any 2 decoders
 4406     ALU    : S3(2);     // both alus
 4407 %}
 4408 
 4409 // Integer ALU reg operation using big decoder
 4410 pipe_class ialu_reg_fat(rRegI dst)
 4411 %{
 4412     single_instruction;
 4413     dst    : S4(write);
 4414     dst    : S3(read);
 4415     D0     : S0;        // big decoder only
 4416     ALU    : S3;        // any alu
 4417 %}
 4418 
 4419 // Integer ALU reg-reg operation
 4420 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4421 %{
 4422     single_instruction;
 4423     dst    : S4(write);
 4424     src    : S3(read);
 4425     DECODE : S0;        // any decoder
 4426     ALU    : S3;        // any alu
 4427 %}
 4428 
 4429 // Integer ALU reg-reg operation
 4430 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4431 %{
 4432     single_instruction;
 4433     dst    : S4(write);
 4434     src    : S3(read);
 4435     D0     : S0;        // big decoder only
 4436     ALU    : S3;        // any alu
 4437 %}
 4438 
 4439 // Integer ALU reg-mem operation
 4440 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4441 %{
 4442     single_instruction;
 4443     dst    : S5(write);
 4444     mem    : S3(read);
 4445     D0     : S0;        // big decoder only
 4446     ALU    : S4;        // any alu
 4447     MEM    : S3;        // any mem
 4448 %}
 4449 
 4450 // Integer mem operation (prefetch)
 4451 pipe_class ialu_mem(memory mem)
 4452 %{
 4453     single_instruction;
 4454     mem    : S3(read);
 4455     D0     : S0;        // big decoder only
 4456     MEM    : S3;        // any mem
 4457 %}
 4458 
 4459 // Integer Store to Memory
 4460 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4461 %{
 4462     single_instruction;
 4463     mem    : S3(read);
 4464     src    : S5(read);
 4465     D0     : S0;        // big decoder only
 4466     ALU    : S4;        // any alu
 4467     MEM    : S3;
 4468 %}
 4469 
 4470 // // Long Store to Memory
 4471 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4472 // %{
 4473 //     instruction_count(2);
 4474 //     mem    : S3(read);
 4475 //     src    : S5(read);
 4476 //     D0     : S0(2);          // big decoder only; twice
 4477 //     ALU    : S4(2);     // any 2 alus
 4478 //     MEM    : S3(2);  // Both mems
 4479 // %}
 4480 
 4481 // Integer Store to Memory
 4482 pipe_class ialu_mem_imm(memory mem)
 4483 %{
 4484     single_instruction;
 4485     mem    : S3(read);
 4486     D0     : S0;        // big decoder only
 4487     ALU    : S4;        // any alu
 4488     MEM    : S3;
 4489 %}
 4490 
 4491 // Integer ALU0 reg-reg operation
 4492 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4493 %{
 4494     single_instruction;
 4495     dst    : S4(write);
 4496     src    : S3(read);
 4497     D0     : S0;        // Big decoder only
 4498     ALU0   : S3;        // only alu0
 4499 %}
 4500 
 4501 // Integer ALU0 reg-mem operation
 4502 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4503 %{
 4504     single_instruction;
 4505     dst    : S5(write);
 4506     mem    : S3(read);
 4507     D0     : S0;        // big decoder only
 4508     ALU0   : S4;        // ALU0 only
 4509     MEM    : S3;        // any mem
 4510 %}
 4511 
 4512 // Integer ALU reg-reg operation
 4513 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     src2   : S3(read);
 4519     DECODE : S0;        // any decoder
 4520     ALU    : S3;        // any alu
 4521 %}
 4522 
 4523 // Integer ALU reg-imm operation
 4524 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4525 %{
 4526     single_instruction;
 4527     cr     : S4(write);
 4528     src1   : S3(read);
 4529     DECODE : S0;        // any decoder
 4530     ALU    : S3;        // any alu
 4531 %}
 4532 
 4533 // Integer ALU reg-mem operation
 4534 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4535 %{
 4536     single_instruction;
 4537     cr     : S4(write);
 4538     src1   : S3(read);
 4539     src2   : S3(read);
 4540     D0     : S0;        // big decoder only
 4541     ALU    : S4;        // any alu
 4542     MEM    : S3;
 4543 %}
 4544 
 4545 // Conditional move reg-reg
 4546 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4547 %{
 4548     instruction_count(4);
 4549     y      : S4(read);
 4550     q      : S3(read);
 4551     p      : S3(read);
 4552     DECODE : S0(4);     // any decoder
 4553 %}
 4554 
 4555 // Conditional move reg-reg
 4556 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4557 %{
 4558     single_instruction;
 4559     dst    : S4(write);
 4560     src    : S3(read);
 4561     cr     : S3(read);
 4562     DECODE : S0;        // any decoder
 4563 %}
 4564 
 4565 // Conditional move reg-mem
 4566 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0;        // any decoder
 4573     MEM    : S3;
 4574 %}
 4575 
 4576 // Conditional move reg-reg long
 4577 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4578 %{
 4579     single_instruction;
 4580     dst    : S4(write);
 4581     src    : S3(read);
 4582     cr     : S3(read);
 4583     DECODE : S0(2);     // any 2 decoders
 4584 %}
 4585 
 4586 // XXX
 4587 // // Conditional move double reg-reg
 4588 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4589 // %{
 4590 //     single_instruction;
 4591 //     dst    : S4(write);
 4592 //     src    : S3(read);
 4593 //     cr     : S3(read);
 4594 //     DECODE : S0;     // any decoder
 4595 // %}
 4596 
 4597 // Float reg-reg operation
 4598 pipe_class fpu_reg(regD dst)
 4599 %{
 4600     instruction_count(2);
 4601     dst    : S3(read);
 4602     DECODE : S0(2);     // any 2 decoders
 4603     FPU    : S3;
 4604 %}
 4605 
 4606 // Float reg-reg operation
 4607 pipe_class fpu_reg_reg(regD dst, regD src)
 4608 %{
 4609     instruction_count(2);
 4610     dst    : S4(write);
 4611     src    : S3(read);
 4612     DECODE : S0(2);     // any 2 decoders
 4613     FPU    : S3;
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4618 %{
 4619     instruction_count(3);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     DECODE : S0(3);     // any 3 decoders
 4624     FPU    : S3(2);
 4625 %}
 4626 
 4627 // Float reg-reg operation
 4628 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4629 %{
 4630     instruction_count(4);
 4631     dst    : S4(write);
 4632     src1   : S3(read);
 4633     src2   : S3(read);
 4634     src3   : S3(read);
 4635     DECODE : S0(4);     // any 3 decoders
 4636     FPU    : S3(2);
 4637 %}
 4638 
 4639 // Float reg-reg operation
 4640 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4641 %{
 4642     instruction_count(4);
 4643     dst    : S4(write);
 4644     src1   : S3(read);
 4645     src2   : S3(read);
 4646     src3   : S3(read);
 4647     DECODE : S1(3);     // any 3 decoders
 4648     D0     : S0;        // Big decoder only
 4649     FPU    : S3(2);
 4650     MEM    : S3;
 4651 %}
 4652 
 4653 // Float reg-mem operation
 4654 pipe_class fpu_reg_mem(regD dst, memory mem)
 4655 %{
 4656     instruction_count(2);
 4657     dst    : S5(write);
 4658     mem    : S3(read);
 4659     D0     : S0;        // big decoder only
 4660     DECODE : S1;        // any decoder for FPU POP
 4661     FPU    : S4;
 4662     MEM    : S3;        // any mem
 4663 %}
 4664 
 4665 // Float reg-mem operation
 4666 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4667 %{
 4668     instruction_count(3);
 4669     dst    : S5(write);
 4670     src1   : S3(read);
 4671     mem    : S3(read);
 4672     D0     : S0;        // big decoder only
 4673     DECODE : S1(2);     // any decoder for FPU POP
 4674     FPU    : S4;
 4675     MEM    : S3;        // any mem
 4676 %}
 4677 
 4678 // Float mem-reg operation
 4679 pipe_class fpu_mem_reg(memory mem, regD src)
 4680 %{
 4681     instruction_count(2);
 4682     src    : S5(read);
 4683     mem    : S3(read);
 4684     DECODE : S0;        // any decoder for FPU PUSH
 4685     D0     : S1;        // big decoder only
 4686     FPU    : S4;
 4687     MEM    : S3;        // any mem
 4688 %}
 4689 
 4690 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4691 %{
 4692     instruction_count(3);
 4693     src1   : S3(read);
 4694     src2   : S3(read);
 4695     mem    : S3(read);
 4696     DECODE : S0(2);     // any decoder for FPU PUSH
 4697     D0     : S1;        // big decoder only
 4698     FPU    : S4;
 4699     MEM    : S3;        // any mem
 4700 %}
 4701 
 4702 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4703 %{
 4704     instruction_count(3);
 4705     src1   : S3(read);
 4706     src2   : S3(read);
 4707     mem    : S4(read);
 4708     DECODE : S0;        // any decoder for FPU PUSH
 4709     D0     : S0(2);     // big decoder only
 4710     FPU    : S4;
 4711     MEM    : S3(2);     // any mem
 4712 %}
 4713 
 4714 pipe_class fpu_mem_mem(memory dst, memory src1)
 4715 %{
 4716     instruction_count(2);
 4717     src1   : S3(read);
 4718     dst    : S4(read);
 4719     D0     : S0(2);     // big decoder only
 4720     MEM    : S3(2);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S3(read);
 4727     src2   : S3(read);
 4728     dst    : S4(read);
 4729     D0     : S0(3);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(3);     // any mem
 4732 %}
 4733 
 4734 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4735 %{
 4736     instruction_count(3);
 4737     src1   : S4(read);
 4738     mem    : S4(read);
 4739     DECODE : S0;        // any decoder for FPU PUSH
 4740     D0     : S0(2);     // big decoder only
 4741     FPU    : S4;
 4742     MEM    : S3(2);     // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_con(regD dst)
 4747 %{
 4748     instruction_count(2);
 4749     dst    : S5(write);
 4750     D0     : S0;        // big decoder only for the load
 4751     DECODE : S1;        // any decoder for FPU POP
 4752     FPU    : S4;
 4753     MEM    : S3;        // any mem
 4754 %}
 4755 
 4756 // Float load constant
 4757 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4758 %{
 4759     instruction_count(3);
 4760     dst    : S5(write);
 4761     src    : S3(read);
 4762     D0     : S0;        // big decoder only for the load
 4763     DECODE : S1(2);     // any decoder for FPU POP
 4764     FPU    : S4;
 4765     MEM    : S3;        // any mem
 4766 %}
 4767 
 4768 // UnConditional branch
 4769 pipe_class pipe_jmp(label labl)
 4770 %{
 4771     single_instruction;
 4772     BR   : S3;
 4773 %}
 4774 
 4775 // Conditional branch
 4776 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4777 %{
 4778     single_instruction;
 4779     cr    : S1(read);
 4780     BR    : S3;
 4781 %}
 4782 
 4783 // Allocation idiom
 4784 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4785 %{
 4786     instruction_count(1); force_serialization;
 4787     fixed_latency(6);
 4788     heap_ptr : S3(read);
 4789     DECODE   : S0(3);
 4790     D0       : S2;
 4791     MEM      : S3;
 4792     ALU      : S3(2);
 4793     dst      : S5(write);
 4794     BR       : S5;
 4795 %}
 4796 
 4797 // Generic big/slow expanded idiom
 4798 pipe_class pipe_slow()
 4799 %{
 4800     instruction_count(10); multiple_bundles; force_serialization;
 4801     fixed_latency(100);
 4802     D0  : S0(2);
 4803     MEM : S3(2);
 4804 %}
 4805 
 4806 // The real do-nothing guy
 4807 pipe_class empty()
 4808 %{
 4809     instruction_count(0);
 4810 %}
 4811 
 4812 // Define the class for the Nop node
 4813 define
 4814 %{
 4815    MachNop = empty;
 4816 %}
 4817 
 4818 %}
 4819 
 4820 //----------INSTRUCTIONS-------------------------------------------------------
 4821 //
 4822 // match      -- States which machine-independent subtree may be replaced
 4823 //               by this instruction.
 4824 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4825 //               selection to identify a minimum cost tree of machine
 4826 //               instructions that matches a tree of machine-independent
 4827 //               instructions.
 4828 // format     -- A string providing the disassembly for this instruction.
 4829 //               The value of an instruction's operand may be inserted
 4830 //               by referring to it with a '$' prefix.
 4831 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4832 //               to within an encode class as $primary, $secondary, and $tertiary
 4833 //               rrspectively.  The primary opcode is commonly used to
 4834 //               indicate the type of machine instruction, while secondary
 4835 //               and tertiary are often used for prefix options or addressing
 4836 //               modes.
 4837 // ins_encode -- A list of encode classes with parameters. The encode class
 4838 //               name must have been defined in an 'enc_class' specification
 4839 //               in the encode section of the architecture description.
 4840 
 4841 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4842 // Load Float
 4843 instruct MoveF2VL(vlRegF dst, regF src) %{
 4844   match(Set dst src);
 4845   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4846   ins_encode %{
 4847     ShouldNotReachHere();
 4848   %}
 4849   ins_pipe( fpu_reg_reg );
 4850 %}
 4851 
 4852 // Load Float
 4853 instruct MoveF2LEG(legRegF dst, regF src) %{
 4854   match(Set dst src);
 4855   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4856   ins_encode %{
 4857     ShouldNotReachHere();
 4858   %}
 4859   ins_pipe( fpu_reg_reg );
 4860 %}
 4861 
 4862 // Load Float
 4863 instruct MoveVL2F(regF dst, vlRegF src) %{
 4864   match(Set dst src);
 4865   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4866   ins_encode %{
 4867     ShouldNotReachHere();
 4868   %}
 4869   ins_pipe( fpu_reg_reg );
 4870 %}
 4871 
 4872 // Load Float
 4873 instruct MoveLEG2F(regF dst, legRegF src) %{
 4874   match(Set dst src);
 4875   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4876   ins_encode %{
 4877     ShouldNotReachHere();
 4878   %}
 4879   ins_pipe( fpu_reg_reg );
 4880 %}
 4881 
 4882 // Load Double
 4883 instruct MoveD2VL(vlRegD dst, regD src) %{
 4884   match(Set dst src);
 4885   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4886   ins_encode %{
 4887     ShouldNotReachHere();
 4888   %}
 4889   ins_pipe( fpu_reg_reg );
 4890 %}
 4891 
 4892 // Load Double
 4893 instruct MoveD2LEG(legRegD dst, regD src) %{
 4894   match(Set dst src);
 4895   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4896   ins_encode %{
 4897     ShouldNotReachHere();
 4898   %}
 4899   ins_pipe( fpu_reg_reg );
 4900 %}
 4901 
 4902 // Load Double
 4903 instruct MoveVL2D(regD dst, vlRegD src) %{
 4904   match(Set dst src);
 4905   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4906   ins_encode %{
 4907     ShouldNotReachHere();
 4908   %}
 4909   ins_pipe( fpu_reg_reg );
 4910 %}
 4911 
 4912 // Load Double
 4913 instruct MoveLEG2D(regD dst, legRegD src) %{
 4914   match(Set dst src);
 4915   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4916   ins_encode %{
 4917     ShouldNotReachHere();
 4918   %}
 4919   ins_pipe( fpu_reg_reg );
 4920 %}
 4921 
 4922 //----------Load/Store/Move Instructions---------------------------------------
 4923 //----------Load Instructions--------------------------------------------------
 4924 
 4925 // Load Byte (8 bit signed)
 4926 instruct loadB(rRegI dst, memory mem)
 4927 %{
 4928   match(Set dst (LoadB mem));
 4929 
 4930   ins_cost(125);
 4931   format %{ "movsbl  $dst, $mem\t# byte" %}
 4932 
 4933   ins_encode %{
 4934     __ movsbl($dst$$Register, $mem$$Address);
 4935   %}
 4936 
 4937   ins_pipe(ialu_reg_mem);
 4938 %}
 4939 
 4940 // Load Byte (8 bit signed) into Long Register
 4941 instruct loadB2L(rRegL dst, memory mem)
 4942 %{
 4943   match(Set dst (ConvI2L (LoadB mem)));
 4944 
 4945   ins_cost(125);
 4946   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4947 
 4948   ins_encode %{
 4949     __ movsbq($dst$$Register, $mem$$Address);
 4950   %}
 4951 
 4952   ins_pipe(ialu_reg_mem);
 4953 %}
 4954 
 4955 // Load Unsigned Byte (8 bit UNsigned)
 4956 instruct loadUB(rRegI dst, memory mem)
 4957 %{
 4958   match(Set dst (LoadUB mem));
 4959 
 4960   ins_cost(125);
 4961   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4962 
 4963   ins_encode %{
 4964     __ movzbl($dst$$Register, $mem$$Address);
 4965   %}
 4966 
 4967   ins_pipe(ialu_reg_mem);
 4968 %}
 4969 
 4970 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4971 instruct loadUB2L(rRegL dst, memory mem)
 4972 %{
 4973   match(Set dst (ConvI2L (LoadUB mem)));
 4974 
 4975   ins_cost(125);
 4976   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4977 
 4978   ins_encode %{
 4979     __ movzbq($dst$$Register, $mem$$Address);
 4980   %}
 4981 
 4982   ins_pipe(ialu_reg_mem);
 4983 %}
 4984 
 4985 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4986 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4987   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4988   effect(KILL cr);
 4989 
 4990   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4991             "andl    $dst, right_n_bits($mask, 8)" %}
 4992   ins_encode %{
 4993     Register Rdst = $dst$$Register;
 4994     __ movzbq(Rdst, $mem$$Address);
 4995     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4996   %}
 4997   ins_pipe(ialu_reg_mem);
 4998 %}
 4999 
 5000 // Load Short (16 bit signed)
 5001 instruct loadS(rRegI dst, memory mem)
 5002 %{
 5003   match(Set dst (LoadS mem));
 5004 
 5005   ins_cost(125);
 5006   format %{ "movswl $dst, $mem\t# short" %}
 5007 
 5008   ins_encode %{
 5009     __ movswl($dst$$Register, $mem$$Address);
 5010   %}
 5011 
 5012   ins_pipe(ialu_reg_mem);
 5013 %}
 5014 
 5015 // Load Short (16 bit signed) to Byte (8 bit signed)
 5016 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5017   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5018 
 5019   ins_cost(125);
 5020   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5021   ins_encode %{
 5022     __ movsbl($dst$$Register, $mem$$Address);
 5023   %}
 5024   ins_pipe(ialu_reg_mem);
 5025 %}
 5026 
 5027 // Load Short (16 bit signed) into Long Register
 5028 instruct loadS2L(rRegL dst, memory mem)
 5029 %{
 5030   match(Set dst (ConvI2L (LoadS mem)));
 5031 
 5032   ins_cost(125);
 5033   format %{ "movswq $dst, $mem\t# short -> long" %}
 5034 
 5035   ins_encode %{
 5036     __ movswq($dst$$Register, $mem$$Address);
 5037   %}
 5038 
 5039   ins_pipe(ialu_reg_mem);
 5040 %}
 5041 
 5042 // Load Unsigned Short/Char (16 bit UNsigned)
 5043 instruct loadUS(rRegI dst, memory mem)
 5044 %{
 5045   match(Set dst (LoadUS mem));
 5046 
 5047   ins_cost(125);
 5048   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5049 
 5050   ins_encode %{
 5051     __ movzwl($dst$$Register, $mem$$Address);
 5052   %}
 5053 
 5054   ins_pipe(ialu_reg_mem);
 5055 %}
 5056 
 5057 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5058 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5059   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5060 
 5061   ins_cost(125);
 5062   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5063   ins_encode %{
 5064     __ movsbl($dst$$Register, $mem$$Address);
 5065   %}
 5066   ins_pipe(ialu_reg_mem);
 5067 %}
 5068 
 5069 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5070 instruct loadUS2L(rRegL dst, memory mem)
 5071 %{
 5072   match(Set dst (ConvI2L (LoadUS mem)));
 5073 
 5074   ins_cost(125);
 5075   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5076 
 5077   ins_encode %{
 5078     __ movzwq($dst$$Register, $mem$$Address);
 5079   %}
 5080 
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5085 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087 
 5088   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5089   ins_encode %{
 5090     __ movzbq($dst$$Register, $mem$$Address);
 5091   %}
 5092   ins_pipe(ialu_reg_mem);
 5093 %}
 5094 
 5095 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5096 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5097   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5098   effect(KILL cr);
 5099 
 5100   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5101             "andl    $dst, right_n_bits($mask, 16)" %}
 5102   ins_encode %{
 5103     Register Rdst = $dst$$Register;
 5104     __ movzwq(Rdst, $mem$$Address);
 5105     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5106   %}
 5107   ins_pipe(ialu_reg_mem);
 5108 %}
 5109 
 5110 // Load Integer
 5111 instruct loadI(rRegI dst, memory mem)
 5112 %{
 5113   match(Set dst (LoadI mem));
 5114 
 5115   ins_cost(125);
 5116   format %{ "movl    $dst, $mem\t# int" %}
 5117 
 5118   ins_encode %{
 5119     __ movl($dst$$Register, $mem$$Address);
 5120   %}
 5121 
 5122   ins_pipe(ialu_reg_mem);
 5123 %}
 5124 
 5125 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5126 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5127   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5128 
 5129   ins_cost(125);
 5130   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5131   ins_encode %{
 5132     __ movsbl($dst$$Register, $mem$$Address);
 5133   %}
 5134   ins_pipe(ialu_reg_mem);
 5135 %}
 5136 
 5137 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5138 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5139   match(Set dst (AndI (LoadI mem) mask));
 5140 
 5141   ins_cost(125);
 5142   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5143   ins_encode %{
 5144     __ movzbl($dst$$Register, $mem$$Address);
 5145   %}
 5146   ins_pipe(ialu_reg_mem);
 5147 %}
 5148 
 5149 // Load Integer (32 bit signed) to Short (16 bit signed)
 5150 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5151   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5152 
 5153   ins_cost(125);
 5154   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5155   ins_encode %{
 5156     __ movswl($dst$$Register, $mem$$Address);
 5157   %}
 5158   ins_pipe(ialu_reg_mem);
 5159 %}
 5160 
 5161 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5162 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5163   match(Set dst (AndI (LoadI mem) mask));
 5164 
 5165   ins_cost(125);
 5166   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5167   ins_encode %{
 5168     __ movzwl($dst$$Register, $mem$$Address);
 5169   %}
 5170   ins_pipe(ialu_reg_mem);
 5171 %}
 5172 
 5173 // Load Integer into Long Register
 5174 instruct loadI2L(rRegL dst, memory mem)
 5175 %{
 5176   match(Set dst (ConvI2L (LoadI mem)));
 5177 
 5178   ins_cost(125);
 5179   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5180 
 5181   ins_encode %{
 5182     __ movslq($dst$$Register, $mem$$Address);
 5183   %}
 5184 
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFF into Long Register
 5189 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5193   ins_encode %{
 5194     __ movzbq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with mask 0xFFFF into Long Register
 5200 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202 
 5203   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5204   ins_encode %{
 5205     __ movzwq($dst$$Register, $mem$$Address);
 5206   %}
 5207   ins_pipe(ialu_reg_mem);
 5208 %}
 5209 
 5210 // Load Integer with a 31-bit mask into Long Register
 5211 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5212   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5213   effect(KILL cr);
 5214 
 5215   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5216             "andl    $dst, $mask" %}
 5217   ins_encode %{
 5218     Register Rdst = $dst$$Register;
 5219     __ movl(Rdst, $mem$$Address);
 5220     __ andl(Rdst, $mask$$constant);
 5221   %}
 5222   ins_pipe(ialu_reg_mem);
 5223 %}
 5224 
 5225 // Load Unsigned Integer into Long Register
 5226 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5227 %{
 5228   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5229 
 5230   ins_cost(125);
 5231   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5232 
 5233   ins_encode %{
 5234     __ movl($dst$$Register, $mem$$Address);
 5235   %}
 5236 
 5237   ins_pipe(ialu_reg_mem);
 5238 %}
 5239 
 5240 // Load Long
 5241 instruct loadL(rRegL dst, memory mem)
 5242 %{
 5243   match(Set dst (LoadL mem));
 5244 
 5245   ins_cost(125);
 5246   format %{ "movq    $dst, $mem\t# long" %}
 5247 
 5248   ins_encode %{
 5249     __ movq($dst$$Register, $mem$$Address);
 5250   %}
 5251 
 5252   ins_pipe(ialu_reg_mem); // XXX
 5253 %}
 5254 
 5255 // Load Range
 5256 instruct loadRange(rRegI dst, memory mem)
 5257 %{
 5258   match(Set dst (LoadRange mem));
 5259 
 5260   ins_cost(125); // XXX
 5261   format %{ "movl    $dst, $mem\t# range" %}
 5262   ins_encode %{
 5263     __ movl($dst$$Register, $mem$$Address);
 5264   %}
 5265   ins_pipe(ialu_reg_mem);
 5266 %}
 5267 
 5268 // Load Pointer
 5269 instruct loadP(rRegP dst, memory mem)
 5270 %{
 5271   match(Set dst (LoadP mem));
 5272   predicate(n->as_Load()->barrier_data() == 0);
 5273 
 5274   ins_cost(125); // XXX
 5275   format %{ "movq    $dst, $mem\t# ptr" %}
 5276   ins_encode %{
 5277     __ movq($dst$$Register, $mem$$Address);
 5278   %}
 5279   ins_pipe(ialu_reg_mem); // XXX
 5280 %}
 5281 
 5282 // Load Compressed Pointer
 5283 instruct loadN(rRegN dst, memory mem)
 5284 %{
 5285    match(Set dst (LoadN mem));
 5286 
 5287    ins_cost(125); // XXX
 5288    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5289    ins_encode %{
 5290      __ movl($dst$$Register, $mem$$Address);
 5291    %}
 5292    ins_pipe(ialu_reg_mem); // XXX
 5293 %}
 5294 
 5295 
 5296 // Load Klass Pointer
 5297 instruct loadKlass(rRegP dst, memory mem)
 5298 %{
 5299   match(Set dst (LoadKlass mem));
 5300 
 5301   ins_cost(125); // XXX
 5302   format %{ "movq    $dst, $mem\t# class" %}
 5303   ins_encode %{
 5304     __ movq($dst$$Register, $mem$$Address);
 5305   %}
 5306   ins_pipe(ialu_reg_mem); // XXX
 5307 %}
 5308 
 5309 // Load narrow Klass Pointer
 5310 instruct loadNKlass(rRegN dst, memory mem)
 5311 %{
 5312   match(Set dst (LoadNKlass mem));
 5313 
 5314   ins_cost(125); // XXX
 5315   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5316   ins_encode %{
 5317     __ movl($dst$$Register, $mem$$Address);
 5318   %}
 5319   ins_pipe(ialu_reg_mem); // XXX
 5320 %}
 5321 
 5322 // Load Float
 5323 instruct loadF(regF dst, memory mem)
 5324 %{
 5325   match(Set dst (LoadF mem));
 5326 
 5327   ins_cost(145); // XXX
 5328   format %{ "movss   $dst, $mem\t# float" %}
 5329   ins_encode %{
 5330     __ movflt($dst$$XMMRegister, $mem$$Address);
 5331   %}
 5332   ins_pipe(pipe_slow); // XXX
 5333 %}
 5334 
 5335 // Load Double
 5336 instruct loadD_partial(regD dst, memory mem)
 5337 %{
 5338   predicate(!UseXmmLoadAndClearUpper);
 5339   match(Set dst (LoadD mem));
 5340 
 5341   ins_cost(145); // XXX
 5342   format %{ "movlpd  $dst, $mem\t# double" %}
 5343   ins_encode %{
 5344     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5345   %}
 5346   ins_pipe(pipe_slow); // XXX
 5347 %}
 5348 
 5349 instruct loadD(regD dst, memory mem)
 5350 %{
 5351   predicate(UseXmmLoadAndClearUpper);
 5352   match(Set dst (LoadD mem));
 5353 
 5354   ins_cost(145); // XXX
 5355   format %{ "movsd   $dst, $mem\t# double" %}
 5356   ins_encode %{
 5357     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5358   %}
 5359   ins_pipe(pipe_slow); // XXX
 5360 %}
 5361 
 5362 
 5363 // Following pseudo code describes the algorithm for max[FD]:
 5364 // Min algorithm is on similar lines
 5365 //  btmp = (b < +0.0) ? a : b
 5366 //  atmp = (b < +0.0) ? b : a
 5367 //  Tmp  = Max_Float(atmp , btmp)
 5368 //  Res  = (atmp == NaN) ? atmp : Tmp
 5369 
 5370 // max = java.lang.Math.max(float a, float b)
 5371 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5372   predicate(UseAVX > 0 && !n->is_reduction());
 5373   match(Set dst (MaxF a b));
 5374   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5375   format %{
 5376      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5377      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5378      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5379      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5380      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5381   %}
 5382   ins_encode %{
 5383     int vector_len = Assembler::AVX_128bit;
 5384     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5385     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5386     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5387     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5388     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5389  %}
 5390   ins_pipe( pipe_slow );
 5391 %}
 5392 
 5393 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5394   predicate(UseAVX > 0 && n->is_reduction());
 5395   match(Set dst (MaxF a b));
 5396   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5397 
 5398   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5399   ins_encode %{
 5400     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5401                     false /*min*/, true /*single*/);
 5402   %}
 5403   ins_pipe( pipe_slow );
 5404 %}
 5405 
 5406 // max = java.lang.Math.max(double a, double b)
 5407 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5408   predicate(UseAVX > 0 && !n->is_reduction());
 5409   match(Set dst (MaxD a b));
 5410   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5411   format %{
 5412      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5413      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5414      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5415      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5416      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5417   %}
 5418   ins_encode %{
 5419     int vector_len = Assembler::AVX_128bit;
 5420     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5421     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5422     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5423     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5424     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5425   %}
 5426   ins_pipe( pipe_slow );
 5427 %}
 5428 
 5429 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5430   predicate(UseAVX > 0 && n->is_reduction());
 5431   match(Set dst (MaxD a b));
 5432   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5433 
 5434   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5435   ins_encode %{
 5436     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5437                     false /*min*/, false /*single*/);
 5438   %}
 5439   ins_pipe( pipe_slow );
 5440 %}
 5441 
 5442 // min = java.lang.Math.min(float a, float b)
 5443 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5444   predicate(UseAVX > 0 && !n->is_reduction());
 5445   match(Set dst (MinF a b));
 5446   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5447   format %{
 5448      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5449      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5450      "vminss           $tmp,$atmp,$btmp           \n\t"
 5451      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5452      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5453   %}
 5454   ins_encode %{
 5455     int vector_len = Assembler::AVX_128bit;
 5456     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5457     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5458     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5459     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5460     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5461   %}
 5462   ins_pipe( pipe_slow );
 5463 %}
 5464 
 5465 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5466   predicate(UseAVX > 0 && n->is_reduction());
 5467   match(Set dst (MinF a b));
 5468   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5469 
 5470   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5471   ins_encode %{
 5472     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5473                     true /*min*/, true /*single*/);
 5474   %}
 5475   ins_pipe( pipe_slow );
 5476 %}
 5477 
 5478 // min = java.lang.Math.min(double a, double b)
 5479 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5480   predicate(UseAVX > 0 && !n->is_reduction());
 5481   match(Set dst (MinD a b));
 5482   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5483   format %{
 5484      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5485      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5486      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5487      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5488      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5489   %}
 5490   ins_encode %{
 5491     int vector_len = Assembler::AVX_128bit;
 5492     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5493     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5494     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5495     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5496     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5497   %}
 5498   ins_pipe( pipe_slow );
 5499 %}
 5500 
 5501 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5502   predicate(UseAVX > 0 && n->is_reduction());
 5503   match(Set dst (MinD a b));
 5504   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5505 
 5506   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5507   ins_encode %{
 5508     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5509                     true /*min*/, false /*single*/);
 5510   %}
 5511   ins_pipe( pipe_slow );
 5512 %}
 5513 
 5514 // Load Effective Address
 5515 instruct leaP8(rRegP dst, indOffset8 mem)
 5516 %{
 5517   match(Set dst mem);
 5518 
 5519   ins_cost(110); // XXX
 5520   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5521   ins_encode %{
 5522     __ leaq($dst$$Register, $mem$$Address);
 5523   %}
 5524   ins_pipe(ialu_reg_reg_fat);
 5525 %}
 5526 
 5527 instruct leaP32(rRegP dst, indOffset32 mem)
 5528 %{
 5529   match(Set dst mem);
 5530 
 5531   ins_cost(110);
 5532   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5533   ins_encode %{
 5534     __ leaq($dst$$Register, $mem$$Address);
 5535   %}
 5536   ins_pipe(ialu_reg_reg_fat);
 5537 %}
 5538 
 5539 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5540 %{
 5541   match(Set dst mem);
 5542 
 5543   ins_cost(110);
 5544   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5545   ins_encode %{
 5546     __ leaq($dst$$Register, $mem$$Address);
 5547   %}
 5548   ins_pipe(ialu_reg_reg_fat);
 5549 %}
 5550 
 5551 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5552 %{
 5553   match(Set dst mem);
 5554 
 5555   ins_cost(110);
 5556   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5557   ins_encode %{
 5558     __ leaq($dst$$Register, $mem$$Address);
 5559   %}
 5560   ins_pipe(ialu_reg_reg_fat);
 5561 %}
 5562 
 5563 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5564 %{
 5565   match(Set dst mem);
 5566 
 5567   ins_cost(110);
 5568   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5569   ins_encode %{
 5570     __ leaq($dst$$Register, $mem$$Address);
 5571   %}
 5572   ins_pipe(ialu_reg_reg_fat);
 5573 %}
 5574 
 5575 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5576 %{
 5577   match(Set dst mem);
 5578 
 5579   ins_cost(110);
 5580   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5581   ins_encode %{
 5582     __ leaq($dst$$Register, $mem$$Address);
 5583   %}
 5584   ins_pipe(ialu_reg_reg_fat);
 5585 %}
 5586 
 5587 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5588 %{
 5589   match(Set dst mem);
 5590 
 5591   ins_cost(110);
 5592   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5593   ins_encode %{
 5594     __ leaq($dst$$Register, $mem$$Address);
 5595   %}
 5596   ins_pipe(ialu_reg_reg_fat);
 5597 %}
 5598 
 5599 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5600 %{
 5601   match(Set dst mem);
 5602 
 5603   ins_cost(110);
 5604   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5605   ins_encode %{
 5606     __ leaq($dst$$Register, $mem$$Address);
 5607   %}
 5608   ins_pipe(ialu_reg_reg_fat);
 5609 %}
 5610 
 5611 // Load Effective Address which uses Narrow (32-bits) oop
 5612 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5613 %{
 5614   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5615   match(Set dst mem);
 5616 
 5617   ins_cost(110);
 5618   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5619   ins_encode %{
 5620     __ leaq($dst$$Register, $mem$$Address);
 5621   %}
 5622   ins_pipe(ialu_reg_reg_fat);
 5623 %}
 5624 
 5625 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5626 %{
 5627   predicate(CompressedOops::shift() == 0);
 5628   match(Set dst mem);
 5629 
 5630   ins_cost(110); // XXX
 5631   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5632   ins_encode %{
 5633     __ leaq($dst$$Register, $mem$$Address);
 5634   %}
 5635   ins_pipe(ialu_reg_reg_fat);
 5636 %}
 5637 
 5638 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5639 %{
 5640   predicate(CompressedOops::shift() == 0);
 5641   match(Set dst mem);
 5642 
 5643   ins_cost(110);
 5644   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5645   ins_encode %{
 5646     __ leaq($dst$$Register, $mem$$Address);
 5647   %}
 5648   ins_pipe(ialu_reg_reg_fat);
 5649 %}
 5650 
 5651 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5652 %{
 5653   predicate(CompressedOops::shift() == 0);
 5654   match(Set dst mem);
 5655 
 5656   ins_cost(110);
 5657   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5658   ins_encode %{
 5659     __ leaq($dst$$Register, $mem$$Address);
 5660   %}
 5661   ins_pipe(ialu_reg_reg_fat);
 5662 %}
 5663 
 5664 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5665 %{
 5666   predicate(CompressedOops::shift() == 0);
 5667   match(Set dst mem);
 5668 
 5669   ins_cost(110);
 5670   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5671   ins_encode %{
 5672     __ leaq($dst$$Register, $mem$$Address);
 5673   %}
 5674   ins_pipe(ialu_reg_reg_fat);
 5675 %}
 5676 
 5677 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5678 %{
 5679   predicate(CompressedOops::shift() == 0);
 5680   match(Set dst mem);
 5681 
 5682   ins_cost(110);
 5683   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5684   ins_encode %{
 5685     __ leaq($dst$$Register, $mem$$Address);
 5686   %}
 5687   ins_pipe(ialu_reg_reg_fat);
 5688 %}
 5689 
 5690 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5691 %{
 5692   predicate(CompressedOops::shift() == 0);
 5693   match(Set dst mem);
 5694 
 5695   ins_cost(110);
 5696   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5697   ins_encode %{
 5698     __ leaq($dst$$Register, $mem$$Address);
 5699   %}
 5700   ins_pipe(ialu_reg_reg_fat);
 5701 %}
 5702 
 5703 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5704 %{
 5705   predicate(CompressedOops::shift() == 0);
 5706   match(Set dst mem);
 5707 
 5708   ins_cost(110);
 5709   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5710   ins_encode %{
 5711     __ leaq($dst$$Register, $mem$$Address);
 5712   %}
 5713   ins_pipe(ialu_reg_reg_fat);
 5714 %}
 5715 
 5716 instruct loadConI(rRegI dst, immI src)
 5717 %{
 5718   match(Set dst src);
 5719 
 5720   format %{ "movl    $dst, $src\t# int" %}
 5721   ins_encode %{
 5722     __ movl($dst$$Register, $src$$constant);
 5723   %}
 5724   ins_pipe(ialu_reg_fat); // XXX
 5725 %}
 5726 
 5727 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5728 %{
 5729   match(Set dst src);
 5730   effect(KILL cr);
 5731 
 5732   ins_cost(50);
 5733   format %{ "xorl    $dst, $dst\t# int" %}
 5734   ins_encode %{
 5735     __ xorl($dst$$Register, $dst$$Register);
 5736   %}
 5737   ins_pipe(ialu_reg);
 5738 %}
 5739 
 5740 instruct loadConL(rRegL dst, immL src)
 5741 %{
 5742   match(Set dst src);
 5743 
 5744   ins_cost(150);
 5745   format %{ "movq    $dst, $src\t# long" %}
 5746   ins_encode %{
 5747     __ mov64($dst$$Register, $src$$constant);
 5748   %}
 5749   ins_pipe(ialu_reg);
 5750 %}
 5751 
 5752 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5753 %{
 5754   match(Set dst src);
 5755   effect(KILL cr);
 5756 
 5757   ins_cost(50);
 5758   format %{ "xorl    $dst, $dst\t# long" %}
 5759   ins_encode %{
 5760     __ xorl($dst$$Register, $dst$$Register);
 5761   %}
 5762   ins_pipe(ialu_reg); // XXX
 5763 %}
 5764 
 5765 instruct loadConUL32(rRegL dst, immUL32 src)
 5766 %{
 5767   match(Set dst src);
 5768 
 5769   ins_cost(60);
 5770   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5771   ins_encode %{
 5772     __ movl($dst$$Register, $src$$constant);
 5773   %}
 5774   ins_pipe(ialu_reg);
 5775 %}
 5776 
 5777 instruct loadConL32(rRegL dst, immL32 src)
 5778 %{
 5779   match(Set dst src);
 5780 
 5781   ins_cost(70);
 5782   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5783   ins_encode %{
 5784     __ movq($dst$$Register, $src$$constant);
 5785   %}
 5786   ins_pipe(ialu_reg);
 5787 %}
 5788 
 5789 instruct loadConP(rRegP dst, immP con) %{
 5790   match(Set dst con);
 5791 
 5792   format %{ "movq    $dst, $con\t# ptr" %}
 5793   ins_encode %{
 5794     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5795   %}
 5796   ins_pipe(ialu_reg_fat); // XXX
 5797 %}
 5798 
 5799 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5800 %{
 5801   match(Set dst src);
 5802   effect(KILL cr);
 5803 
 5804   ins_cost(50);
 5805   format %{ "xorl    $dst, $dst\t# ptr" %}
 5806   ins_encode %{
 5807     __ xorl($dst$$Register, $dst$$Register);
 5808   %}
 5809   ins_pipe(ialu_reg);
 5810 %}
 5811 
 5812 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5813 %{
 5814   match(Set dst src);
 5815   effect(KILL cr);
 5816 
 5817   ins_cost(60);
 5818   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5819   ins_encode %{
 5820     __ movl($dst$$Register, $src$$constant);
 5821   %}
 5822   ins_pipe(ialu_reg);
 5823 %}
 5824 
 5825 instruct loadConF(regF dst, immF con) %{
 5826   match(Set dst con);
 5827   ins_cost(125);
 5828   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5829   ins_encode %{
 5830     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5831   %}
 5832   ins_pipe(pipe_slow);
 5833 %}
 5834 
 5835 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5836   match(Set dst src);
 5837   effect(KILL cr);
 5838   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5839   ins_encode %{
 5840     __ xorq($dst$$Register, $dst$$Register);
 5841   %}
 5842   ins_pipe(ialu_reg);
 5843 %}
 5844 
 5845 instruct loadConN(rRegN dst, immN src) %{
 5846   match(Set dst src);
 5847 
 5848   ins_cost(125);
 5849   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5850   ins_encode %{
 5851     address con = (address)$src$$constant;
 5852     if (con == NULL) {
 5853       ShouldNotReachHere();
 5854     } else {
 5855       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5856     }
 5857   %}
 5858   ins_pipe(ialu_reg_fat); // XXX
 5859 %}
 5860 
 5861 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5862   match(Set dst src);
 5863 
 5864   ins_cost(125);
 5865   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5866   ins_encode %{
 5867     address con = (address)$src$$constant;
 5868     if (con == NULL) {
 5869       ShouldNotReachHere();
 5870     } else {
 5871       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5872     }
 5873   %}
 5874   ins_pipe(ialu_reg_fat); // XXX
 5875 %}
 5876 
 5877 instruct loadConF0(regF dst, immF0 src)
 5878 %{
 5879   match(Set dst src);
 5880   ins_cost(100);
 5881 
 5882   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5883   ins_encode %{
 5884     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5885   %}
 5886   ins_pipe(pipe_slow);
 5887 %}
 5888 
 5889 // Use the same format since predicate() can not be used here.
 5890 instruct loadConD(regD dst, immD con) %{
 5891   match(Set dst con);
 5892   ins_cost(125);
 5893   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5894   ins_encode %{
 5895     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5896   %}
 5897   ins_pipe(pipe_slow);
 5898 %}
 5899 
 5900 instruct loadConD0(regD dst, immD0 src)
 5901 %{
 5902   match(Set dst src);
 5903   ins_cost(100);
 5904 
 5905   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5906   ins_encode %{
 5907     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5908   %}
 5909   ins_pipe(pipe_slow);
 5910 %}
 5911 
 5912 instruct loadSSI(rRegI dst, stackSlotI src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movl    $dst, $src\t# int stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSL(rRegL dst, stackSlotL src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movq    $dst, $src\t# long stk" %}
 5929   opcode(0x8B);
 5930   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5931   ins_pipe(ialu_reg_mem);
 5932 %}
 5933 
 5934 instruct loadSSP(rRegP dst, stackSlotP src)
 5935 %{
 5936   match(Set dst src);
 5937 
 5938   ins_cost(125);
 5939   format %{ "movq    $dst, $src\t# ptr stk" %}
 5940   opcode(0x8B);
 5941   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5942   ins_pipe(ialu_reg_mem);
 5943 %}
 5944 
 5945 instruct loadSSF(regF dst, stackSlotF src)
 5946 %{
 5947   match(Set dst src);
 5948 
 5949   ins_cost(125);
 5950   format %{ "movss   $dst, $src\t# float stk" %}
 5951   ins_encode %{
 5952     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5953   %}
 5954   ins_pipe(pipe_slow); // XXX
 5955 %}
 5956 
 5957 // Use the same format since predicate() can not be used here.
 5958 instruct loadSSD(regD dst, stackSlotD src)
 5959 %{
 5960   match(Set dst src);
 5961 
 5962   ins_cost(125);
 5963   format %{ "movsd   $dst, $src\t# double stk" %}
 5964   ins_encode  %{
 5965     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5966   %}
 5967   ins_pipe(pipe_slow); // XXX
 5968 %}
 5969 
 5970 // Prefetch instructions for allocation.
 5971 // Must be safe to execute with invalid address (cannot fault).
 5972 
 5973 instruct prefetchAlloc( memory mem ) %{
 5974   predicate(AllocatePrefetchInstr==3);
 5975   match(PrefetchAllocation mem);
 5976   ins_cost(125);
 5977 
 5978   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5979   ins_encode %{
 5980     __ prefetchw($mem$$Address);
 5981   %}
 5982   ins_pipe(ialu_mem);
 5983 %}
 5984 
 5985 instruct prefetchAllocNTA( memory mem ) %{
 5986   predicate(AllocatePrefetchInstr==0);
 5987   match(PrefetchAllocation mem);
 5988   ins_cost(125);
 5989 
 5990   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5991   ins_encode %{
 5992     __ prefetchnta($mem$$Address);
 5993   %}
 5994   ins_pipe(ialu_mem);
 5995 %}
 5996 
 5997 instruct prefetchAllocT0( memory mem ) %{
 5998   predicate(AllocatePrefetchInstr==1);
 5999   match(PrefetchAllocation mem);
 6000   ins_cost(125);
 6001 
 6002   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6003   ins_encode %{
 6004     __ prefetcht0($mem$$Address);
 6005   %}
 6006   ins_pipe(ialu_mem);
 6007 %}
 6008 
 6009 instruct prefetchAllocT2( memory mem ) %{
 6010   predicate(AllocatePrefetchInstr==2);
 6011   match(PrefetchAllocation mem);
 6012   ins_cost(125);
 6013 
 6014   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6015   ins_encode %{
 6016     __ prefetcht2($mem$$Address);
 6017   %}
 6018   ins_pipe(ialu_mem);
 6019 %}
 6020 
 6021 //----------Store Instructions-------------------------------------------------
 6022 
 6023 // Store Byte
 6024 instruct storeB(memory mem, rRegI src)
 6025 %{
 6026   match(Set mem (StoreB mem src));
 6027 
 6028   ins_cost(125); // XXX
 6029   format %{ "movb    $mem, $src\t# byte" %}
 6030   ins_encode %{
 6031     __ movb($mem$$Address, $src$$Register);
 6032   %}
 6033   ins_pipe(ialu_mem_reg);
 6034 %}
 6035 
 6036 // Store Char/Short
 6037 instruct storeC(memory mem, rRegI src)
 6038 %{
 6039   match(Set mem (StoreC mem src));
 6040 
 6041   ins_cost(125); // XXX
 6042   format %{ "movw    $mem, $src\t# char/short" %}
 6043   ins_encode %{
 6044     __ movw($mem$$Address, $src$$Register);
 6045   %}
 6046   ins_pipe(ialu_mem_reg);
 6047 %}
 6048 
 6049 // Store Integer
 6050 instruct storeI(memory mem, rRegI src)
 6051 %{
 6052   match(Set mem (StoreI mem src));
 6053 
 6054   ins_cost(125); // XXX
 6055   format %{ "movl    $mem, $src\t# int" %}
 6056   ins_encode %{
 6057     __ movl($mem$$Address, $src$$Register);
 6058   %}
 6059   ins_pipe(ialu_mem_reg);
 6060 %}
 6061 
 6062 // Store Long
 6063 instruct storeL(memory mem, rRegL src)
 6064 %{
 6065   match(Set mem (StoreL mem src));
 6066 
 6067   ins_cost(125); // XXX
 6068   format %{ "movq    $mem, $src\t# long" %}
 6069   ins_encode %{
 6070     __ movq($mem$$Address, $src$$Register);
 6071   %}
 6072   ins_pipe(ialu_mem_reg); // XXX
 6073 %}
 6074 
 6075 // Store Pointer
 6076 instruct storeP(memory mem, any_RegP src)
 6077 %{
 6078   match(Set mem (StoreP mem src));
 6079 
 6080   ins_cost(125); // XXX
 6081   format %{ "movq    $mem, $src\t# ptr" %}
 6082   ins_encode %{
 6083     __ movq($mem$$Address, $src$$Register);
 6084   %}
 6085   ins_pipe(ialu_mem_reg);
 6086 %}
 6087 
 6088 instruct storeImmP0(memory mem, immP0 zero)
 6089 %{
 6090   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6091   match(Set mem (StoreP mem zero));
 6092 
 6093   ins_cost(125); // XXX
 6094   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6095   ins_encode %{
 6096     __ movq($mem$$Address, r12);
 6097   %}
 6098   ins_pipe(ialu_mem_reg);
 6099 %}
 6100 
 6101 // Store NULL Pointer, mark word, or other simple pointer constant.
 6102 instruct storeImmP(memory mem, immP31 src)
 6103 %{
 6104   match(Set mem (StoreP mem src));
 6105 
 6106   ins_cost(150); // XXX
 6107   format %{ "movq    $mem, $src\t# ptr" %}
 6108   ins_encode %{
 6109     __ movq($mem$$Address, $src$$constant);
 6110   %}
 6111   ins_pipe(ialu_mem_imm);
 6112 %}
 6113 
 6114 // Store Compressed Pointer
 6115 instruct storeN(memory mem, rRegN src)
 6116 %{
 6117   match(Set mem (StoreN mem src));
 6118 
 6119   ins_cost(125); // XXX
 6120   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6121   ins_encode %{
 6122     __ movl($mem$$Address, $src$$Register);
 6123   %}
 6124   ins_pipe(ialu_mem_reg);
 6125 %}
 6126 
 6127 instruct storeNKlass(memory mem, rRegN src)
 6128 %{
 6129   match(Set mem (StoreNKlass mem src));
 6130 
 6131   ins_cost(125); // XXX
 6132   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6133   ins_encode %{
 6134     __ movl($mem$$Address, $src$$Register);
 6135   %}
 6136   ins_pipe(ialu_mem_reg);
 6137 %}
 6138 
 6139 instruct storeImmN0(memory mem, immN0 zero)
 6140 %{
 6141   predicate(CompressedOops::base() == NULL);
 6142   match(Set mem (StoreN mem zero));
 6143 
 6144   ins_cost(125); // XXX
 6145   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6146   ins_encode %{
 6147     __ movl($mem$$Address, r12);
 6148   %}
 6149   ins_pipe(ialu_mem_reg);
 6150 %}
 6151 
 6152 instruct storeImmN(memory mem, immN src)
 6153 %{
 6154   match(Set mem (StoreN mem src));
 6155 
 6156   ins_cost(150); // XXX
 6157   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6158   ins_encode %{
 6159     address con = (address)$src$$constant;
 6160     if (con == NULL) {
 6161       __ movl($mem$$Address, (int32_t)0);
 6162     } else {
 6163       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6164     }
 6165   %}
 6166   ins_pipe(ialu_mem_imm);
 6167 %}
 6168 
 6169 instruct storeImmNKlass(memory mem, immNKlass src)
 6170 %{
 6171   match(Set mem (StoreNKlass mem src));
 6172 
 6173   ins_cost(150); // XXX
 6174   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6175   ins_encode %{
 6176     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6177   %}
 6178   ins_pipe(ialu_mem_imm);
 6179 %}
 6180 
 6181 // Store Integer Immediate
 6182 instruct storeImmI0(memory mem, immI_0 zero)
 6183 %{
 6184   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6185   match(Set mem (StoreI mem zero));
 6186 
 6187   ins_cost(125); // XXX
 6188   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6189   ins_encode %{
 6190     __ movl($mem$$Address, r12);
 6191   %}
 6192   ins_pipe(ialu_mem_reg);
 6193 %}
 6194 
 6195 instruct storeImmI(memory mem, immI src)
 6196 %{
 6197   match(Set mem (StoreI mem src));
 6198 
 6199   ins_cost(150);
 6200   format %{ "movl    $mem, $src\t# int" %}
 6201   ins_encode %{
 6202     __ movl($mem$$Address, $src$$constant);
 6203   %}
 6204   ins_pipe(ialu_mem_imm);
 6205 %}
 6206 
 6207 // Store Long Immediate
 6208 instruct storeImmL0(memory mem, immL0 zero)
 6209 %{
 6210   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6211   match(Set mem (StoreL mem zero));
 6212 
 6213   ins_cost(125); // XXX
 6214   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6215   ins_encode %{
 6216     __ movq($mem$$Address, r12);
 6217   %}
 6218   ins_pipe(ialu_mem_reg);
 6219 %}
 6220 
 6221 instruct storeImmL(memory mem, immL32 src)
 6222 %{
 6223   match(Set mem (StoreL mem src));
 6224 
 6225   ins_cost(150);
 6226   format %{ "movq    $mem, $src\t# long" %}
 6227   ins_encode %{
 6228     __ movq($mem$$Address, $src$$constant);
 6229   %}
 6230   ins_pipe(ialu_mem_imm);
 6231 %}
 6232 
 6233 // Store Short/Char Immediate
 6234 instruct storeImmC0(memory mem, immI_0 zero)
 6235 %{
 6236   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6237   match(Set mem (StoreC mem zero));
 6238 
 6239   ins_cost(125); // XXX
 6240   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6241   ins_encode %{
 6242     __ movw($mem$$Address, r12);
 6243   %}
 6244   ins_pipe(ialu_mem_reg);
 6245 %}
 6246 
 6247 instruct storeImmI16(memory mem, immI16 src)
 6248 %{
 6249   predicate(UseStoreImmI16);
 6250   match(Set mem (StoreC mem src));
 6251 
 6252   ins_cost(150);
 6253   format %{ "movw    $mem, $src\t# short/char" %}
 6254   ins_encode %{
 6255     __ movw($mem$$Address, $src$$constant);
 6256   %}
 6257   ins_pipe(ialu_mem_imm);
 6258 %}
 6259 
 6260 // Store Byte Immediate
 6261 instruct storeImmB0(memory mem, immI_0 zero)
 6262 %{
 6263   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6264   match(Set mem (StoreB mem zero));
 6265 
 6266   ins_cost(125); // XXX
 6267   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6268   ins_encode %{
 6269     __ movb($mem$$Address, r12);
 6270   %}
 6271   ins_pipe(ialu_mem_reg);
 6272 %}
 6273 
 6274 instruct storeImmB(memory mem, immI8 src)
 6275 %{
 6276   match(Set mem (StoreB mem src));
 6277 
 6278   ins_cost(150); // XXX
 6279   format %{ "movb    $mem, $src\t# byte" %}
 6280   ins_encode %{
 6281     __ movb($mem$$Address, $src$$constant);
 6282   %}
 6283   ins_pipe(ialu_mem_imm);
 6284 %}
 6285 
 6286 // Store CMS card-mark Immediate
 6287 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6288 %{
 6289   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6290   match(Set mem (StoreCM mem zero));
 6291 
 6292   ins_cost(125); // XXX
 6293   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6294   ins_encode %{
 6295     __ movb($mem$$Address, r12);
 6296   %}
 6297   ins_pipe(ialu_mem_reg);
 6298 %}
 6299 
 6300 instruct storeImmCM0(memory mem, immI_0 src)
 6301 %{
 6302   match(Set mem (StoreCM mem src));
 6303 
 6304   ins_cost(150); // XXX
 6305   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6306   ins_encode %{
 6307     __ movb($mem$$Address, $src$$constant);
 6308   %}
 6309   ins_pipe(ialu_mem_imm);
 6310 %}
 6311 
 6312 // Store Float
 6313 instruct storeF(memory mem, regF src)
 6314 %{
 6315   match(Set mem (StoreF mem src));
 6316 
 6317   ins_cost(95); // XXX
 6318   format %{ "movss   $mem, $src\t# float" %}
 6319   ins_encode %{
 6320     __ movflt($mem$$Address, $src$$XMMRegister);
 6321   %}
 6322   ins_pipe(pipe_slow); // XXX
 6323 %}
 6324 
 6325 // Store immediate Float value (it is faster than store from XMM register)
 6326 instruct storeF0(memory mem, immF0 zero)
 6327 %{
 6328   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6329   match(Set mem (StoreF mem zero));
 6330 
 6331   ins_cost(25); // XXX
 6332   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6333   ins_encode %{
 6334     __ movl($mem$$Address, r12);
 6335   %}
 6336   ins_pipe(ialu_mem_reg);
 6337 %}
 6338 
 6339 instruct storeF_imm(memory mem, immF src)
 6340 %{
 6341   match(Set mem (StoreF mem src));
 6342 
 6343   ins_cost(50);
 6344   format %{ "movl    $mem, $src\t# float" %}
 6345   ins_encode %{
 6346     __ movl($mem$$Address, jint_cast($src$$constant));
 6347   %}
 6348   ins_pipe(ialu_mem_imm);
 6349 %}
 6350 
 6351 // Store Double
 6352 instruct storeD(memory mem, regD src)
 6353 %{
 6354   match(Set mem (StoreD mem src));
 6355 
 6356   ins_cost(95); // XXX
 6357   format %{ "movsd   $mem, $src\t# double" %}
 6358   ins_encode %{
 6359     __ movdbl($mem$$Address, $src$$XMMRegister);
 6360   %}
 6361   ins_pipe(pipe_slow); // XXX
 6362 %}
 6363 
 6364 // Store immediate double 0.0 (it is faster than store from XMM register)
 6365 instruct storeD0_imm(memory mem, immD0 src)
 6366 %{
 6367   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6368   match(Set mem (StoreD mem src));
 6369 
 6370   ins_cost(50);
 6371   format %{ "movq    $mem, $src\t# double 0." %}
 6372   ins_encode %{
 6373     __ movq($mem$$Address, $src$$constant);
 6374   %}
 6375   ins_pipe(ialu_mem_imm);
 6376 %}
 6377 
 6378 instruct storeD0(memory mem, immD0 zero)
 6379 %{
 6380   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6381   match(Set mem (StoreD mem zero));
 6382 
 6383   ins_cost(25); // XXX
 6384   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6385   ins_encode %{
 6386     __ movq($mem$$Address, r12);
 6387   %}
 6388   ins_pipe(ialu_mem_reg);
 6389 %}
 6390 
 6391 instruct storeSSI(stackSlotI dst, rRegI src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movl    $dst, $src\t# int stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe( ialu_mem_reg );
 6400 %}
 6401 
 6402 instruct storeSSL(stackSlotL dst, rRegL src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(100);
 6407   format %{ "movq    $dst, $src\t# long stk" %}
 6408   opcode(0x89);
 6409   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6410   ins_pipe(ialu_mem_reg);
 6411 %}
 6412 
 6413 instruct storeSSP(stackSlotP dst, rRegP src)
 6414 %{
 6415   match(Set dst src);
 6416 
 6417   ins_cost(100);
 6418   format %{ "movq    $dst, $src\t# ptr stk" %}
 6419   opcode(0x89);
 6420   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6421   ins_pipe(ialu_mem_reg);
 6422 %}
 6423 
 6424 instruct storeSSF(stackSlotF dst, regF src)
 6425 %{
 6426   match(Set dst src);
 6427 
 6428   ins_cost(95); // XXX
 6429   format %{ "movss   $dst, $src\t# float stk" %}
 6430   ins_encode %{
 6431     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6432   %}
 6433   ins_pipe(pipe_slow); // XXX
 6434 %}
 6435 
 6436 instruct storeSSD(stackSlotD dst, regD src)
 6437 %{
 6438   match(Set dst src);
 6439 
 6440   ins_cost(95); // XXX
 6441   format %{ "movsd   $dst, $src\t# double stk" %}
 6442   ins_encode %{
 6443     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6444   %}
 6445   ins_pipe(pipe_slow); // XXX
 6446 %}
 6447 
 6448 instruct cacheWB(indirect addr)
 6449 %{
 6450   predicate(VM_Version::supports_data_cache_line_flush());
 6451   match(CacheWB addr);
 6452 
 6453   ins_cost(100);
 6454   format %{"cache wb $addr" %}
 6455   ins_encode %{
 6456     assert($addr->index_position() < 0, "should be");
 6457     assert($addr$$disp == 0, "should be");
 6458     __ cache_wb(Address($addr$$base$$Register, 0));
 6459   %}
 6460   ins_pipe(pipe_slow); // XXX
 6461 %}
 6462 
 6463 instruct cacheWBPreSync()
 6464 %{
 6465   predicate(VM_Version::supports_data_cache_line_flush());
 6466   match(CacheWBPreSync);
 6467 
 6468   ins_cost(100);
 6469   format %{"cache wb presync" %}
 6470   ins_encode %{
 6471     __ cache_wbsync(true);
 6472   %}
 6473   ins_pipe(pipe_slow); // XXX
 6474 %}
 6475 
 6476 instruct cacheWBPostSync()
 6477 %{
 6478   predicate(VM_Version::supports_data_cache_line_flush());
 6479   match(CacheWBPostSync);
 6480 
 6481   ins_cost(100);
 6482   format %{"cache wb postsync" %}
 6483   ins_encode %{
 6484     __ cache_wbsync(false);
 6485   %}
 6486   ins_pipe(pipe_slow); // XXX
 6487 %}
 6488 
 6489 //----------BSWAP Instructions-------------------------------------------------
 6490 instruct bytes_reverse_int(rRegI dst) %{
 6491   match(Set dst (ReverseBytesI dst));
 6492 
 6493   format %{ "bswapl  $dst" %}
 6494   ins_encode %{
 6495     __ bswapl($dst$$Register);
 6496   %}
 6497   ins_pipe( ialu_reg );
 6498 %}
 6499 
 6500 instruct bytes_reverse_long(rRegL dst) %{
 6501   match(Set dst (ReverseBytesL dst));
 6502 
 6503   format %{ "bswapq  $dst" %}
 6504   ins_encode %{
 6505     __ bswapq($dst$$Register);
 6506   %}
 6507   ins_pipe( ialu_reg);
 6508 %}
 6509 
 6510 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6511   match(Set dst (ReverseBytesUS dst));
 6512   effect(KILL cr);
 6513 
 6514   format %{ "bswapl  $dst\n\t"
 6515             "shrl    $dst,16\n\t" %}
 6516   ins_encode %{
 6517     __ bswapl($dst$$Register);
 6518     __ shrl($dst$$Register, 16);
 6519   %}
 6520   ins_pipe( ialu_reg );
 6521 %}
 6522 
 6523 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6524   match(Set dst (ReverseBytesS dst));
 6525   effect(KILL cr);
 6526 
 6527   format %{ "bswapl  $dst\n\t"
 6528             "sar     $dst,16\n\t" %}
 6529   ins_encode %{
 6530     __ bswapl($dst$$Register);
 6531     __ sarl($dst$$Register, 16);
 6532   %}
 6533   ins_pipe( ialu_reg );
 6534 %}
 6535 
 6536 //---------- Zeros Count Instructions ------------------------------------------
 6537 
 6538 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6539   predicate(UseCountLeadingZerosInstruction);
 6540   match(Set dst (CountLeadingZerosI src));
 6541   effect(KILL cr);
 6542 
 6543   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6544   ins_encode %{
 6545     __ lzcntl($dst$$Register, $src$$Register);
 6546   %}
 6547   ins_pipe(ialu_reg);
 6548 %}
 6549 
 6550 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6551   predicate(UseCountLeadingZerosInstruction);
 6552   match(Set dst (CountLeadingZerosI (LoadI src)));
 6553   effect(KILL cr);
 6554   ins_cost(175);
 6555   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6556   ins_encode %{
 6557     __ lzcntl($dst$$Register, $src$$Address);
 6558   %}
 6559   ins_pipe(ialu_reg_mem);
 6560 %}
 6561 
 6562 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6563   predicate(!UseCountLeadingZerosInstruction);
 6564   match(Set dst (CountLeadingZerosI src));
 6565   effect(KILL cr);
 6566 
 6567   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6568             "jnz     skip\n\t"
 6569             "movl    $dst, -1\n"
 6570       "skip:\n\t"
 6571             "negl    $dst\n\t"
 6572             "addl    $dst, 31" %}
 6573   ins_encode %{
 6574     Register Rdst = $dst$$Register;
 6575     Register Rsrc = $src$$Register;
 6576     Label skip;
 6577     __ bsrl(Rdst, Rsrc);
 6578     __ jccb(Assembler::notZero, skip);
 6579     __ movl(Rdst, -1);
 6580     __ bind(skip);
 6581     __ negl(Rdst);
 6582     __ addl(Rdst, BitsPerInt - 1);
 6583   %}
 6584   ins_pipe(ialu_reg);
 6585 %}
 6586 
 6587 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6588   predicate(UseCountLeadingZerosInstruction);
 6589   match(Set dst (CountLeadingZerosL src));
 6590   effect(KILL cr);
 6591 
 6592   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6593   ins_encode %{
 6594     __ lzcntq($dst$$Register, $src$$Register);
 6595   %}
 6596   ins_pipe(ialu_reg);
 6597 %}
 6598 
 6599 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6600   predicate(UseCountLeadingZerosInstruction);
 6601   match(Set dst (CountLeadingZerosL (LoadL src)));
 6602   effect(KILL cr);
 6603   ins_cost(175);
 6604   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6605   ins_encode %{
 6606     __ lzcntq($dst$$Register, $src$$Address);
 6607   %}
 6608   ins_pipe(ialu_reg_mem);
 6609 %}
 6610 
 6611 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6612   predicate(!UseCountLeadingZerosInstruction);
 6613   match(Set dst (CountLeadingZerosL src));
 6614   effect(KILL cr);
 6615 
 6616   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6617             "jnz     skip\n\t"
 6618             "movl    $dst, -1\n"
 6619       "skip:\n\t"
 6620             "negl    $dst\n\t"
 6621             "addl    $dst, 63" %}
 6622   ins_encode %{
 6623     Register Rdst = $dst$$Register;
 6624     Register Rsrc = $src$$Register;
 6625     Label skip;
 6626     __ bsrq(Rdst, Rsrc);
 6627     __ jccb(Assembler::notZero, skip);
 6628     __ movl(Rdst, -1);
 6629     __ bind(skip);
 6630     __ negl(Rdst);
 6631     __ addl(Rdst, BitsPerLong - 1);
 6632   %}
 6633   ins_pipe(ialu_reg);
 6634 %}
 6635 
 6636 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6637   predicate(UseCountTrailingZerosInstruction);
 6638   match(Set dst (CountTrailingZerosI src));
 6639   effect(KILL cr);
 6640 
 6641   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6642   ins_encode %{
 6643     __ tzcntl($dst$$Register, $src$$Register);
 6644   %}
 6645   ins_pipe(ialu_reg);
 6646 %}
 6647 
 6648 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6649   predicate(UseCountTrailingZerosInstruction);
 6650   match(Set dst (CountTrailingZerosI (LoadI src)));
 6651   effect(KILL cr);
 6652   ins_cost(175);
 6653   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6654   ins_encode %{
 6655     __ tzcntl($dst$$Register, $src$$Address);
 6656   %}
 6657   ins_pipe(ialu_reg_mem);
 6658 %}
 6659 
 6660 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6661   predicate(!UseCountTrailingZerosInstruction);
 6662   match(Set dst (CountTrailingZerosI src));
 6663   effect(KILL cr);
 6664 
 6665   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6666             "jnz     done\n\t"
 6667             "movl    $dst, 32\n"
 6668       "done:" %}
 6669   ins_encode %{
 6670     Register Rdst = $dst$$Register;
 6671     Label done;
 6672     __ bsfl(Rdst, $src$$Register);
 6673     __ jccb(Assembler::notZero, done);
 6674     __ movl(Rdst, BitsPerInt);
 6675     __ bind(done);
 6676   %}
 6677   ins_pipe(ialu_reg);
 6678 %}
 6679 
 6680 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6681   predicate(UseCountTrailingZerosInstruction);
 6682   match(Set dst (CountTrailingZerosL src));
 6683   effect(KILL cr);
 6684 
 6685   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6686   ins_encode %{
 6687     __ tzcntq($dst$$Register, $src$$Register);
 6688   %}
 6689   ins_pipe(ialu_reg);
 6690 %}
 6691 
 6692 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6693   predicate(UseCountTrailingZerosInstruction);
 6694   match(Set dst (CountTrailingZerosL (LoadL src)));
 6695   effect(KILL cr);
 6696   ins_cost(175);
 6697   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6698   ins_encode %{
 6699     __ tzcntq($dst$$Register, $src$$Address);
 6700   %}
 6701   ins_pipe(ialu_reg_mem);
 6702 %}
 6703 
 6704 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6705   predicate(!UseCountTrailingZerosInstruction);
 6706   match(Set dst (CountTrailingZerosL src));
 6707   effect(KILL cr);
 6708 
 6709   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6710             "jnz     done\n\t"
 6711             "movl    $dst, 64\n"
 6712       "done:" %}
 6713   ins_encode %{
 6714     Register Rdst = $dst$$Register;
 6715     Label done;
 6716     __ bsfq(Rdst, $src$$Register);
 6717     __ jccb(Assembler::notZero, done);
 6718     __ movl(Rdst, BitsPerLong);
 6719     __ bind(done);
 6720   %}
 6721   ins_pipe(ialu_reg);
 6722 %}
 6723 
 6724 //--------------- Reverse Operation Instructions ----------------
 6725 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 6726   predicate(!VM_Version::supports_gfni());
 6727   match(Set dst (ReverseI src));
 6728   effect(TEMP dst, TEMP rtmp, KILL cr);
 6729   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 6730   ins_encode %{
 6731     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 6732   %}
 6733   ins_pipe( ialu_reg );
 6734 %}
 6735 
 6736 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6737   predicate(VM_Version::supports_gfni());
 6738   match(Set dst (ReverseI src));
 6739   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6740   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6741   ins_encode %{
 6742     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 6743   %}
 6744   ins_pipe( ialu_reg );
 6745 %}
 6746 
 6747 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 6748   predicate(!VM_Version::supports_gfni());
 6749   match(Set dst (ReverseL src));
 6750   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 6751   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 6752   ins_encode %{
 6753     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 6754   %}
 6755   ins_pipe( ialu_reg );
 6756 %}
 6757 
 6758 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6759   predicate(VM_Version::supports_gfni());
 6760   match(Set dst (ReverseL src));
 6761   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6762   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6763   ins_encode %{
 6764     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 6765   %}
 6766   ins_pipe( ialu_reg );
 6767 %}
 6768 
 6769 //---------- Population Count Instructions -------------------------------------
 6770 
 6771 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6772   predicate(UsePopCountInstruction);
 6773   match(Set dst (PopCountI src));
 6774   effect(KILL cr);
 6775 
 6776   format %{ "popcnt  $dst, $src" %}
 6777   ins_encode %{
 6778     __ popcntl($dst$$Register, $src$$Register);
 6779   %}
 6780   ins_pipe(ialu_reg);
 6781 %}
 6782 
 6783 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6784   predicate(UsePopCountInstruction);
 6785   match(Set dst (PopCountI (LoadI mem)));
 6786   effect(KILL cr);
 6787 
 6788   format %{ "popcnt  $dst, $mem" %}
 6789   ins_encode %{
 6790     __ popcntl($dst$$Register, $mem$$Address);
 6791   %}
 6792   ins_pipe(ialu_reg);
 6793 %}
 6794 
 6795 // Note: Long.bitCount(long) returns an int.
 6796 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6797   predicate(UsePopCountInstruction);
 6798   match(Set dst (PopCountL src));
 6799   effect(KILL cr);
 6800 
 6801   format %{ "popcnt  $dst, $src" %}
 6802   ins_encode %{
 6803     __ popcntq($dst$$Register, $src$$Register);
 6804   %}
 6805   ins_pipe(ialu_reg);
 6806 %}
 6807 
 6808 // Note: Long.bitCount(long) returns an int.
 6809 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6810   predicate(UsePopCountInstruction);
 6811   match(Set dst (PopCountL (LoadL mem)));
 6812   effect(KILL cr);
 6813 
 6814   format %{ "popcnt  $dst, $mem" %}
 6815   ins_encode %{
 6816     __ popcntq($dst$$Register, $mem$$Address);
 6817   %}
 6818   ins_pipe(ialu_reg);
 6819 %}
 6820 
 6821 
 6822 //----------MemBar Instructions-----------------------------------------------
 6823 // Memory barrier flavors
 6824 
 6825 instruct membar_acquire()
 6826 %{
 6827   match(MemBarAcquire);
 6828   match(LoadFence);
 6829   ins_cost(0);
 6830 
 6831   size(0);
 6832   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6833   ins_encode();
 6834   ins_pipe(empty);
 6835 %}
 6836 
 6837 instruct membar_acquire_lock()
 6838 %{
 6839   match(MemBarAcquireLock);
 6840   ins_cost(0);
 6841 
 6842   size(0);
 6843   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6844   ins_encode();
 6845   ins_pipe(empty);
 6846 %}
 6847 
 6848 instruct membar_release()
 6849 %{
 6850   match(MemBarRelease);
 6851   match(StoreFence);
 6852   ins_cost(0);
 6853 
 6854   size(0);
 6855   format %{ "MEMBAR-release ! (empty encoding)" %}
 6856   ins_encode();
 6857   ins_pipe(empty);
 6858 %}
 6859 
 6860 instruct membar_release_lock()
 6861 %{
 6862   match(MemBarReleaseLock);
 6863   ins_cost(0);
 6864 
 6865   size(0);
 6866   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6867   ins_encode();
 6868   ins_pipe(empty);
 6869 %}
 6870 
 6871 instruct membar_volatile(rFlagsReg cr) %{
 6872   match(MemBarVolatile);
 6873   effect(KILL cr);
 6874   ins_cost(400);
 6875 
 6876   format %{
 6877     $$template
 6878     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6879   %}
 6880   ins_encode %{
 6881     __ membar(Assembler::StoreLoad);
 6882   %}
 6883   ins_pipe(pipe_slow);
 6884 %}
 6885 
 6886 instruct unnecessary_membar_volatile()
 6887 %{
 6888   match(MemBarVolatile);
 6889   predicate(Matcher::post_store_load_barrier(n));
 6890   ins_cost(0);
 6891 
 6892   size(0);
 6893   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6894   ins_encode();
 6895   ins_pipe(empty);
 6896 %}
 6897 
 6898 instruct membar_storestore() %{
 6899   match(MemBarStoreStore);
 6900   match(StoreStoreFence);
 6901   ins_cost(0);
 6902 
 6903   size(0);
 6904   format %{ "MEMBAR-storestore (empty encoding)" %}
 6905   ins_encode( );
 6906   ins_pipe(empty);
 6907 %}
 6908 
 6909 //----------Move Instructions--------------------------------------------------
 6910 
 6911 instruct castX2P(rRegP dst, rRegL src)
 6912 %{
 6913   match(Set dst (CastX2P src));
 6914 
 6915   format %{ "movq    $dst, $src\t# long->ptr" %}
 6916   ins_encode %{
 6917     if ($dst$$reg != $src$$reg) {
 6918       __ movptr($dst$$Register, $src$$Register);
 6919     }
 6920   %}
 6921   ins_pipe(ialu_reg_reg); // XXX
 6922 %}
 6923 
 6924 instruct castP2X(rRegL dst, rRegP src)
 6925 %{
 6926   match(Set dst (CastP2X src));
 6927 
 6928   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6929   ins_encode %{
 6930     if ($dst$$reg != $src$$reg) {
 6931       __ movptr($dst$$Register, $src$$Register);
 6932     }
 6933   %}
 6934   ins_pipe(ialu_reg_reg); // XXX
 6935 %}
 6936 
 6937 // Convert oop into int for vectors alignment masking
 6938 instruct convP2I(rRegI dst, rRegP src)
 6939 %{
 6940   match(Set dst (ConvL2I (CastP2X src)));
 6941 
 6942   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6943   ins_encode %{
 6944     __ movl($dst$$Register, $src$$Register);
 6945   %}
 6946   ins_pipe(ialu_reg_reg); // XXX
 6947 %}
 6948 
 6949 // Convert compressed oop into int for vectors alignment masking
 6950 // in case of 32bit oops (heap < 4Gb).
 6951 instruct convN2I(rRegI dst, rRegN src)
 6952 %{
 6953   predicate(CompressedOops::shift() == 0);
 6954   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6955 
 6956   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6957   ins_encode %{
 6958     __ movl($dst$$Register, $src$$Register);
 6959   %}
 6960   ins_pipe(ialu_reg_reg); // XXX
 6961 %}
 6962 
 6963 // Convert oop pointer into compressed form
 6964 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6965   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6966   match(Set dst (EncodeP src));
 6967   effect(KILL cr);
 6968   format %{ "encode_heap_oop $dst,$src" %}
 6969   ins_encode %{
 6970     Register s = $src$$Register;
 6971     Register d = $dst$$Register;
 6972     if (s != d) {
 6973       __ movq(d, s);
 6974     }
 6975     __ encode_heap_oop(d);
 6976   %}
 6977   ins_pipe(ialu_reg_long);
 6978 %}
 6979 
 6980 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6981   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6982   match(Set dst (EncodeP src));
 6983   effect(KILL cr);
 6984   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6985   ins_encode %{
 6986     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6987   %}
 6988   ins_pipe(ialu_reg_long);
 6989 %}
 6990 
 6991 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6992   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6993             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6994   match(Set dst (DecodeN src));
 6995   effect(KILL cr);
 6996   format %{ "decode_heap_oop $dst,$src" %}
 6997   ins_encode %{
 6998     Register s = $src$$Register;
 6999     Register d = $dst$$Register;
 7000     if (s != d) {
 7001       __ movq(d, s);
 7002     }
 7003     __ decode_heap_oop(d);
 7004   %}
 7005   ins_pipe(ialu_reg_long);
 7006 %}
 7007 
 7008 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7009   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 7010             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7011   match(Set dst (DecodeN src));
 7012   effect(KILL cr);
 7013   format %{ "decode_heap_oop_not_null $dst,$src" %}
 7014   ins_encode %{
 7015     Register s = $src$$Register;
 7016     Register d = $dst$$Register;
 7017     if (s != d) {
 7018       __ decode_heap_oop_not_null(d, s);
 7019     } else {
 7020       __ decode_heap_oop_not_null(d);
 7021     }
 7022   %}
 7023   ins_pipe(ialu_reg_long);
 7024 %}
 7025 
 7026 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7027   match(Set dst (EncodePKlass src));
 7028   effect(TEMP dst, KILL cr);
 7029   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7030   ins_encode %{
 7031     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7032   %}
 7033   ins_pipe(ialu_reg_long);
 7034 %}
 7035 
 7036 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7037   match(Set dst (DecodeNKlass src));
 7038   effect(TEMP dst, KILL cr);
 7039   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7040   ins_encode %{
 7041     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7042   %}
 7043   ins_pipe(ialu_reg_long);
 7044 %}
 7045 
 7046 //----------Conditional Move---------------------------------------------------
 7047 // Jump
 7048 // dummy instruction for generating temp registers
 7049 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7050   match(Jump (LShiftL switch_val shift));
 7051   ins_cost(350);
 7052   predicate(false);
 7053   effect(TEMP dest);
 7054 
 7055   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7056             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7057   ins_encode %{
 7058     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7059     // to do that and the compiler is using that register as one it can allocate.
 7060     // So we build it all by hand.
 7061     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7062     // ArrayAddress dispatch(table, index);
 7063     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7064     __ lea($dest$$Register, $constantaddress);
 7065     __ jmp(dispatch);
 7066   %}
 7067   ins_pipe(pipe_jmp);
 7068 %}
 7069 
 7070 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7071   match(Jump (AddL (LShiftL switch_val shift) offset));
 7072   ins_cost(350);
 7073   effect(TEMP dest);
 7074 
 7075   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7076             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7077   ins_encode %{
 7078     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7079     // to do that and the compiler is using that register as one it can allocate.
 7080     // So we build it all by hand.
 7081     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7082     // ArrayAddress dispatch(table, index);
 7083     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7084     __ lea($dest$$Register, $constantaddress);
 7085     __ jmp(dispatch);
 7086   %}
 7087   ins_pipe(pipe_jmp);
 7088 %}
 7089 
 7090 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7091   match(Jump switch_val);
 7092   ins_cost(350);
 7093   effect(TEMP dest);
 7094 
 7095   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7096             "jmp     [$dest + $switch_val]\n\t" %}
 7097   ins_encode %{
 7098     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7099     // to do that and the compiler is using that register as one it can allocate.
 7100     // So we build it all by hand.
 7101     // Address index(noreg, switch_reg, Address::times_1);
 7102     // ArrayAddress dispatch(table, index);
 7103     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7104     __ lea($dest$$Register, $constantaddress);
 7105     __ jmp(dispatch);
 7106   %}
 7107   ins_pipe(pipe_jmp);
 7108 %}
 7109 
 7110 // Conditional move
 7111 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7112 %{
 7113   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7114   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7115 
 7116   ins_cost(100); // XXX
 7117   format %{ "setbn$cop $dst\t# signed, int" %}
 7118   ins_encode %{
 7119     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7120     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7121   %}
 7122   ins_pipe(ialu_reg);
 7123 %}
 7124 
 7125 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7126 %{
 7127   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7128 
 7129   ins_cost(200); // XXX
 7130   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7131   ins_encode %{
 7132     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7133   %}
 7134   ins_pipe(pipe_cmov_reg);
 7135 %}
 7136 
 7137 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7138 %{
 7139   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7140   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7141 
 7142   ins_cost(100); // XXX
 7143   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7144   ins_encode %{
 7145     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7146     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7147   %}
 7148   ins_pipe(ialu_reg);
 7149 %}
 7150 
 7151 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7152   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7153 
 7154   ins_cost(200); // XXX
 7155   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7156   ins_encode %{
 7157     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7158   %}
 7159   ins_pipe(pipe_cmov_reg);
 7160 %}
 7161 
 7162 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7163 %{
 7164   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7165   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7166 
 7167   ins_cost(100); // XXX
 7168   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7169   ins_encode %{
 7170     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7171     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7172   %}
 7173   ins_pipe(ialu_reg);
 7174 %}
 7175 
 7176 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7177   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7178   ins_cost(200);
 7179   expand %{
 7180     cmovI_regU(cop, cr, dst, src);
 7181   %}
 7182 %}
 7183 
 7184 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7185   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7186   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7187 
 7188   ins_cost(200); // XXX
 7189   format %{ "cmovpl  $dst, $src\n\t"
 7190             "cmovnel $dst, $src" %}
 7191   ins_encode %{
 7192     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7193     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7194   %}
 7195   ins_pipe(pipe_cmov_reg);
 7196 %}
 7197 
 7198 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7199 // inputs of the CMove
 7200 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7201   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7202   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7203 
 7204   ins_cost(200); // XXX
 7205   format %{ "cmovpl  $dst, $src\n\t"
 7206             "cmovnel $dst, $src" %}
 7207   ins_encode %{
 7208     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7209     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7210   %}
 7211   ins_pipe(pipe_cmov_reg);
 7212 %}
 7213 
 7214 // Conditional move
 7215 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7216   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7217 
 7218   ins_cost(250); // XXX
 7219   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7220   ins_encode %{
 7221     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7222   %}
 7223   ins_pipe(pipe_cmov_mem);
 7224 %}
 7225 
 7226 // Conditional move
 7227 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7228 %{
 7229   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7230 
 7231   ins_cost(250); // XXX
 7232   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7233   ins_encode %{
 7234     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7235   %}
 7236   ins_pipe(pipe_cmov_mem);
 7237 %}
 7238 
 7239 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7240   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7241   ins_cost(250);
 7242   expand %{
 7243     cmovI_memU(cop, cr, dst, src);
 7244   %}
 7245 %}
 7246 
 7247 // Conditional move
 7248 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7249 %{
 7250   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7251 
 7252   ins_cost(200); // XXX
 7253   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7254   ins_encode %{
 7255     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7256   %}
 7257   ins_pipe(pipe_cmov_reg);
 7258 %}
 7259 
 7260 // Conditional move
 7261 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7262 %{
 7263   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7264 
 7265   ins_cost(200); // XXX
 7266   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7267   ins_encode %{
 7268     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7269   %}
 7270   ins_pipe(pipe_cmov_reg);
 7271 %}
 7272 
 7273 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7274   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7275   ins_cost(200);
 7276   expand %{
 7277     cmovN_regU(cop, cr, dst, src);
 7278   %}
 7279 %}
 7280 
 7281 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7282   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7283   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7284 
 7285   ins_cost(200); // XXX
 7286   format %{ "cmovpl  $dst, $src\n\t"
 7287             "cmovnel $dst, $src" %}
 7288   ins_encode %{
 7289     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7290     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7291   %}
 7292   ins_pipe(pipe_cmov_reg);
 7293 %}
 7294 
 7295 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7296 // inputs of the CMove
 7297 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7298   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7299   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7300 
 7301   ins_cost(200); // XXX
 7302   format %{ "cmovpl  $dst, $src\n\t"
 7303             "cmovnel $dst, $src" %}
 7304   ins_encode %{
 7305     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7306     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7307   %}
 7308   ins_pipe(pipe_cmov_reg);
 7309 %}
 7310 
 7311 // Conditional move
 7312 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7313 %{
 7314   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7315 
 7316   ins_cost(200); // XXX
 7317   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7318   ins_encode %{
 7319     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7320   %}
 7321   ins_pipe(pipe_cmov_reg);  // XXX
 7322 %}
 7323 
 7324 // Conditional move
 7325 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7326 %{
 7327   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7328 
 7329   ins_cost(200); // XXX
 7330   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7331   ins_encode %{
 7332     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7333   %}
 7334   ins_pipe(pipe_cmov_reg); // XXX
 7335 %}
 7336 
 7337 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7338   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7339   ins_cost(200);
 7340   expand %{
 7341     cmovP_regU(cop, cr, dst, src);
 7342   %}
 7343 %}
 7344 
 7345 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7346   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7347   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7348 
 7349   ins_cost(200); // XXX
 7350   format %{ "cmovpq  $dst, $src\n\t"
 7351             "cmovneq $dst, $src" %}
 7352   ins_encode %{
 7353     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7354     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7355   %}
 7356   ins_pipe(pipe_cmov_reg);
 7357 %}
 7358 
 7359 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7360 // inputs of the CMove
 7361 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7362   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7363   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7364 
 7365   ins_cost(200); // XXX
 7366   format %{ "cmovpq  $dst, $src\n\t"
 7367             "cmovneq $dst, $src" %}
 7368   ins_encode %{
 7369     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7370     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7371   %}
 7372   ins_pipe(pipe_cmov_reg);
 7373 %}
 7374 
 7375 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7376 // correctly meets the two pointer arguments; one is an incoming
 7377 // register but the other is a memory operand.  ALSO appears to
 7378 // be buggy with implicit null checks.
 7379 //
 7380 //// Conditional move
 7381 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7382 //%{
 7383 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7384 //  ins_cost(250);
 7385 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7386 //  opcode(0x0F,0x40);
 7387 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7388 //  ins_pipe( pipe_cmov_mem );
 7389 //%}
 7390 //
 7391 //// Conditional move
 7392 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7393 //%{
 7394 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7395 //  ins_cost(250);
 7396 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7397 //  opcode(0x0F,0x40);
 7398 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7399 //  ins_pipe( pipe_cmov_mem );
 7400 //%}
 7401 
 7402 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7403 %{
 7404   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7405   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7406 
 7407   ins_cost(100); // XXX
 7408   format %{ "setbn$cop $dst\t# signed, long" %}
 7409   ins_encode %{
 7410     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7411     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7412   %}
 7413   ins_pipe(ialu_reg);
 7414 %}
 7415 
 7416 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7417 %{
 7418   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7419 
 7420   ins_cost(200); // XXX
 7421   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7422   ins_encode %{
 7423     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7424   %}
 7425   ins_pipe(pipe_cmov_reg);  // XXX
 7426 %}
 7427 
 7428 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7429 %{
 7430   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7431 
 7432   ins_cost(200); // XXX
 7433   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7434   ins_encode %{
 7435     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7436   %}
 7437   ins_pipe(pipe_cmov_mem);  // XXX
 7438 %}
 7439 
 7440 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7441 %{
 7442   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7443   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7444 
 7445   ins_cost(100); // XXX
 7446   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7447   ins_encode %{
 7448     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7449     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7450   %}
 7451   ins_pipe(ialu_reg);
 7452 %}
 7453 
 7454 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7455 %{
 7456   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7457 
 7458   ins_cost(200); // XXX
 7459   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7460   ins_encode %{
 7461     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7462   %}
 7463   ins_pipe(pipe_cmov_reg); // XXX
 7464 %}
 7465 
 7466 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7467 %{
 7468   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7469   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7470 
 7471   ins_cost(100); // XXX
 7472   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7473   ins_encode %{
 7474     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7475     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7476   %}
 7477   ins_pipe(ialu_reg);
 7478 %}
 7479 
 7480 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7481   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7482   ins_cost(200);
 7483   expand %{
 7484     cmovL_regU(cop, cr, dst, src);
 7485   %}
 7486 %}
 7487 
 7488 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7489   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7490   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7491 
 7492   ins_cost(200); // XXX
 7493   format %{ "cmovpq  $dst, $src\n\t"
 7494             "cmovneq $dst, $src" %}
 7495   ins_encode %{
 7496     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7497     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7498   %}
 7499   ins_pipe(pipe_cmov_reg);
 7500 %}
 7501 
 7502 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7503 // inputs of the CMove
 7504 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7505   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7506   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7507 
 7508   ins_cost(200); // XXX
 7509   format %{ "cmovpq  $dst, $src\n\t"
 7510             "cmovneq $dst, $src" %}
 7511   ins_encode %{
 7512     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7513     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7514   %}
 7515   ins_pipe(pipe_cmov_reg);
 7516 %}
 7517 
 7518 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7519 %{
 7520   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7521 
 7522   ins_cost(200); // XXX
 7523   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7524   ins_encode %{
 7525     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7526   %}
 7527   ins_pipe(pipe_cmov_mem); // XXX
 7528 %}
 7529 
 7530 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7531   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7532   ins_cost(200);
 7533   expand %{
 7534     cmovL_memU(cop, cr, dst, src);
 7535   %}
 7536 %}
 7537 
 7538 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7539 %{
 7540   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7541 
 7542   ins_cost(200); // XXX
 7543   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7544             "movss     $dst, $src\n"
 7545     "skip:" %}
 7546   ins_encode %{
 7547     Label Lskip;
 7548     // Invert sense of branch from sense of CMOV
 7549     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7550     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7551     __ bind(Lskip);
 7552   %}
 7553   ins_pipe(pipe_slow);
 7554 %}
 7555 
 7556 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7557 // %{
 7558 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7559 
 7560 //   ins_cost(200); // XXX
 7561 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7562 //             "movss     $dst, $src\n"
 7563 //     "skip:" %}
 7564 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7565 //   ins_pipe(pipe_slow);
 7566 // %}
 7567 
 7568 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7569 %{
 7570   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7571 
 7572   ins_cost(200); // XXX
 7573   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7574             "movss     $dst, $src\n"
 7575     "skip:" %}
 7576   ins_encode %{
 7577     Label Lskip;
 7578     // Invert sense of branch from sense of CMOV
 7579     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7580     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7581     __ bind(Lskip);
 7582   %}
 7583   ins_pipe(pipe_slow);
 7584 %}
 7585 
 7586 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7587   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7588   ins_cost(200);
 7589   expand %{
 7590     cmovF_regU(cop, cr, dst, src);
 7591   %}
 7592 %}
 7593 
 7594 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7595 %{
 7596   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7597 
 7598   ins_cost(200); // XXX
 7599   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7600             "movsd     $dst, $src\n"
 7601     "skip:" %}
 7602   ins_encode %{
 7603     Label Lskip;
 7604     // Invert sense of branch from sense of CMOV
 7605     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7606     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7607     __ bind(Lskip);
 7608   %}
 7609   ins_pipe(pipe_slow);
 7610 %}
 7611 
 7612 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7613 %{
 7614   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7615 
 7616   ins_cost(200); // XXX
 7617   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7618             "movsd     $dst, $src\n"
 7619     "skip:" %}
 7620   ins_encode %{
 7621     Label Lskip;
 7622     // Invert sense of branch from sense of CMOV
 7623     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7624     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7625     __ bind(Lskip);
 7626   %}
 7627   ins_pipe(pipe_slow);
 7628 %}
 7629 
 7630 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7631   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7632   ins_cost(200);
 7633   expand %{
 7634     cmovD_regU(cop, cr, dst, src);
 7635   %}
 7636 %}
 7637 
 7638 //----------Arithmetic Instructions--------------------------------------------
 7639 //----------Addition Instructions----------------------------------------------
 7640 
 7641 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7642 %{
 7643   match(Set dst (AddI dst src));
 7644   effect(KILL cr);
 7645 
 7646   format %{ "addl    $dst, $src\t# int" %}
 7647   ins_encode %{
 7648     __ addl($dst$$Register, $src$$Register);
 7649   %}
 7650   ins_pipe(ialu_reg_reg);
 7651 %}
 7652 
 7653 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7654 %{
 7655   match(Set dst (AddI dst src));
 7656   effect(KILL cr);
 7657 
 7658   format %{ "addl    $dst, $src\t# int" %}
 7659   ins_encode %{
 7660     __ addl($dst$$Register, $src$$constant);
 7661   %}
 7662   ins_pipe( ialu_reg );
 7663 %}
 7664 
 7665 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7666 %{
 7667   match(Set dst (AddI dst (LoadI src)));
 7668   effect(KILL cr);
 7669 
 7670   ins_cost(125); // XXX
 7671   format %{ "addl    $dst, $src\t# int" %}
 7672   ins_encode %{
 7673     __ addl($dst$$Register, $src$$Address);
 7674   %}
 7675   ins_pipe(ialu_reg_mem);
 7676 %}
 7677 
 7678 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7679 %{
 7680   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7681   effect(KILL cr);
 7682 
 7683   ins_cost(150); // XXX
 7684   format %{ "addl    $dst, $src\t# int" %}
 7685   ins_encode %{
 7686     __ addl($dst$$Address, $src$$Register);
 7687   %}
 7688   ins_pipe(ialu_mem_reg);
 7689 %}
 7690 
 7691 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7692 %{
 7693   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7694   effect(KILL cr);
 7695 
 7696   ins_cost(125); // XXX
 7697   format %{ "addl    $dst, $src\t# int" %}
 7698   ins_encode %{
 7699     __ addl($dst$$Address, $src$$constant);
 7700   %}
 7701   ins_pipe(ialu_mem_imm);
 7702 %}
 7703 
 7704 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7705 %{
 7706   predicate(UseIncDec);
 7707   match(Set dst (AddI dst src));
 7708   effect(KILL cr);
 7709 
 7710   format %{ "incl    $dst\t# int" %}
 7711   ins_encode %{
 7712     __ incrementl($dst$$Register);
 7713   %}
 7714   ins_pipe(ialu_reg);
 7715 %}
 7716 
 7717 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7718 %{
 7719   predicate(UseIncDec);
 7720   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7721   effect(KILL cr);
 7722 
 7723   ins_cost(125); // XXX
 7724   format %{ "incl    $dst\t# int" %}
 7725   ins_encode %{
 7726     __ incrementl($dst$$Address);
 7727   %}
 7728   ins_pipe(ialu_mem_imm);
 7729 %}
 7730 
 7731 // XXX why does that use AddI
 7732 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7733 %{
 7734   predicate(UseIncDec);
 7735   match(Set dst (AddI dst src));
 7736   effect(KILL cr);
 7737 
 7738   format %{ "decl    $dst\t# int" %}
 7739   ins_encode %{
 7740     __ decrementl($dst$$Register);
 7741   %}
 7742   ins_pipe(ialu_reg);
 7743 %}
 7744 
 7745 // XXX why does that use AddI
 7746 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7747 %{
 7748   predicate(UseIncDec);
 7749   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7750   effect(KILL cr);
 7751 
 7752   ins_cost(125); // XXX
 7753   format %{ "decl    $dst\t# int" %}
 7754   ins_encode %{
 7755     __ decrementl($dst$$Address);
 7756   %}
 7757   ins_pipe(ialu_mem_imm);
 7758 %}
 7759 
 7760 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7761 %{
 7762   predicate(VM_Version::supports_fast_2op_lea());
 7763   match(Set dst (AddI (LShiftI index scale) disp));
 7764 
 7765   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7766   ins_encode %{
 7767     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7768     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7769   %}
 7770   ins_pipe(ialu_reg_reg);
 7771 %}
 7772 
 7773 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7774 %{
 7775   predicate(VM_Version::supports_fast_3op_lea());
 7776   match(Set dst (AddI (AddI base index) disp));
 7777 
 7778   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7779   ins_encode %{
 7780     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7781   %}
 7782   ins_pipe(ialu_reg_reg);
 7783 %}
 7784 
 7785 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7786 %{
 7787   predicate(VM_Version::supports_fast_2op_lea());
 7788   match(Set dst (AddI base (LShiftI index scale)));
 7789 
 7790   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7791   ins_encode %{
 7792     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7793     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7794   %}
 7795   ins_pipe(ialu_reg_reg);
 7796 %}
 7797 
 7798 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7799 %{
 7800   predicate(VM_Version::supports_fast_3op_lea());
 7801   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7802 
 7803   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7804   ins_encode %{
 7805     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7806     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7807   %}
 7808   ins_pipe(ialu_reg_reg);
 7809 %}
 7810 
 7811 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7812 %{
 7813   match(Set dst (AddL dst src));
 7814   effect(KILL cr);
 7815 
 7816   format %{ "addq    $dst, $src\t# long" %}
 7817   ins_encode %{
 7818     __ addq($dst$$Register, $src$$Register);
 7819   %}
 7820   ins_pipe(ialu_reg_reg);
 7821 %}
 7822 
 7823 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7824 %{
 7825   match(Set dst (AddL dst src));
 7826   effect(KILL cr);
 7827 
 7828   format %{ "addq    $dst, $src\t# long" %}
 7829   ins_encode %{
 7830     __ addq($dst$$Register, $src$$constant);
 7831   %}
 7832   ins_pipe( ialu_reg );
 7833 %}
 7834 
 7835 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7836 %{
 7837   match(Set dst (AddL dst (LoadL src)));
 7838   effect(KILL cr);
 7839 
 7840   ins_cost(125); // XXX
 7841   format %{ "addq    $dst, $src\t# long" %}
 7842   ins_encode %{
 7843     __ addq($dst$$Register, $src$$Address);
 7844   %}
 7845   ins_pipe(ialu_reg_mem);
 7846 %}
 7847 
 7848 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7849 %{
 7850   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7851   effect(KILL cr);
 7852 
 7853   ins_cost(150); // XXX
 7854   format %{ "addq    $dst, $src\t# long" %}
 7855   ins_encode %{
 7856     __ addq($dst$$Address, $src$$Register);
 7857   %}
 7858   ins_pipe(ialu_mem_reg);
 7859 %}
 7860 
 7861 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7862 %{
 7863   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7864   effect(KILL cr);
 7865 
 7866   ins_cost(125); // XXX
 7867   format %{ "addq    $dst, $src\t# long" %}
 7868   ins_encode %{
 7869     __ addq($dst$$Address, $src$$constant);
 7870   %}
 7871   ins_pipe(ialu_mem_imm);
 7872 %}
 7873 
 7874 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7875 %{
 7876   predicate(UseIncDec);
 7877   match(Set dst (AddL dst src));
 7878   effect(KILL cr);
 7879 
 7880   format %{ "incq    $dst\t# long" %}
 7881   ins_encode %{
 7882     __ incrementq($dst$$Register);
 7883   %}
 7884   ins_pipe(ialu_reg);
 7885 %}
 7886 
 7887 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7888 %{
 7889   predicate(UseIncDec);
 7890   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7891   effect(KILL cr);
 7892 
 7893   ins_cost(125); // XXX
 7894   format %{ "incq    $dst\t# long" %}
 7895   ins_encode %{
 7896     __ incrementq($dst$$Address);
 7897   %}
 7898   ins_pipe(ialu_mem_imm);
 7899 %}
 7900 
 7901 // XXX why does that use AddL
 7902 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7903 %{
 7904   predicate(UseIncDec);
 7905   match(Set dst (AddL dst src));
 7906   effect(KILL cr);
 7907 
 7908   format %{ "decq    $dst\t# long" %}
 7909   ins_encode %{
 7910     __ decrementq($dst$$Register);
 7911   %}
 7912   ins_pipe(ialu_reg);
 7913 %}
 7914 
 7915 // XXX why does that use AddL
 7916 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7917 %{
 7918   predicate(UseIncDec);
 7919   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7920   effect(KILL cr);
 7921 
 7922   ins_cost(125); // XXX
 7923   format %{ "decq    $dst\t# long" %}
 7924   ins_encode %{
 7925     __ decrementq($dst$$Address);
 7926   %}
 7927   ins_pipe(ialu_mem_imm);
 7928 %}
 7929 
 7930 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7931 %{
 7932   predicate(VM_Version::supports_fast_2op_lea());
 7933   match(Set dst (AddL (LShiftL index scale) disp));
 7934 
 7935   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7936   ins_encode %{
 7937     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7938     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7939   %}
 7940   ins_pipe(ialu_reg_reg);
 7941 %}
 7942 
 7943 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7944 %{
 7945   predicate(VM_Version::supports_fast_3op_lea());
 7946   match(Set dst (AddL (AddL base index) disp));
 7947 
 7948   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7949   ins_encode %{
 7950     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7951   %}
 7952   ins_pipe(ialu_reg_reg);
 7953 %}
 7954 
 7955 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7956 %{
 7957   predicate(VM_Version::supports_fast_2op_lea());
 7958   match(Set dst (AddL base (LShiftL index scale)));
 7959 
 7960   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7961   ins_encode %{
 7962     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7963     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7964   %}
 7965   ins_pipe(ialu_reg_reg);
 7966 %}
 7967 
 7968 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7969 %{
 7970   predicate(VM_Version::supports_fast_3op_lea());
 7971   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7972 
 7973   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7974   ins_encode %{
 7975     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7976     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7977   %}
 7978   ins_pipe(ialu_reg_reg);
 7979 %}
 7980 
 7981 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7982 %{
 7983   match(Set dst (AddP dst src));
 7984   effect(KILL cr);
 7985 
 7986   format %{ "addq    $dst, $src\t# ptr" %}
 7987   ins_encode %{
 7988     __ addq($dst$$Register, $src$$Register);
 7989   %}
 7990   ins_pipe(ialu_reg_reg);
 7991 %}
 7992 
 7993 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7994 %{
 7995   match(Set dst (AddP dst src));
 7996   effect(KILL cr);
 7997 
 7998   format %{ "addq    $dst, $src\t# ptr" %}
 7999   ins_encode %{
 8000     __ addq($dst$$Register, $src$$constant);
 8001   %}
 8002   ins_pipe( ialu_reg );
 8003 %}
 8004 
 8005 // XXX addP mem ops ????
 8006 
 8007 instruct checkCastPP(rRegP dst)
 8008 %{
 8009   match(Set dst (CheckCastPP dst));
 8010 
 8011   size(0);
 8012   format %{ "# checkcastPP of $dst" %}
 8013   ins_encode(/* empty encoding */);
 8014   ins_pipe(empty);
 8015 %}
 8016 
 8017 instruct castPP(rRegP dst)
 8018 %{
 8019   match(Set dst (CastPP dst));
 8020 
 8021   size(0);
 8022   format %{ "# castPP of $dst" %}
 8023   ins_encode(/* empty encoding */);
 8024   ins_pipe(empty);
 8025 %}
 8026 
 8027 instruct castII(rRegI dst)
 8028 %{
 8029   match(Set dst (CastII dst));
 8030 
 8031   size(0);
 8032   format %{ "# castII of $dst" %}
 8033   ins_encode(/* empty encoding */);
 8034   ins_cost(0);
 8035   ins_pipe(empty);
 8036 %}
 8037 
 8038 instruct castLL(rRegL dst)
 8039 %{
 8040   match(Set dst (CastLL dst));
 8041 
 8042   size(0);
 8043   format %{ "# castLL of $dst" %}
 8044   ins_encode(/* empty encoding */);
 8045   ins_cost(0);
 8046   ins_pipe(empty);
 8047 %}
 8048 
 8049 instruct castFF(regF dst)
 8050 %{
 8051   match(Set dst (CastFF dst));
 8052 
 8053   size(0);
 8054   format %{ "# castFF of $dst" %}
 8055   ins_encode(/* empty encoding */);
 8056   ins_cost(0);
 8057   ins_pipe(empty);
 8058 %}
 8059 
 8060 instruct castDD(regD dst)
 8061 %{
 8062   match(Set dst (CastDD dst));
 8063 
 8064   size(0);
 8065   format %{ "# castDD of $dst" %}
 8066   ins_encode(/* empty encoding */);
 8067   ins_cost(0);
 8068   ins_pipe(empty);
 8069 %}
 8070 
 8071 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8072 instruct compareAndSwapP(rRegI res,
 8073                          memory mem_ptr,
 8074                          rax_RegP oldval, rRegP newval,
 8075                          rFlagsReg cr)
 8076 %{
 8077   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8078   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8079   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8080   effect(KILL cr, KILL oldval);
 8081 
 8082   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8083             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8084             "sete    $res\n\t"
 8085             "movzbl  $res, $res" %}
 8086   ins_encode %{
 8087     __ lock();
 8088     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8089     __ sete($res$$Register);
 8090     __ movzbl($res$$Register, $res$$Register);
 8091   %}
 8092   ins_pipe( pipe_cmpxchg );
 8093 %}
 8094 
 8095 instruct compareAndSwapL(rRegI res,
 8096                          memory mem_ptr,
 8097                          rax_RegL oldval, rRegL newval,
 8098                          rFlagsReg cr)
 8099 %{
 8100   predicate(VM_Version::supports_cx8());
 8101   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8102   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8103   effect(KILL cr, KILL oldval);
 8104 
 8105   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8106             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8107             "sete    $res\n\t"
 8108             "movzbl  $res, $res" %}
 8109   ins_encode %{
 8110     __ lock();
 8111     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8112     __ sete($res$$Register);
 8113     __ movzbl($res$$Register, $res$$Register);
 8114   %}
 8115   ins_pipe( pipe_cmpxchg );
 8116 %}
 8117 
 8118 instruct compareAndSwapI(rRegI res,
 8119                          memory mem_ptr,
 8120                          rax_RegI oldval, rRegI newval,
 8121                          rFlagsReg cr)
 8122 %{
 8123   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8124   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8125   effect(KILL cr, KILL oldval);
 8126 
 8127   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8128             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8129             "sete    $res\n\t"
 8130             "movzbl  $res, $res" %}
 8131   ins_encode %{
 8132     __ lock();
 8133     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8134     __ sete($res$$Register);
 8135     __ movzbl($res$$Register, $res$$Register);
 8136   %}
 8137   ins_pipe( pipe_cmpxchg );
 8138 %}
 8139 
 8140 instruct compareAndSwapB(rRegI res,
 8141                          memory mem_ptr,
 8142                          rax_RegI oldval, rRegI newval,
 8143                          rFlagsReg cr)
 8144 %{
 8145   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8146   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8147   effect(KILL cr, KILL oldval);
 8148 
 8149   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8150             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8151             "sete    $res\n\t"
 8152             "movzbl  $res, $res" %}
 8153   ins_encode %{
 8154     __ lock();
 8155     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8156     __ sete($res$$Register);
 8157     __ movzbl($res$$Register, $res$$Register);
 8158   %}
 8159   ins_pipe( pipe_cmpxchg );
 8160 %}
 8161 
 8162 instruct compareAndSwapS(rRegI res,
 8163                          memory mem_ptr,
 8164                          rax_RegI oldval, rRegI newval,
 8165                          rFlagsReg cr)
 8166 %{
 8167   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8168   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8169   effect(KILL cr, KILL oldval);
 8170 
 8171   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8172             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8173             "sete    $res\n\t"
 8174             "movzbl  $res, $res" %}
 8175   ins_encode %{
 8176     __ lock();
 8177     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8178     __ sete($res$$Register);
 8179     __ movzbl($res$$Register, $res$$Register);
 8180   %}
 8181   ins_pipe( pipe_cmpxchg );
 8182 %}
 8183 
 8184 instruct compareAndSwapN(rRegI res,
 8185                           memory mem_ptr,
 8186                           rax_RegN oldval, rRegN newval,
 8187                           rFlagsReg cr) %{
 8188   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8189   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8190   effect(KILL cr, KILL oldval);
 8191 
 8192   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8193             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8194             "sete    $res\n\t"
 8195             "movzbl  $res, $res" %}
 8196   ins_encode %{
 8197     __ lock();
 8198     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8199     __ sete($res$$Register);
 8200     __ movzbl($res$$Register, $res$$Register);
 8201   %}
 8202   ins_pipe( pipe_cmpxchg );
 8203 %}
 8204 
 8205 instruct compareAndExchangeB(
 8206                          memory mem_ptr,
 8207                          rax_RegI oldval, rRegI newval,
 8208                          rFlagsReg cr)
 8209 %{
 8210   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8211   effect(KILL cr);
 8212 
 8213   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8214             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8215   ins_encode %{
 8216     __ lock();
 8217     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8218   %}
 8219   ins_pipe( pipe_cmpxchg );
 8220 %}
 8221 
 8222 instruct compareAndExchangeS(
 8223                          memory mem_ptr,
 8224                          rax_RegI oldval, rRegI newval,
 8225                          rFlagsReg cr)
 8226 %{
 8227   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8228   effect(KILL cr);
 8229 
 8230   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8231             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8232   ins_encode %{
 8233     __ lock();
 8234     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8235   %}
 8236   ins_pipe( pipe_cmpxchg );
 8237 %}
 8238 
 8239 instruct compareAndExchangeI(
 8240                          memory mem_ptr,
 8241                          rax_RegI oldval, rRegI newval,
 8242                          rFlagsReg cr)
 8243 %{
 8244   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8245   effect(KILL cr);
 8246 
 8247   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8248             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8249   ins_encode %{
 8250     __ lock();
 8251     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8252   %}
 8253   ins_pipe( pipe_cmpxchg );
 8254 %}
 8255 
 8256 instruct compareAndExchangeL(
 8257                          memory mem_ptr,
 8258                          rax_RegL oldval, rRegL newval,
 8259                          rFlagsReg cr)
 8260 %{
 8261   predicate(VM_Version::supports_cx8());
 8262   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8263   effect(KILL cr);
 8264 
 8265   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8266             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8267   ins_encode %{
 8268     __ lock();
 8269     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8270   %}
 8271   ins_pipe( pipe_cmpxchg );
 8272 %}
 8273 
 8274 instruct compareAndExchangeN(
 8275                           memory mem_ptr,
 8276                           rax_RegN oldval, rRegN newval,
 8277                           rFlagsReg cr) %{
 8278   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8279   effect(KILL cr);
 8280 
 8281   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8282             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8283   ins_encode %{
 8284     __ lock();
 8285     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8286   %}
 8287   ins_pipe( pipe_cmpxchg );
 8288 %}
 8289 
 8290 instruct compareAndExchangeP(
 8291                          memory mem_ptr,
 8292                          rax_RegP oldval, rRegP newval,
 8293                          rFlagsReg cr)
 8294 %{
 8295   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8296   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8297   effect(KILL cr);
 8298 
 8299   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8300             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8301   ins_encode %{
 8302     __ lock();
 8303     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8304   %}
 8305   ins_pipe( pipe_cmpxchg );
 8306 %}
 8307 
 8308 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8309   predicate(n->as_LoadStore()->result_not_used());
 8310   match(Set dummy (GetAndAddB mem add));
 8311   effect(KILL cr);
 8312   format %{ "ADDB  [$mem],$add" %}
 8313   ins_encode %{
 8314     __ lock();
 8315     __ addb($mem$$Address, $add$$constant);
 8316   %}
 8317   ins_pipe( pipe_cmpxchg );
 8318 %}
 8319 
 8320 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8321   match(Set newval (GetAndAddB mem newval));
 8322   effect(KILL cr);
 8323   format %{ "XADDB  [$mem],$newval" %}
 8324   ins_encode %{
 8325     __ lock();
 8326     __ xaddb($mem$$Address, $newval$$Register);
 8327   %}
 8328   ins_pipe( pipe_cmpxchg );
 8329 %}
 8330 
 8331 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8332   predicate(n->as_LoadStore()->result_not_used());
 8333   match(Set dummy (GetAndAddS mem add));
 8334   effect(KILL cr);
 8335   format %{ "ADDW  [$mem],$add" %}
 8336   ins_encode %{
 8337     __ lock();
 8338     __ addw($mem$$Address, $add$$constant);
 8339   %}
 8340   ins_pipe( pipe_cmpxchg );
 8341 %}
 8342 
 8343 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8344   match(Set newval (GetAndAddS mem newval));
 8345   effect(KILL cr);
 8346   format %{ "XADDW  [$mem],$newval" %}
 8347   ins_encode %{
 8348     __ lock();
 8349     __ xaddw($mem$$Address, $newval$$Register);
 8350   %}
 8351   ins_pipe( pipe_cmpxchg );
 8352 %}
 8353 
 8354 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8355   predicate(n->as_LoadStore()->result_not_used());
 8356   match(Set dummy (GetAndAddI mem add));
 8357   effect(KILL cr);
 8358   format %{ "ADDL  [$mem],$add" %}
 8359   ins_encode %{
 8360     __ lock();
 8361     __ addl($mem$$Address, $add$$constant);
 8362   %}
 8363   ins_pipe( pipe_cmpxchg );
 8364 %}
 8365 
 8366 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8367   match(Set newval (GetAndAddI mem newval));
 8368   effect(KILL cr);
 8369   format %{ "XADDL  [$mem],$newval" %}
 8370   ins_encode %{
 8371     __ lock();
 8372     __ xaddl($mem$$Address, $newval$$Register);
 8373   %}
 8374   ins_pipe( pipe_cmpxchg );
 8375 %}
 8376 
 8377 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8378   predicate(n->as_LoadStore()->result_not_used());
 8379   match(Set dummy (GetAndAddL mem add));
 8380   effect(KILL cr);
 8381   format %{ "ADDQ  [$mem],$add" %}
 8382   ins_encode %{
 8383     __ lock();
 8384     __ addq($mem$$Address, $add$$constant);
 8385   %}
 8386   ins_pipe( pipe_cmpxchg );
 8387 %}
 8388 
 8389 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8390   match(Set newval (GetAndAddL mem newval));
 8391   effect(KILL cr);
 8392   format %{ "XADDQ  [$mem],$newval" %}
 8393   ins_encode %{
 8394     __ lock();
 8395     __ xaddq($mem$$Address, $newval$$Register);
 8396   %}
 8397   ins_pipe( pipe_cmpxchg );
 8398 %}
 8399 
 8400 instruct xchgB( memory mem, rRegI newval) %{
 8401   match(Set newval (GetAndSetB mem newval));
 8402   format %{ "XCHGB  $newval,[$mem]" %}
 8403   ins_encode %{
 8404     __ xchgb($newval$$Register, $mem$$Address);
 8405   %}
 8406   ins_pipe( pipe_cmpxchg );
 8407 %}
 8408 
 8409 instruct xchgS( memory mem, rRegI newval) %{
 8410   match(Set newval (GetAndSetS mem newval));
 8411   format %{ "XCHGW  $newval,[$mem]" %}
 8412   ins_encode %{
 8413     __ xchgw($newval$$Register, $mem$$Address);
 8414   %}
 8415   ins_pipe( pipe_cmpxchg );
 8416 %}
 8417 
 8418 instruct xchgI( memory mem, rRegI newval) %{
 8419   match(Set newval (GetAndSetI mem newval));
 8420   format %{ "XCHGL  $newval,[$mem]" %}
 8421   ins_encode %{
 8422     __ xchgl($newval$$Register, $mem$$Address);
 8423   %}
 8424   ins_pipe( pipe_cmpxchg );
 8425 %}
 8426 
 8427 instruct xchgL( memory mem, rRegL newval) %{
 8428   match(Set newval (GetAndSetL mem newval));
 8429   format %{ "XCHGL  $newval,[$mem]" %}
 8430   ins_encode %{
 8431     __ xchgq($newval$$Register, $mem$$Address);
 8432   %}
 8433   ins_pipe( pipe_cmpxchg );
 8434 %}
 8435 
 8436 instruct xchgP( memory mem, rRegP newval) %{
 8437   match(Set newval (GetAndSetP mem newval));
 8438   predicate(n->as_LoadStore()->barrier_data() == 0);
 8439   format %{ "XCHGQ  $newval,[$mem]" %}
 8440   ins_encode %{
 8441     __ xchgq($newval$$Register, $mem$$Address);
 8442   %}
 8443   ins_pipe( pipe_cmpxchg );
 8444 %}
 8445 
 8446 instruct xchgN( memory mem, rRegN newval) %{
 8447   match(Set newval (GetAndSetN mem newval));
 8448   format %{ "XCHGL  $newval,$mem]" %}
 8449   ins_encode %{
 8450     __ xchgl($newval$$Register, $mem$$Address);
 8451   %}
 8452   ins_pipe( pipe_cmpxchg );
 8453 %}
 8454 
 8455 //----------Abs Instructions-------------------------------------------
 8456 
 8457 // Integer Absolute Instructions
 8458 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8459 %{
 8460   match(Set dst (AbsI src));
 8461   effect(TEMP dst, TEMP tmp, KILL cr);
 8462   format %{ "movl $tmp, $src\n\t"
 8463             "sarl $tmp, 31\n\t"
 8464             "movl $dst, $src\n\t"
 8465             "xorl $dst, $tmp\n\t"
 8466             "subl $dst, $tmp\n"
 8467           %}
 8468   ins_encode %{
 8469     __ movl($tmp$$Register, $src$$Register);
 8470     __ sarl($tmp$$Register, 31);
 8471     __ movl($dst$$Register, $src$$Register);
 8472     __ xorl($dst$$Register, $tmp$$Register);
 8473     __ subl($dst$$Register, $tmp$$Register);
 8474   %}
 8475 
 8476   ins_pipe(ialu_reg_reg);
 8477 %}
 8478 
 8479 // Long Absolute Instructions
 8480 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8481 %{
 8482   match(Set dst (AbsL src));
 8483   effect(TEMP dst, TEMP tmp, KILL cr);
 8484   format %{ "movq $tmp, $src\n\t"
 8485             "sarq $tmp, 63\n\t"
 8486             "movq $dst, $src\n\t"
 8487             "xorq $dst, $tmp\n\t"
 8488             "subq $dst, $tmp\n"
 8489           %}
 8490   ins_encode %{
 8491     __ movq($tmp$$Register, $src$$Register);
 8492     __ sarq($tmp$$Register, 63);
 8493     __ movq($dst$$Register, $src$$Register);
 8494     __ xorq($dst$$Register, $tmp$$Register);
 8495     __ subq($dst$$Register, $tmp$$Register);
 8496   %}
 8497 
 8498   ins_pipe(ialu_reg_reg);
 8499 %}
 8500 
 8501 //----------Subtraction Instructions-------------------------------------------
 8502 
 8503 // Integer Subtraction Instructions
 8504 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8505 %{
 8506   match(Set dst (SubI dst src));
 8507   effect(KILL cr);
 8508 
 8509   format %{ "subl    $dst, $src\t# int" %}
 8510   ins_encode %{
 8511     __ subl($dst$$Register, $src$$Register);
 8512   %}
 8513   ins_pipe(ialu_reg_reg);
 8514 %}
 8515 
 8516 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8517 %{
 8518   match(Set dst (SubI dst src));
 8519   effect(KILL cr);
 8520 
 8521   format %{ "subl    $dst, $src\t# int" %}
 8522   ins_encode %{
 8523     __ subl($dst$$Register, $src$$constant);
 8524   %}
 8525   ins_pipe(ialu_reg);
 8526 %}
 8527 
 8528 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8529 %{
 8530   match(Set dst (SubI dst (LoadI src)));
 8531   effect(KILL cr);
 8532 
 8533   ins_cost(125);
 8534   format %{ "subl    $dst, $src\t# int" %}
 8535   ins_encode %{
 8536     __ subl($dst$$Register, $src$$Address);
 8537   %}
 8538   ins_pipe(ialu_reg_mem);
 8539 %}
 8540 
 8541 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8542 %{
 8543   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8544   effect(KILL cr);
 8545 
 8546   ins_cost(150);
 8547   format %{ "subl    $dst, $src\t# int" %}
 8548   ins_encode %{
 8549     __ subl($dst$$Address, $src$$Register);
 8550   %}
 8551   ins_pipe(ialu_mem_reg);
 8552 %}
 8553 
 8554 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8555 %{
 8556   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8557   effect(KILL cr);
 8558 
 8559   ins_cost(125); // XXX
 8560   format %{ "subl    $dst, $src\t# int" %}
 8561   ins_encode %{
 8562     __ subl($dst$$Address, $src$$constant);
 8563   %}
 8564   ins_pipe(ialu_mem_imm);
 8565 %}
 8566 
 8567 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8568 %{
 8569   match(Set dst (SubL dst src));
 8570   effect(KILL cr);
 8571 
 8572   format %{ "subq    $dst, $src\t# long" %}
 8573   ins_encode %{
 8574     __ subq($dst$$Register, $src$$Register);
 8575   %}
 8576   ins_pipe(ialu_reg_reg);
 8577 %}
 8578 
 8579 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8580 %{
 8581   match(Set dst (SubL dst src));
 8582   effect(KILL cr);
 8583 
 8584   format %{ "subq    $dst, $src\t# long" %}
 8585   ins_encode %{
 8586     __ subq($dst$$Register, $src$$constant);
 8587   %}
 8588   ins_pipe(ialu_reg);
 8589 %}
 8590 
 8591 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8592 %{
 8593   match(Set dst (SubL dst (LoadL src)));
 8594   effect(KILL cr);
 8595 
 8596   ins_cost(125);
 8597   format %{ "subq    $dst, $src\t# long" %}
 8598   ins_encode %{
 8599     __ subq($dst$$Register, $src$$Address);
 8600   %}
 8601   ins_pipe(ialu_reg_mem);
 8602 %}
 8603 
 8604 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8605 %{
 8606   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8607   effect(KILL cr);
 8608 
 8609   ins_cost(150);
 8610   format %{ "subq    $dst, $src\t# long" %}
 8611   ins_encode %{
 8612     __ subq($dst$$Address, $src$$Register);
 8613   %}
 8614   ins_pipe(ialu_mem_reg);
 8615 %}
 8616 
 8617 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8618 %{
 8619   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8620   effect(KILL cr);
 8621 
 8622   ins_cost(125); // XXX
 8623   format %{ "subq    $dst, $src\t# long" %}
 8624   ins_encode %{
 8625     __ subq($dst$$Address, $src$$constant);
 8626   %}
 8627   ins_pipe(ialu_mem_imm);
 8628 %}
 8629 
 8630 // Subtract from a pointer
 8631 // XXX hmpf???
 8632 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8633 %{
 8634   match(Set dst (AddP dst (SubI zero src)));
 8635   effect(KILL cr);
 8636 
 8637   format %{ "subq    $dst, $src\t# ptr - int" %}
 8638   opcode(0x2B);
 8639   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8640   ins_pipe(ialu_reg_reg);
 8641 %}
 8642 
 8643 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8644 %{
 8645   match(Set dst (SubI zero dst));
 8646   effect(KILL cr);
 8647 
 8648   format %{ "negl    $dst\t# int" %}
 8649   ins_encode %{
 8650     __ negl($dst$$Register);
 8651   %}
 8652   ins_pipe(ialu_reg);
 8653 %}
 8654 
 8655 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8656 %{
 8657   match(Set dst (NegI dst));
 8658   effect(KILL cr);
 8659 
 8660   format %{ "negl    $dst\t# int" %}
 8661   ins_encode %{
 8662     __ negl($dst$$Register);
 8663   %}
 8664   ins_pipe(ialu_reg);
 8665 %}
 8666 
 8667 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8668 %{
 8669   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8670   effect(KILL cr);
 8671 
 8672   format %{ "negl    $dst\t# int" %}
 8673   ins_encode %{
 8674     __ negl($dst$$Address);
 8675   %}
 8676   ins_pipe(ialu_reg);
 8677 %}
 8678 
 8679 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8680 %{
 8681   match(Set dst (SubL zero dst));
 8682   effect(KILL cr);
 8683 
 8684   format %{ "negq    $dst\t# long" %}
 8685   ins_encode %{
 8686     __ negq($dst$$Register);
 8687   %}
 8688   ins_pipe(ialu_reg);
 8689 %}
 8690 
 8691 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8692 %{
 8693   match(Set dst (NegL dst));
 8694   effect(KILL cr);
 8695 
 8696   format %{ "negq    $dst\t# int" %}
 8697   ins_encode %{
 8698     __ negq($dst$$Register);
 8699   %}
 8700   ins_pipe(ialu_reg);
 8701 %}
 8702 
 8703 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8704 %{
 8705   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8706   effect(KILL cr);
 8707 
 8708   format %{ "negq    $dst\t# long" %}
 8709   ins_encode %{
 8710     __ negq($dst$$Address);
 8711   %}
 8712   ins_pipe(ialu_reg);
 8713 %}
 8714 
 8715 //----------Multiplication/Division Instructions-------------------------------
 8716 // Integer Multiplication Instructions
 8717 // Multiply Register
 8718 
 8719 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8720 %{
 8721   match(Set dst (MulI dst src));
 8722   effect(KILL cr);
 8723 
 8724   ins_cost(300);
 8725   format %{ "imull   $dst, $src\t# int" %}
 8726   ins_encode %{
 8727     __ imull($dst$$Register, $src$$Register);
 8728   %}
 8729   ins_pipe(ialu_reg_reg_alu0);
 8730 %}
 8731 
 8732 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8733 %{
 8734   match(Set dst (MulI src imm));
 8735   effect(KILL cr);
 8736 
 8737   ins_cost(300);
 8738   format %{ "imull   $dst, $src, $imm\t# int" %}
 8739   ins_encode %{
 8740     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8741   %}
 8742   ins_pipe(ialu_reg_reg_alu0);
 8743 %}
 8744 
 8745 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8746 %{
 8747   match(Set dst (MulI dst (LoadI src)));
 8748   effect(KILL cr);
 8749 
 8750   ins_cost(350);
 8751   format %{ "imull   $dst, $src\t# int" %}
 8752   ins_encode %{
 8753     __ imull($dst$$Register, $src$$Address);
 8754   %}
 8755   ins_pipe(ialu_reg_mem_alu0);
 8756 %}
 8757 
 8758 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8759 %{
 8760   match(Set dst (MulI (LoadI src) imm));
 8761   effect(KILL cr);
 8762 
 8763   ins_cost(300);
 8764   format %{ "imull   $dst, $src, $imm\t# int" %}
 8765   ins_encode %{
 8766     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8767   %}
 8768   ins_pipe(ialu_reg_mem_alu0);
 8769 %}
 8770 
 8771 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8772 %{
 8773   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8774   effect(KILL cr, KILL src2);
 8775 
 8776   expand %{ mulI_rReg(dst, src1, cr);
 8777            mulI_rReg(src2, src3, cr);
 8778            addI_rReg(dst, src2, cr); %}
 8779 %}
 8780 
 8781 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8782 %{
 8783   match(Set dst (MulL dst src));
 8784   effect(KILL cr);
 8785 
 8786   ins_cost(300);
 8787   format %{ "imulq   $dst, $src\t# long" %}
 8788   ins_encode %{
 8789     __ imulq($dst$$Register, $src$$Register);
 8790   %}
 8791   ins_pipe(ialu_reg_reg_alu0);
 8792 %}
 8793 
 8794 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8795 %{
 8796   match(Set dst (MulL src imm));
 8797   effect(KILL cr);
 8798 
 8799   ins_cost(300);
 8800   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8801   ins_encode %{
 8802     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8803   %}
 8804   ins_pipe(ialu_reg_reg_alu0);
 8805 %}
 8806 
 8807 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8808 %{
 8809   match(Set dst (MulL dst (LoadL src)));
 8810   effect(KILL cr);
 8811 
 8812   ins_cost(350);
 8813   format %{ "imulq   $dst, $src\t# long" %}
 8814   ins_encode %{
 8815     __ imulq($dst$$Register, $src$$Address);
 8816   %}
 8817   ins_pipe(ialu_reg_mem_alu0);
 8818 %}
 8819 
 8820 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8821 %{
 8822   match(Set dst (MulL (LoadL src) imm));
 8823   effect(KILL cr);
 8824 
 8825   ins_cost(300);
 8826   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8827   ins_encode %{
 8828     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8829   %}
 8830   ins_pipe(ialu_reg_mem_alu0);
 8831 %}
 8832 
 8833 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8834 %{
 8835   match(Set dst (MulHiL src rax));
 8836   effect(USE_KILL rax, KILL cr);
 8837 
 8838   ins_cost(300);
 8839   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8840   ins_encode %{
 8841     __ imulq($src$$Register);
 8842   %}
 8843   ins_pipe(ialu_reg_reg_alu0);
 8844 %}
 8845 
 8846 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8847 %{
 8848   match(Set dst (UMulHiL src rax));
 8849   effect(USE_KILL rax, KILL cr);
 8850 
 8851   ins_cost(300);
 8852   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8853   ins_encode %{
 8854     __ mulq($src$$Register);
 8855   %}
 8856   ins_pipe(ialu_reg_reg_alu0);
 8857 %}
 8858 
 8859 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8860                    rFlagsReg cr)
 8861 %{
 8862   match(Set rax (DivI rax div));
 8863   effect(KILL rdx, KILL cr);
 8864 
 8865   ins_cost(30*100+10*100); // XXX
 8866   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8867             "jne,s   normal\n\t"
 8868             "xorl    rdx, rdx\n\t"
 8869             "cmpl    $div, -1\n\t"
 8870             "je,s    done\n"
 8871     "normal: cdql\n\t"
 8872             "idivl   $div\n"
 8873     "done:"        %}
 8874   ins_encode(cdql_enc(div));
 8875   ins_pipe(ialu_reg_reg_alu0);
 8876 %}
 8877 
 8878 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8879                    rFlagsReg cr)
 8880 %{
 8881   match(Set rax (DivL rax div));
 8882   effect(KILL rdx, KILL cr);
 8883 
 8884   ins_cost(30*100+10*100); // XXX
 8885   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8886             "cmpq    rax, rdx\n\t"
 8887             "jne,s   normal\n\t"
 8888             "xorl    rdx, rdx\n\t"
 8889             "cmpq    $div, -1\n\t"
 8890             "je,s    done\n"
 8891     "normal: cdqq\n\t"
 8892             "idivq   $div\n"
 8893     "done:"        %}
 8894   ins_encode(cdqq_enc(div));
 8895   ins_pipe(ialu_reg_reg_alu0);
 8896 %}
 8897 
 8898 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8899 %{
 8900   match(Set rax (UDivI rax div));
 8901   effect(KILL rdx, KILL cr);
 8902 
 8903   ins_cost(300);
 8904   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8905   ins_encode %{
 8906     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8907   %}
 8908   ins_pipe(ialu_reg_reg_alu0);
 8909 %}
 8910 
 8911 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8912 %{
 8913   match(Set rax (UDivL rax div));
 8914   effect(KILL rdx, KILL cr);
 8915 
 8916   ins_cost(300);
 8917   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8918   ins_encode %{
 8919      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8920   %}
 8921   ins_pipe(ialu_reg_reg_alu0);
 8922 %}
 8923 
 8924 // Integer DIVMOD with Register, both quotient and mod results
 8925 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8926                              rFlagsReg cr)
 8927 %{
 8928   match(DivModI rax div);
 8929   effect(KILL cr);
 8930 
 8931   ins_cost(30*100+10*100); // XXX
 8932   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8933             "jne,s   normal\n\t"
 8934             "xorl    rdx, rdx\n\t"
 8935             "cmpl    $div, -1\n\t"
 8936             "je,s    done\n"
 8937     "normal: cdql\n\t"
 8938             "idivl   $div\n"
 8939     "done:"        %}
 8940   ins_encode(cdql_enc(div));
 8941   ins_pipe(pipe_slow);
 8942 %}
 8943 
 8944 // Long DIVMOD with Register, both quotient and mod results
 8945 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8946                              rFlagsReg cr)
 8947 %{
 8948   match(DivModL rax div);
 8949   effect(KILL cr);
 8950 
 8951   ins_cost(30*100+10*100); // XXX
 8952   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8953             "cmpq    rax, rdx\n\t"
 8954             "jne,s   normal\n\t"
 8955             "xorl    rdx, rdx\n\t"
 8956             "cmpq    $div, -1\n\t"
 8957             "je,s    done\n"
 8958     "normal: cdqq\n\t"
 8959             "idivq   $div\n"
 8960     "done:"        %}
 8961   ins_encode(cdqq_enc(div));
 8962   ins_pipe(pipe_slow);
 8963 %}
 8964 
 8965 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8966 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8967                               no_rax_rdx_RegI div, rFlagsReg cr)
 8968 %{
 8969   match(UDivModI rax div);
 8970   effect(TEMP tmp, KILL cr);
 8971 
 8972   ins_cost(300);
 8973   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8974             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8975           %}
 8976   ins_encode %{
 8977     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8978   %}
 8979   ins_pipe(pipe_slow);
 8980 %}
 8981 
 8982 // Unsigned long DIVMOD with Register, both quotient and mod results
 8983 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8984                               no_rax_rdx_RegL div, rFlagsReg cr)
 8985 %{
 8986   match(UDivModL rax div);
 8987   effect(TEMP tmp, KILL cr);
 8988 
 8989   ins_cost(300);
 8990   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8991             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8992           %}
 8993   ins_encode %{
 8994     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8995   %}
 8996   ins_pipe(pipe_slow);
 8997 %}
 8998 
 8999 
 9000 //----------- DivL-By-Constant-Expansions--------------------------------------
 9001 // DivI cases are handled by the compiler
 9002 
 9003 // Magic constant, reciprocal of 10
 9004 instruct loadConL_0x6666666666666667(rRegL dst)
 9005 %{
 9006   effect(DEF dst);
 9007 
 9008   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 9009   ins_encode(load_immL(dst, 0x6666666666666667));
 9010   ins_pipe(ialu_reg);
 9011 %}
 9012 
 9013 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 9014 %{
 9015   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 9016 
 9017   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 9018   ins_encode %{
 9019     __ imulq($src$$Register);
 9020   %}
 9021   ins_pipe(ialu_reg_reg_alu0);
 9022 %}
 9023 
 9024 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 9025 %{
 9026   effect(USE_DEF dst, KILL cr);
 9027 
 9028   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 9029   ins_encode %{
 9030     __ sarq($dst$$Register, 63);
 9031   %}
 9032   ins_pipe(ialu_reg);
 9033 %}
 9034 
 9035 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 9036 %{
 9037   effect(USE_DEF dst, KILL cr);
 9038 
 9039   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 9040   ins_encode %{
 9041     __ sarq($dst$$Register, 2);
 9042   %}
 9043   ins_pipe(ialu_reg);
 9044 %}
 9045 
 9046 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 9047 %{
 9048   match(Set dst (DivL src div));
 9049 
 9050   ins_cost((5+8)*100);
 9051   expand %{
 9052     rax_RegL rax;                     // Killed temp
 9053     rFlagsReg cr;                     // Killed
 9054     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 9055     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 9056     sarL_rReg_63(src, cr);            // sarq  src, 63
 9057     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 9058     subL_rReg(dst, src, cr);          // subl  rdx, src
 9059   %}
 9060 %}
 9061 
 9062 //-----------------------------------------------------------------------------
 9063 
 9064 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 9065                    rFlagsReg cr)
 9066 %{
 9067   match(Set rdx (ModI rax div));
 9068   effect(KILL rax, KILL cr);
 9069 
 9070   ins_cost(300); // XXX
 9071   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 9072             "jne,s   normal\n\t"
 9073             "xorl    rdx, rdx\n\t"
 9074             "cmpl    $div, -1\n\t"
 9075             "je,s    done\n"
 9076     "normal: cdql\n\t"
 9077             "idivl   $div\n"
 9078     "done:"        %}
 9079   ins_encode(cdql_enc(div));
 9080   ins_pipe(ialu_reg_reg_alu0);
 9081 %}
 9082 
 9083 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 9084                    rFlagsReg cr)
 9085 %{
 9086   match(Set rdx (ModL rax div));
 9087   effect(KILL rax, KILL cr);
 9088 
 9089   ins_cost(300); // XXX
 9090   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 9091             "cmpq    rax, rdx\n\t"
 9092             "jne,s   normal\n\t"
 9093             "xorl    rdx, rdx\n\t"
 9094             "cmpq    $div, -1\n\t"
 9095             "je,s    done\n"
 9096     "normal: cdqq\n\t"
 9097             "idivq   $div\n"
 9098     "done:"        %}
 9099   ins_encode(cdqq_enc(div));
 9100   ins_pipe(ialu_reg_reg_alu0);
 9101 %}
 9102 
 9103 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9104 %{
 9105   match(Set rdx (UModI rax div));
 9106   effect(KILL rax, KILL cr);
 9107 
 9108   ins_cost(300);
 9109   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9110   ins_encode %{
 9111     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9112   %}
 9113   ins_pipe(ialu_reg_reg_alu0);
 9114 %}
 9115 
 9116 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9117 %{
 9118   match(Set rdx (UModL rax div));
 9119   effect(KILL rax, KILL cr);
 9120 
 9121   ins_cost(300);
 9122   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9123   ins_encode %{
 9124     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9125   %}
 9126   ins_pipe(ialu_reg_reg_alu0);
 9127 %}
 9128 
 9129 // Integer Shift Instructions
 9130 // Shift Left by one
 9131 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9132 %{
 9133   match(Set dst (LShiftI dst shift));
 9134   effect(KILL cr);
 9135 
 9136   format %{ "sall    $dst, $shift" %}
 9137   ins_encode %{
 9138     __ sall($dst$$Register, $shift$$constant);
 9139   %}
 9140   ins_pipe(ialu_reg);
 9141 %}
 9142 
 9143 // Shift Left by one
 9144 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9145 %{
 9146   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9147   effect(KILL cr);
 9148 
 9149   format %{ "sall    $dst, $shift\t" %}
 9150   ins_encode %{
 9151     __ sall($dst$$Address, $shift$$constant);
 9152   %}
 9153   ins_pipe(ialu_mem_imm);
 9154 %}
 9155 
 9156 // Shift Left by 8-bit immediate
 9157 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9158 %{
 9159   match(Set dst (LShiftI dst shift));
 9160   effect(KILL cr);
 9161 
 9162   format %{ "sall    $dst, $shift" %}
 9163   ins_encode %{
 9164     __ sall($dst$$Register, $shift$$constant);
 9165   %}
 9166   ins_pipe(ialu_reg);
 9167 %}
 9168 
 9169 // Shift Left by 8-bit immediate
 9170 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9171 %{
 9172   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9173   effect(KILL cr);
 9174 
 9175   format %{ "sall    $dst, $shift" %}
 9176   ins_encode %{
 9177     __ sall($dst$$Address, $shift$$constant);
 9178   %}
 9179   ins_pipe(ialu_mem_imm);
 9180 %}
 9181 
 9182 // Shift Left by variable
 9183 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9184 %{
 9185   predicate(!VM_Version::supports_bmi2());
 9186   match(Set dst (LShiftI dst shift));
 9187   effect(KILL cr);
 9188 
 9189   format %{ "sall    $dst, $shift" %}
 9190   ins_encode %{
 9191     __ sall($dst$$Register);
 9192   %}
 9193   ins_pipe(ialu_reg_reg);
 9194 %}
 9195 
 9196 // Shift Left by variable
 9197 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9198 %{
 9199   predicate(!VM_Version::supports_bmi2());
 9200   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9201   effect(KILL cr);
 9202 
 9203   format %{ "sall    $dst, $shift" %}
 9204   ins_encode %{
 9205     __ sall($dst$$Address);
 9206   %}
 9207   ins_pipe(ialu_mem_reg);
 9208 %}
 9209 
 9210 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9211 %{
 9212   predicate(VM_Version::supports_bmi2());
 9213   match(Set dst (LShiftI src shift));
 9214 
 9215   format %{ "shlxl   $dst, $src, $shift" %}
 9216   ins_encode %{
 9217     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9218   %}
 9219   ins_pipe(ialu_reg_reg);
 9220 %}
 9221 
 9222 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9223 %{
 9224   predicate(VM_Version::supports_bmi2());
 9225   match(Set dst (LShiftI (LoadI src) shift));
 9226   ins_cost(175);
 9227   format %{ "shlxl   $dst, $src, $shift" %}
 9228   ins_encode %{
 9229     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9230   %}
 9231   ins_pipe(ialu_reg_mem);
 9232 %}
 9233 
 9234 // Arithmetic shift right by one
 9235 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9236 %{
 9237   match(Set dst (RShiftI dst shift));
 9238   effect(KILL cr);
 9239 
 9240   format %{ "sarl    $dst, $shift" %}
 9241   ins_encode %{
 9242     __ sarl($dst$$Register, $shift$$constant);
 9243   %}
 9244   ins_pipe(ialu_reg);
 9245 %}
 9246 
 9247 // Arithmetic shift right by one
 9248 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9249 %{
 9250   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9251   effect(KILL cr);
 9252 
 9253   format %{ "sarl    $dst, $shift" %}
 9254   ins_encode %{
 9255     __ sarl($dst$$Address, $shift$$constant);
 9256   %}
 9257   ins_pipe(ialu_mem_imm);
 9258 %}
 9259 
 9260 // Arithmetic Shift Right by 8-bit immediate
 9261 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9262 %{
 9263   match(Set dst (RShiftI dst shift));
 9264   effect(KILL cr);
 9265 
 9266   format %{ "sarl    $dst, $shift" %}
 9267   ins_encode %{
 9268     __ sarl($dst$$Register, $shift$$constant);
 9269   %}
 9270   ins_pipe(ialu_mem_imm);
 9271 %}
 9272 
 9273 // Arithmetic Shift Right by 8-bit immediate
 9274 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9275 %{
 9276   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9277   effect(KILL cr);
 9278 
 9279   format %{ "sarl    $dst, $shift" %}
 9280   ins_encode %{
 9281     __ sarl($dst$$Address, $shift$$constant);
 9282   %}
 9283   ins_pipe(ialu_mem_imm);
 9284 %}
 9285 
 9286 // Arithmetic Shift Right by variable
 9287 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9288 %{
 9289   predicate(!VM_Version::supports_bmi2());
 9290   match(Set dst (RShiftI dst shift));
 9291   effect(KILL cr);
 9292   format %{ "sarl    $dst, $shift" %}
 9293   ins_encode %{
 9294     __ sarl($dst$$Register);
 9295   %}
 9296   ins_pipe(ialu_reg_reg);
 9297 %}
 9298 
 9299 // Arithmetic Shift Right by variable
 9300 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9301 %{
 9302   predicate(!VM_Version::supports_bmi2());
 9303   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9304   effect(KILL cr);
 9305 
 9306   format %{ "sarl    $dst, $shift" %}
 9307   ins_encode %{
 9308     __ sarl($dst$$Address);
 9309   %}
 9310   ins_pipe(ialu_mem_reg);
 9311 %}
 9312 
 9313 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9314 %{
 9315   predicate(VM_Version::supports_bmi2());
 9316   match(Set dst (RShiftI src shift));
 9317 
 9318   format %{ "sarxl   $dst, $src, $shift" %}
 9319   ins_encode %{
 9320     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9321   %}
 9322   ins_pipe(ialu_reg_reg);
 9323 %}
 9324 
 9325 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9326 %{
 9327   predicate(VM_Version::supports_bmi2());
 9328   match(Set dst (RShiftI (LoadI src) shift));
 9329   ins_cost(175);
 9330   format %{ "sarxl   $dst, $src, $shift" %}
 9331   ins_encode %{
 9332     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9333   %}
 9334   ins_pipe(ialu_reg_mem);
 9335 %}
 9336 
 9337 // Logical shift right by one
 9338 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9339 %{
 9340   match(Set dst (URShiftI dst shift));
 9341   effect(KILL cr);
 9342 
 9343   format %{ "shrl    $dst, $shift" %}
 9344   ins_encode %{
 9345     __ shrl($dst$$Register, $shift$$constant);
 9346   %}
 9347   ins_pipe(ialu_reg);
 9348 %}
 9349 
 9350 // Logical shift right by one
 9351 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9352 %{
 9353   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9354   effect(KILL cr);
 9355 
 9356   format %{ "shrl    $dst, $shift" %}
 9357   ins_encode %{
 9358     __ shrl($dst$$Address, $shift$$constant);
 9359   %}
 9360   ins_pipe(ialu_mem_imm);
 9361 %}
 9362 
 9363 // Logical Shift Right by 8-bit immediate
 9364 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9365 %{
 9366   match(Set dst (URShiftI dst shift));
 9367   effect(KILL cr);
 9368 
 9369   format %{ "shrl    $dst, $shift" %}
 9370   ins_encode %{
 9371     __ shrl($dst$$Register, $shift$$constant);
 9372   %}
 9373   ins_pipe(ialu_reg);
 9374 %}
 9375 
 9376 // Logical Shift Right by 8-bit immediate
 9377 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9378 %{
 9379   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9380   effect(KILL cr);
 9381 
 9382   format %{ "shrl    $dst, $shift" %}
 9383   ins_encode %{
 9384     __ shrl($dst$$Address, $shift$$constant);
 9385   %}
 9386   ins_pipe(ialu_mem_imm);
 9387 %}
 9388 
 9389 // Logical Shift Right by variable
 9390 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9391 %{
 9392   predicate(!VM_Version::supports_bmi2());
 9393   match(Set dst (URShiftI dst shift));
 9394   effect(KILL cr);
 9395 
 9396   format %{ "shrl    $dst, $shift" %}
 9397   ins_encode %{
 9398     __ shrl($dst$$Register);
 9399   %}
 9400   ins_pipe(ialu_reg_reg);
 9401 %}
 9402 
 9403 // Logical Shift Right by variable
 9404 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9405 %{
 9406   predicate(!VM_Version::supports_bmi2());
 9407   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9408   effect(KILL cr);
 9409 
 9410   format %{ "shrl    $dst, $shift" %}
 9411   ins_encode %{
 9412     __ shrl($dst$$Address);
 9413   %}
 9414   ins_pipe(ialu_mem_reg);
 9415 %}
 9416 
 9417 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9418 %{
 9419   predicate(VM_Version::supports_bmi2());
 9420   match(Set dst (URShiftI src shift));
 9421 
 9422   format %{ "shrxl   $dst, $src, $shift" %}
 9423   ins_encode %{
 9424     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9425   %}
 9426   ins_pipe(ialu_reg_reg);
 9427 %}
 9428 
 9429 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9430 %{
 9431   predicate(VM_Version::supports_bmi2());
 9432   match(Set dst (URShiftI (LoadI src) shift));
 9433   ins_cost(175);
 9434   format %{ "shrxl   $dst, $src, $shift" %}
 9435   ins_encode %{
 9436     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9437   %}
 9438   ins_pipe(ialu_reg_mem);
 9439 %}
 9440 
 9441 // Long Shift Instructions
 9442 // Shift Left by one
 9443 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9444 %{
 9445   match(Set dst (LShiftL dst shift));
 9446   effect(KILL cr);
 9447 
 9448   format %{ "salq    $dst, $shift" %}
 9449   ins_encode %{
 9450     __ salq($dst$$Register, $shift$$constant);
 9451   %}
 9452   ins_pipe(ialu_reg);
 9453 %}
 9454 
 9455 // Shift Left by one
 9456 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9457 %{
 9458   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9459   effect(KILL cr);
 9460 
 9461   format %{ "salq    $dst, $shift" %}
 9462   ins_encode %{
 9463     __ salq($dst$$Address, $shift$$constant);
 9464   %}
 9465   ins_pipe(ialu_mem_imm);
 9466 %}
 9467 
 9468 // Shift Left by 8-bit immediate
 9469 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9470 %{
 9471   match(Set dst (LShiftL dst shift));
 9472   effect(KILL cr);
 9473 
 9474   format %{ "salq    $dst, $shift" %}
 9475   ins_encode %{
 9476     __ salq($dst$$Register, $shift$$constant);
 9477   %}
 9478   ins_pipe(ialu_reg);
 9479 %}
 9480 
 9481 // Shift Left by 8-bit immediate
 9482 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9483 %{
 9484   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9485   effect(KILL cr);
 9486 
 9487   format %{ "salq    $dst, $shift" %}
 9488   ins_encode %{
 9489     __ salq($dst$$Address, $shift$$constant);
 9490   %}
 9491   ins_pipe(ialu_mem_imm);
 9492 %}
 9493 
 9494 // Shift Left by variable
 9495 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9496 %{
 9497   predicate(!VM_Version::supports_bmi2());
 9498   match(Set dst (LShiftL dst shift));
 9499   effect(KILL cr);
 9500 
 9501   format %{ "salq    $dst, $shift" %}
 9502   ins_encode %{
 9503     __ salq($dst$$Register);
 9504   %}
 9505   ins_pipe(ialu_reg_reg);
 9506 %}
 9507 
 9508 // Shift Left by variable
 9509 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9510 %{
 9511   predicate(!VM_Version::supports_bmi2());
 9512   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9513   effect(KILL cr);
 9514 
 9515   format %{ "salq    $dst, $shift" %}
 9516   ins_encode %{
 9517     __ salq($dst$$Address);
 9518   %}
 9519   ins_pipe(ialu_mem_reg);
 9520 %}
 9521 
 9522 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9523 %{
 9524   predicate(VM_Version::supports_bmi2());
 9525   match(Set dst (LShiftL src shift));
 9526 
 9527   format %{ "shlxq   $dst, $src, $shift" %}
 9528   ins_encode %{
 9529     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9530   %}
 9531   ins_pipe(ialu_reg_reg);
 9532 %}
 9533 
 9534 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9535 %{
 9536   predicate(VM_Version::supports_bmi2());
 9537   match(Set dst (LShiftL (LoadL src) shift));
 9538   ins_cost(175);
 9539   format %{ "shlxq   $dst, $src, $shift" %}
 9540   ins_encode %{
 9541     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9542   %}
 9543   ins_pipe(ialu_reg_mem);
 9544 %}
 9545 
 9546 // Arithmetic shift right by one
 9547 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9548 %{
 9549   match(Set dst (RShiftL dst shift));
 9550   effect(KILL cr);
 9551 
 9552   format %{ "sarq    $dst, $shift" %}
 9553   ins_encode %{
 9554     __ sarq($dst$$Register, $shift$$constant);
 9555   %}
 9556   ins_pipe(ialu_reg);
 9557 %}
 9558 
 9559 // Arithmetic shift right by one
 9560 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9561 %{
 9562   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9563   effect(KILL cr);
 9564 
 9565   format %{ "sarq    $dst, $shift" %}
 9566   ins_encode %{
 9567     __ sarq($dst$$Address, $shift$$constant);
 9568   %}
 9569   ins_pipe(ialu_mem_imm);
 9570 %}
 9571 
 9572 // Arithmetic Shift Right by 8-bit immediate
 9573 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9574 %{
 9575   match(Set dst (RShiftL dst shift));
 9576   effect(KILL cr);
 9577 
 9578   format %{ "sarq    $dst, $shift" %}
 9579   ins_encode %{
 9580     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9581   %}
 9582   ins_pipe(ialu_mem_imm);
 9583 %}
 9584 
 9585 // Arithmetic Shift Right by 8-bit immediate
 9586 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9587 %{
 9588   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9589   effect(KILL cr);
 9590 
 9591   format %{ "sarq    $dst, $shift" %}
 9592   ins_encode %{
 9593     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9594   %}
 9595   ins_pipe(ialu_mem_imm);
 9596 %}
 9597 
 9598 // Arithmetic Shift Right by variable
 9599 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9600 %{
 9601   predicate(!VM_Version::supports_bmi2());
 9602   match(Set dst (RShiftL dst shift));
 9603   effect(KILL cr);
 9604 
 9605   format %{ "sarq    $dst, $shift" %}
 9606   ins_encode %{
 9607     __ sarq($dst$$Register);
 9608   %}
 9609   ins_pipe(ialu_reg_reg);
 9610 %}
 9611 
 9612 // Arithmetic Shift Right by variable
 9613 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9614 %{
 9615   predicate(!VM_Version::supports_bmi2());
 9616   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9617   effect(KILL cr);
 9618 
 9619   format %{ "sarq    $dst, $shift" %}
 9620   ins_encode %{
 9621     __ sarq($dst$$Address);
 9622   %}
 9623   ins_pipe(ialu_mem_reg);
 9624 %}
 9625 
 9626 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9627 %{
 9628   predicate(VM_Version::supports_bmi2());
 9629   match(Set dst (RShiftL src shift));
 9630 
 9631   format %{ "sarxq   $dst, $src, $shift" %}
 9632   ins_encode %{
 9633     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9634   %}
 9635   ins_pipe(ialu_reg_reg);
 9636 %}
 9637 
 9638 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9639 %{
 9640   predicate(VM_Version::supports_bmi2());
 9641   match(Set dst (RShiftL (LoadL src) shift));
 9642   ins_cost(175);
 9643   format %{ "sarxq   $dst, $src, $shift" %}
 9644   ins_encode %{
 9645     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9646   %}
 9647   ins_pipe(ialu_reg_mem);
 9648 %}
 9649 
 9650 // Logical shift right by one
 9651 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9652 %{
 9653   match(Set dst (URShiftL dst shift));
 9654   effect(KILL cr);
 9655 
 9656   format %{ "shrq    $dst, $shift" %}
 9657   ins_encode %{
 9658     __ shrq($dst$$Register, $shift$$constant);
 9659   %}
 9660   ins_pipe(ialu_reg);
 9661 %}
 9662 
 9663 // Logical shift right by one
 9664 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9665 %{
 9666   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9667   effect(KILL cr);
 9668 
 9669   format %{ "shrq    $dst, $shift" %}
 9670   ins_encode %{
 9671     __ shrq($dst$$Address, $shift$$constant);
 9672   %}
 9673   ins_pipe(ialu_mem_imm);
 9674 %}
 9675 
 9676 // Logical Shift Right by 8-bit immediate
 9677 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9678 %{
 9679   match(Set dst (URShiftL dst shift));
 9680   effect(KILL cr);
 9681 
 9682   format %{ "shrq    $dst, $shift" %}
 9683   ins_encode %{
 9684     __ shrq($dst$$Register, $shift$$constant);
 9685   %}
 9686   ins_pipe(ialu_reg);
 9687 %}
 9688 
 9689 // Logical Shift Right by 8-bit immediate
 9690 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9691 %{
 9692   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9693   effect(KILL cr);
 9694 
 9695   format %{ "shrq    $dst, $shift" %}
 9696   ins_encode %{
 9697     __ shrq($dst$$Address, $shift$$constant);
 9698   %}
 9699   ins_pipe(ialu_mem_imm);
 9700 %}
 9701 
 9702 // Logical Shift Right by variable
 9703 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9704 %{
 9705   predicate(!VM_Version::supports_bmi2());
 9706   match(Set dst (URShiftL dst shift));
 9707   effect(KILL cr);
 9708 
 9709   format %{ "shrq    $dst, $shift" %}
 9710   ins_encode %{
 9711     __ shrq($dst$$Register);
 9712   %}
 9713   ins_pipe(ialu_reg_reg);
 9714 %}
 9715 
 9716 // Logical Shift Right by variable
 9717 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9718 %{
 9719   predicate(!VM_Version::supports_bmi2());
 9720   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9721   effect(KILL cr);
 9722 
 9723   format %{ "shrq    $dst, $shift" %}
 9724   ins_encode %{
 9725     __ shrq($dst$$Address);
 9726   %}
 9727   ins_pipe(ialu_mem_reg);
 9728 %}
 9729 
 9730 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9731 %{
 9732   predicate(VM_Version::supports_bmi2());
 9733   match(Set dst (URShiftL src shift));
 9734 
 9735   format %{ "shrxq   $dst, $src, $shift" %}
 9736   ins_encode %{
 9737     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9738   %}
 9739   ins_pipe(ialu_reg_reg);
 9740 %}
 9741 
 9742 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9743 %{
 9744   predicate(VM_Version::supports_bmi2());
 9745   match(Set dst (URShiftL (LoadL src) shift));
 9746   ins_cost(175);
 9747   format %{ "shrxq   $dst, $src, $shift" %}
 9748   ins_encode %{
 9749     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9750   %}
 9751   ins_pipe(ialu_reg_mem);
 9752 %}
 9753 
 9754 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9755 // This idiom is used by the compiler for the i2b bytecode.
 9756 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9757 %{
 9758   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9759 
 9760   format %{ "movsbl  $dst, $src\t# i2b" %}
 9761   ins_encode %{
 9762     __ movsbl($dst$$Register, $src$$Register);
 9763   %}
 9764   ins_pipe(ialu_reg_reg);
 9765 %}
 9766 
 9767 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9768 // This idiom is used by the compiler the i2s bytecode.
 9769 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9770 %{
 9771   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9772 
 9773   format %{ "movswl  $dst, $src\t# i2s" %}
 9774   ins_encode %{
 9775     __ movswl($dst$$Register, $src$$Register);
 9776   %}
 9777   ins_pipe(ialu_reg_reg);
 9778 %}
 9779 
 9780 // ROL/ROR instructions
 9781 
 9782 // Rotate left by constant.
 9783 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9784 %{
 9785   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9786   match(Set dst (RotateLeft dst shift));
 9787   effect(KILL cr);
 9788   format %{ "roll    $dst, $shift" %}
 9789   ins_encode %{
 9790     __ roll($dst$$Register, $shift$$constant);
 9791   %}
 9792   ins_pipe(ialu_reg);
 9793 %}
 9794 
 9795 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9796 %{
 9797   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9798   match(Set dst (RotateLeft src shift));
 9799   format %{ "rolxl   $dst, $src, $shift" %}
 9800   ins_encode %{
 9801     int shift = 32 - ($shift$$constant & 31);
 9802     __ rorxl($dst$$Register, $src$$Register, shift);
 9803   %}
 9804   ins_pipe(ialu_reg_reg);
 9805 %}
 9806 
 9807 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9808 %{
 9809   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9810   match(Set dst (RotateLeft (LoadI src) shift));
 9811   ins_cost(175);
 9812   format %{ "rolxl   $dst, $src, $shift" %}
 9813   ins_encode %{
 9814     int shift = 32 - ($shift$$constant & 31);
 9815     __ rorxl($dst$$Register, $src$$Address, shift);
 9816   %}
 9817   ins_pipe(ialu_reg_mem);
 9818 %}
 9819 
 9820 // Rotate Left by variable
 9821 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9822 %{
 9823   predicate(n->bottom_type()->basic_type() == T_INT);
 9824   match(Set dst (RotateLeft dst shift));
 9825   effect(KILL cr);
 9826   format %{ "roll    $dst, $shift" %}
 9827   ins_encode %{
 9828     __ roll($dst$$Register);
 9829   %}
 9830   ins_pipe(ialu_reg_reg);
 9831 %}
 9832 
 9833 // Rotate Right by constant.
 9834 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9835 %{
 9836   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9837   match(Set dst (RotateRight dst shift));
 9838   effect(KILL cr);
 9839   format %{ "rorl    $dst, $shift" %}
 9840   ins_encode %{
 9841     __ rorl($dst$$Register, $shift$$constant);
 9842   %}
 9843   ins_pipe(ialu_reg);
 9844 %}
 9845 
 9846 // Rotate Right by constant.
 9847 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9848 %{
 9849   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9850   match(Set dst (RotateRight src shift));
 9851   format %{ "rorxl   $dst, $src, $shift" %}
 9852   ins_encode %{
 9853     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9854   %}
 9855   ins_pipe(ialu_reg_reg);
 9856 %}
 9857 
 9858 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9859 %{
 9860   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9861   match(Set dst (RotateRight (LoadI src) shift));
 9862   ins_cost(175);
 9863   format %{ "rorxl   $dst, $src, $shift" %}
 9864   ins_encode %{
 9865     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9866   %}
 9867   ins_pipe(ialu_reg_mem);
 9868 %}
 9869 
 9870 // Rotate Right by variable
 9871 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9872 %{
 9873   predicate(n->bottom_type()->basic_type() == T_INT);
 9874   match(Set dst (RotateRight dst shift));
 9875   effect(KILL cr);
 9876   format %{ "rorl    $dst, $shift" %}
 9877   ins_encode %{
 9878     __ rorl($dst$$Register);
 9879   %}
 9880   ins_pipe(ialu_reg_reg);
 9881 %}
 9882 
 9883 // Rotate Left by constant.
 9884 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9885 %{
 9886   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9887   match(Set dst (RotateLeft dst shift));
 9888   effect(KILL cr);
 9889   format %{ "rolq    $dst, $shift" %}
 9890   ins_encode %{
 9891     __ rolq($dst$$Register, $shift$$constant);
 9892   %}
 9893   ins_pipe(ialu_reg);
 9894 %}
 9895 
 9896 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9897 %{
 9898   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9899   match(Set dst (RotateLeft src shift));
 9900   format %{ "rolxq   $dst, $src, $shift" %}
 9901   ins_encode %{
 9902     int shift = 64 - ($shift$$constant & 63);
 9903     __ rorxq($dst$$Register, $src$$Register, shift);
 9904   %}
 9905   ins_pipe(ialu_reg_reg);
 9906 %}
 9907 
 9908 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9909 %{
 9910   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9911   match(Set dst (RotateLeft (LoadL src) shift));
 9912   ins_cost(175);
 9913   format %{ "rolxq   $dst, $src, $shift" %}
 9914   ins_encode %{
 9915     int shift = 64 - ($shift$$constant & 63);
 9916     __ rorxq($dst$$Register, $src$$Address, shift);
 9917   %}
 9918   ins_pipe(ialu_reg_mem);
 9919 %}
 9920 
 9921 // Rotate Left by variable
 9922 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9923 %{
 9924   predicate(n->bottom_type()->basic_type() == T_LONG);
 9925   match(Set dst (RotateLeft dst shift));
 9926   effect(KILL cr);
 9927   format %{ "rolq    $dst, $shift" %}
 9928   ins_encode %{
 9929     __ rolq($dst$$Register);
 9930   %}
 9931   ins_pipe(ialu_reg_reg);
 9932 %}
 9933 
 9934 // Rotate Right by constant.
 9935 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9936 %{
 9937   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9938   match(Set dst (RotateRight dst shift));
 9939   effect(KILL cr);
 9940   format %{ "rorq    $dst, $shift" %}
 9941   ins_encode %{
 9942     __ rorq($dst$$Register, $shift$$constant);
 9943   %}
 9944   ins_pipe(ialu_reg);
 9945 %}
 9946 
 9947 // Rotate Right by constant
 9948 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9949 %{
 9950   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9951   match(Set dst (RotateRight src shift));
 9952   format %{ "rorxq   $dst, $src, $shift" %}
 9953   ins_encode %{
 9954     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9955   %}
 9956   ins_pipe(ialu_reg_reg);
 9957 %}
 9958 
 9959 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9960 %{
 9961   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9962   match(Set dst (RotateRight (LoadL src) shift));
 9963   ins_cost(175);
 9964   format %{ "rorxq   $dst, $src, $shift" %}
 9965   ins_encode %{
 9966     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9967   %}
 9968   ins_pipe(ialu_reg_mem);
 9969 %}
 9970 
 9971 // Rotate Right by variable
 9972 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9973 %{
 9974   predicate(n->bottom_type()->basic_type() == T_LONG);
 9975   match(Set dst (RotateRight dst shift));
 9976   effect(KILL cr);
 9977   format %{ "rorq    $dst, $shift" %}
 9978   ins_encode %{
 9979     __ rorq($dst$$Register);
 9980   %}
 9981   ins_pipe(ialu_reg_reg);
 9982 %}
 9983 
 9984 //----------------------------- CompressBits/ExpandBits ------------------------
 9985 
 9986 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9987   predicate(n->bottom_type()->isa_long());
 9988   match(Set dst (CompressBits src mask));
 9989   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9990   ins_encode %{
 9991     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9992   %}
 9993   ins_pipe( pipe_slow );
 9994 %}
 9995 
 9996 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9997   predicate(n->bottom_type()->isa_long());
 9998   match(Set dst (ExpandBits src mask));
 9999   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10000   ins_encode %{
10001     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10002   %}
10003   ins_pipe( pipe_slow );
10004 %}
10005 
10006 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10007   predicate(n->bottom_type()->isa_long());
10008   match(Set dst (CompressBits src (LoadL mask)));
10009   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10010   ins_encode %{
10011     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10012   %}
10013   ins_pipe( pipe_slow );
10014 %}
10015 
10016 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10017   predicate(n->bottom_type()->isa_long());
10018   match(Set dst (ExpandBits src (LoadL mask)));
10019   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10020   ins_encode %{
10021     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10022   %}
10023   ins_pipe( pipe_slow );
10024 %}
10025 
10026 
10027 // Logical Instructions
10028 
10029 // Integer Logical Instructions
10030 
10031 // And Instructions
10032 // And Register with Register
10033 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10034 %{
10035   match(Set dst (AndI dst src));
10036   effect(KILL cr);
10037 
10038   format %{ "andl    $dst, $src\t# int" %}
10039   ins_encode %{
10040     __ andl($dst$$Register, $src$$Register);
10041   %}
10042   ins_pipe(ialu_reg_reg);
10043 %}
10044 
10045 // And Register with Immediate 255
10046 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
10047 %{
10048   match(Set dst (AndI dst src));
10049 
10050   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
10051   ins_encode %{
10052     __ movzbl($dst$$Register, $dst$$Register);
10053   %}
10054   ins_pipe(ialu_reg);
10055 %}
10056 
10057 // And Register with Immediate 255 and promote to long
10058 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10059 %{
10060   match(Set dst (ConvI2L (AndI src mask)));
10061 
10062   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10063   ins_encode %{
10064     __ movzbl($dst$$Register, $src$$Register);
10065   %}
10066   ins_pipe(ialu_reg);
10067 %}
10068 
10069 // And Register with Immediate 65535
10070 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
10071 %{
10072   match(Set dst (AndI dst src));
10073 
10074   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
10075   ins_encode %{
10076     __ movzwl($dst$$Register, $dst$$Register);
10077   %}
10078   ins_pipe(ialu_reg);
10079 %}
10080 
10081 // And Register with Immediate 65535 and promote to long
10082 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10083 %{
10084   match(Set dst (ConvI2L (AndI src mask)));
10085 
10086   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10087   ins_encode %{
10088     __ movzwl($dst$$Register, $src$$Register);
10089   %}
10090   ins_pipe(ialu_reg);
10091 %}
10092 
10093 // Can skip int2long conversions after AND with small bitmask
10094 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10095 %{
10096   predicate(VM_Version::supports_bmi2());
10097   ins_cost(125);
10098   effect(TEMP tmp, KILL cr);
10099   match(Set dst (ConvI2L (AndI src mask)));
10100   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10101   ins_encode %{
10102     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10103     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10104   %}
10105   ins_pipe(ialu_reg_reg);
10106 %}
10107 
10108 // And Register with Immediate
10109 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10110 %{
10111   match(Set dst (AndI dst src));
10112   effect(KILL cr);
10113 
10114   format %{ "andl    $dst, $src\t# int" %}
10115   ins_encode %{
10116     __ andl($dst$$Register, $src$$constant);
10117   %}
10118   ins_pipe(ialu_reg);
10119 %}
10120 
10121 // And Register with Memory
10122 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10123 %{
10124   match(Set dst (AndI dst (LoadI src)));
10125   effect(KILL cr);
10126 
10127   ins_cost(125);
10128   format %{ "andl    $dst, $src\t# int" %}
10129   ins_encode %{
10130     __ andl($dst$$Register, $src$$Address);
10131   %}
10132   ins_pipe(ialu_reg_mem);
10133 %}
10134 
10135 // And Memory with Register
10136 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10137 %{
10138   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10139   effect(KILL cr);
10140 
10141   ins_cost(150);
10142   format %{ "andb    $dst, $src\t# byte" %}
10143   ins_encode %{
10144     __ andb($dst$$Address, $src$$Register);
10145   %}
10146   ins_pipe(ialu_mem_reg);
10147 %}
10148 
10149 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10150 %{
10151   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10152   effect(KILL cr);
10153 
10154   ins_cost(150);
10155   format %{ "andl    $dst, $src\t# int" %}
10156   ins_encode %{
10157     __ andl($dst$$Address, $src$$Register);
10158   %}
10159   ins_pipe(ialu_mem_reg);
10160 %}
10161 
10162 // And Memory with Immediate
10163 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10164 %{
10165   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10166   effect(KILL cr);
10167 
10168   ins_cost(125);
10169   format %{ "andl    $dst, $src\t# int" %}
10170   ins_encode %{
10171     __ andl($dst$$Address, $src$$constant);
10172   %}
10173   ins_pipe(ialu_mem_imm);
10174 %}
10175 
10176 // BMI1 instructions
10177 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10178   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10179   predicate(UseBMI1Instructions);
10180   effect(KILL cr);
10181 
10182   ins_cost(125);
10183   format %{ "andnl  $dst, $src1, $src2" %}
10184 
10185   ins_encode %{
10186     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10187   %}
10188   ins_pipe(ialu_reg_mem);
10189 %}
10190 
10191 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10192   match(Set dst (AndI (XorI src1 minus_1) src2));
10193   predicate(UseBMI1Instructions);
10194   effect(KILL cr);
10195 
10196   format %{ "andnl  $dst, $src1, $src2" %}
10197 
10198   ins_encode %{
10199     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10200   %}
10201   ins_pipe(ialu_reg);
10202 %}
10203 
10204 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10205   match(Set dst (AndI (SubI imm_zero src) src));
10206   predicate(UseBMI1Instructions);
10207   effect(KILL cr);
10208 
10209   format %{ "blsil  $dst, $src" %}
10210 
10211   ins_encode %{
10212     __ blsil($dst$$Register, $src$$Register);
10213   %}
10214   ins_pipe(ialu_reg);
10215 %}
10216 
10217 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10218   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10219   predicate(UseBMI1Instructions);
10220   effect(KILL cr);
10221 
10222   ins_cost(125);
10223   format %{ "blsil  $dst, $src" %}
10224 
10225   ins_encode %{
10226     __ blsil($dst$$Register, $src$$Address);
10227   %}
10228   ins_pipe(ialu_reg_mem);
10229 %}
10230 
10231 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10232 %{
10233   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10234   predicate(UseBMI1Instructions);
10235   effect(KILL cr);
10236 
10237   ins_cost(125);
10238   format %{ "blsmskl $dst, $src" %}
10239 
10240   ins_encode %{
10241     __ blsmskl($dst$$Register, $src$$Address);
10242   %}
10243   ins_pipe(ialu_reg_mem);
10244 %}
10245 
10246 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10247 %{
10248   match(Set dst (XorI (AddI src minus_1) src));
10249   predicate(UseBMI1Instructions);
10250   effect(KILL cr);
10251 
10252   format %{ "blsmskl $dst, $src" %}
10253 
10254   ins_encode %{
10255     __ blsmskl($dst$$Register, $src$$Register);
10256   %}
10257 
10258   ins_pipe(ialu_reg);
10259 %}
10260 
10261 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10262 %{
10263   match(Set dst (AndI (AddI src minus_1) src) );
10264   predicate(UseBMI1Instructions);
10265   effect(KILL cr);
10266 
10267   format %{ "blsrl  $dst, $src" %}
10268 
10269   ins_encode %{
10270     __ blsrl($dst$$Register, $src$$Register);
10271   %}
10272 
10273   ins_pipe(ialu_reg_mem);
10274 %}
10275 
10276 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10277 %{
10278   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10279   predicate(UseBMI1Instructions);
10280   effect(KILL cr);
10281 
10282   ins_cost(125);
10283   format %{ "blsrl  $dst, $src" %}
10284 
10285   ins_encode %{
10286     __ blsrl($dst$$Register, $src$$Address);
10287   %}
10288 
10289   ins_pipe(ialu_reg);
10290 %}
10291 
10292 // Or Instructions
10293 // Or Register with Register
10294 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10295 %{
10296   match(Set dst (OrI dst src));
10297   effect(KILL cr);
10298 
10299   format %{ "orl     $dst, $src\t# int" %}
10300   ins_encode %{
10301     __ orl($dst$$Register, $src$$Register);
10302   %}
10303   ins_pipe(ialu_reg_reg);
10304 %}
10305 
10306 // Or Register with Immediate
10307 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10308 %{
10309   match(Set dst (OrI dst src));
10310   effect(KILL cr);
10311 
10312   format %{ "orl     $dst, $src\t# int" %}
10313   ins_encode %{
10314     __ orl($dst$$Register, $src$$constant);
10315   %}
10316   ins_pipe(ialu_reg);
10317 %}
10318 
10319 // Or Register with Memory
10320 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10321 %{
10322   match(Set dst (OrI dst (LoadI src)));
10323   effect(KILL cr);
10324 
10325   ins_cost(125);
10326   format %{ "orl     $dst, $src\t# int" %}
10327   ins_encode %{
10328     __ orl($dst$$Register, $src$$Address);
10329   %}
10330   ins_pipe(ialu_reg_mem);
10331 %}
10332 
10333 // Or Memory with Register
10334 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10335 %{
10336   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10337   effect(KILL cr);
10338 
10339   ins_cost(150);
10340   format %{ "orb    $dst, $src\t# byte" %}
10341   ins_encode %{
10342     __ orb($dst$$Address, $src$$Register);
10343   %}
10344   ins_pipe(ialu_mem_reg);
10345 %}
10346 
10347 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10348 %{
10349   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10350   effect(KILL cr);
10351 
10352   ins_cost(150);
10353   format %{ "orl     $dst, $src\t# int" %}
10354   ins_encode %{
10355     __ orl($dst$$Address, $src$$Register);
10356   %}
10357   ins_pipe(ialu_mem_reg);
10358 %}
10359 
10360 // Or Memory with Immediate
10361 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10362 %{
10363   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10364   effect(KILL cr);
10365 
10366   ins_cost(125);
10367   format %{ "orl     $dst, $src\t# int" %}
10368   ins_encode %{
10369     __ orl($dst$$Address, $src$$constant);
10370   %}
10371   ins_pipe(ialu_mem_imm);
10372 %}
10373 
10374 // Xor Instructions
10375 // Xor Register with Register
10376 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10377 %{
10378   match(Set dst (XorI dst src));
10379   effect(KILL cr);
10380 
10381   format %{ "xorl    $dst, $src\t# int" %}
10382   ins_encode %{
10383     __ xorl($dst$$Register, $src$$Register);
10384   %}
10385   ins_pipe(ialu_reg_reg);
10386 %}
10387 
10388 // Xor Register with Immediate -1
10389 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10390   match(Set dst (XorI dst imm));
10391 
10392   format %{ "not    $dst" %}
10393   ins_encode %{
10394      __ notl($dst$$Register);
10395   %}
10396   ins_pipe(ialu_reg);
10397 %}
10398 
10399 // Xor Register with Immediate
10400 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10401 %{
10402   match(Set dst (XorI dst src));
10403   effect(KILL cr);
10404 
10405   format %{ "xorl    $dst, $src\t# int" %}
10406   ins_encode %{
10407     __ xorl($dst$$Register, $src$$constant);
10408   %}
10409   ins_pipe(ialu_reg);
10410 %}
10411 
10412 // Xor Register with Memory
10413 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10414 %{
10415   match(Set dst (XorI dst (LoadI src)));
10416   effect(KILL cr);
10417 
10418   ins_cost(125);
10419   format %{ "xorl    $dst, $src\t# int" %}
10420   ins_encode %{
10421     __ xorl($dst$$Register, $src$$Address);
10422   %}
10423   ins_pipe(ialu_reg_mem);
10424 %}
10425 
10426 // Xor Memory with Register
10427 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10428 %{
10429   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10430   effect(KILL cr);
10431 
10432   ins_cost(150);
10433   format %{ "xorb    $dst, $src\t# byte" %}
10434   ins_encode %{
10435     __ xorb($dst$$Address, $src$$Register);
10436   %}
10437   ins_pipe(ialu_mem_reg);
10438 %}
10439 
10440 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10441 %{
10442   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10443   effect(KILL cr);
10444 
10445   ins_cost(150);
10446   format %{ "xorl    $dst, $src\t# int" %}
10447   ins_encode %{
10448     __ xorl($dst$$Address, $src$$Register);
10449   %}
10450   ins_pipe(ialu_mem_reg);
10451 %}
10452 
10453 // Xor Memory with Immediate
10454 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10455 %{
10456   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10457   effect(KILL cr);
10458 
10459   ins_cost(125);
10460   format %{ "xorl    $dst, $src\t# int" %}
10461   ins_encode %{
10462     __ xorl($dst$$Address, $src$$constant);
10463   %}
10464   ins_pipe(ialu_mem_imm);
10465 %}
10466 
10467 
10468 // Long Logical Instructions
10469 
10470 // And Instructions
10471 // And Register with Register
10472 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10473 %{
10474   match(Set dst (AndL dst src));
10475   effect(KILL cr);
10476 
10477   format %{ "andq    $dst, $src\t# long" %}
10478   ins_encode %{
10479     __ andq($dst$$Register, $src$$Register);
10480   %}
10481   ins_pipe(ialu_reg_reg);
10482 %}
10483 
10484 // And Register with Immediate 255
10485 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
10486 %{
10487   match(Set dst (AndL dst src));
10488 
10489   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
10490   ins_encode %{
10491     __ movzbq($dst$$Register, $dst$$Register);
10492   %}
10493   ins_pipe(ialu_reg);
10494 %}
10495 
10496 // And Register with Immediate 65535
10497 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
10498 %{
10499   match(Set dst (AndL dst src));
10500 
10501   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
10502   ins_encode %{
10503     __ movzwq($dst$$Register, $dst$$Register);
10504   %}
10505   ins_pipe(ialu_reg);
10506 %}
10507 
10508 // And Register with Immediate
10509 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10510 %{
10511   match(Set dst (AndL dst src));
10512   effect(KILL cr);
10513 
10514   format %{ "andq    $dst, $src\t# long" %}
10515   ins_encode %{
10516     __ andq($dst$$Register, $src$$constant);
10517   %}
10518   ins_pipe(ialu_reg);
10519 %}
10520 
10521 // And Register with Memory
10522 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10523 %{
10524   match(Set dst (AndL dst (LoadL src)));
10525   effect(KILL cr);
10526 
10527   ins_cost(125);
10528   format %{ "andq    $dst, $src\t# long" %}
10529   ins_encode %{
10530     __ andq($dst$$Register, $src$$Address);
10531   %}
10532   ins_pipe(ialu_reg_mem);
10533 %}
10534 
10535 // And Memory with Register
10536 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10537 %{
10538   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10539   effect(KILL cr);
10540 
10541   ins_cost(150);
10542   format %{ "andq    $dst, $src\t# long" %}
10543   ins_encode %{
10544     __ andq($dst$$Address, $src$$Register);
10545   %}
10546   ins_pipe(ialu_mem_reg);
10547 %}
10548 
10549 // And Memory with Immediate
10550 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10551 %{
10552   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10553   effect(KILL cr);
10554 
10555   ins_cost(125);
10556   format %{ "andq    $dst, $src\t# long" %}
10557   ins_encode %{
10558     __ andq($dst$$Address, $src$$constant);
10559   %}
10560   ins_pipe(ialu_mem_imm);
10561 %}
10562 
10563 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10564 %{
10565   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10566   // because AND/OR works well enough for 8/32-bit values.
10567   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10568 
10569   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10570   effect(KILL cr);
10571 
10572   ins_cost(125);
10573   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10574   ins_encode %{
10575     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10576   %}
10577   ins_pipe(ialu_mem_imm);
10578 %}
10579 
10580 // BMI1 instructions
10581 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10582   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10583   predicate(UseBMI1Instructions);
10584   effect(KILL cr);
10585 
10586   ins_cost(125);
10587   format %{ "andnq  $dst, $src1, $src2" %}
10588 
10589   ins_encode %{
10590     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10591   %}
10592   ins_pipe(ialu_reg_mem);
10593 %}
10594 
10595 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10596   match(Set dst (AndL (XorL src1 minus_1) src2));
10597   predicate(UseBMI1Instructions);
10598   effect(KILL cr);
10599 
10600   format %{ "andnq  $dst, $src1, $src2" %}
10601 
10602   ins_encode %{
10603   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10604   %}
10605   ins_pipe(ialu_reg_mem);
10606 %}
10607 
10608 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10609   match(Set dst (AndL (SubL imm_zero src) src));
10610   predicate(UseBMI1Instructions);
10611   effect(KILL cr);
10612 
10613   format %{ "blsiq  $dst, $src" %}
10614 
10615   ins_encode %{
10616     __ blsiq($dst$$Register, $src$$Register);
10617   %}
10618   ins_pipe(ialu_reg);
10619 %}
10620 
10621 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10622   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10623   predicate(UseBMI1Instructions);
10624   effect(KILL cr);
10625 
10626   ins_cost(125);
10627   format %{ "blsiq  $dst, $src" %}
10628 
10629   ins_encode %{
10630     __ blsiq($dst$$Register, $src$$Address);
10631   %}
10632   ins_pipe(ialu_reg_mem);
10633 %}
10634 
10635 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10636 %{
10637   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10638   predicate(UseBMI1Instructions);
10639   effect(KILL cr);
10640 
10641   ins_cost(125);
10642   format %{ "blsmskq $dst, $src" %}
10643 
10644   ins_encode %{
10645     __ blsmskq($dst$$Register, $src$$Address);
10646   %}
10647   ins_pipe(ialu_reg_mem);
10648 %}
10649 
10650 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10651 %{
10652   match(Set dst (XorL (AddL src minus_1) src));
10653   predicate(UseBMI1Instructions);
10654   effect(KILL cr);
10655 
10656   format %{ "blsmskq $dst, $src" %}
10657 
10658   ins_encode %{
10659     __ blsmskq($dst$$Register, $src$$Register);
10660   %}
10661 
10662   ins_pipe(ialu_reg);
10663 %}
10664 
10665 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10666 %{
10667   match(Set dst (AndL (AddL src minus_1) src) );
10668   predicate(UseBMI1Instructions);
10669   effect(KILL cr);
10670 
10671   format %{ "blsrq  $dst, $src" %}
10672 
10673   ins_encode %{
10674     __ blsrq($dst$$Register, $src$$Register);
10675   %}
10676 
10677   ins_pipe(ialu_reg);
10678 %}
10679 
10680 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10681 %{
10682   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10683   predicate(UseBMI1Instructions);
10684   effect(KILL cr);
10685 
10686   ins_cost(125);
10687   format %{ "blsrq  $dst, $src" %}
10688 
10689   ins_encode %{
10690     __ blsrq($dst$$Register, $src$$Address);
10691   %}
10692 
10693   ins_pipe(ialu_reg);
10694 %}
10695 
10696 // Or Instructions
10697 // Or Register with Register
10698 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10699 %{
10700   match(Set dst (OrL dst src));
10701   effect(KILL cr);
10702 
10703   format %{ "orq     $dst, $src\t# long" %}
10704   ins_encode %{
10705     __ orq($dst$$Register, $src$$Register);
10706   %}
10707   ins_pipe(ialu_reg_reg);
10708 %}
10709 
10710 // Use any_RegP to match R15 (TLS register) without spilling.
10711 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10712   match(Set dst (OrL dst (CastP2X src)));
10713   effect(KILL cr);
10714 
10715   format %{ "orq     $dst, $src\t# long" %}
10716   ins_encode %{
10717     __ orq($dst$$Register, $src$$Register);
10718   %}
10719   ins_pipe(ialu_reg_reg);
10720 %}
10721 
10722 
10723 // Or Register with Immediate
10724 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10725 %{
10726   match(Set dst (OrL dst src));
10727   effect(KILL cr);
10728 
10729   format %{ "orq     $dst, $src\t# long" %}
10730   ins_encode %{
10731     __ orq($dst$$Register, $src$$constant);
10732   %}
10733   ins_pipe(ialu_reg);
10734 %}
10735 
10736 // Or Register with Memory
10737 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10738 %{
10739   match(Set dst (OrL dst (LoadL src)));
10740   effect(KILL cr);
10741 
10742   ins_cost(125);
10743   format %{ "orq     $dst, $src\t# long" %}
10744   ins_encode %{
10745     __ orq($dst$$Register, $src$$Address);
10746   %}
10747   ins_pipe(ialu_reg_mem);
10748 %}
10749 
10750 // Or Memory with Register
10751 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10752 %{
10753   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10754   effect(KILL cr);
10755 
10756   ins_cost(150);
10757   format %{ "orq     $dst, $src\t# long" %}
10758   ins_encode %{
10759     __ orq($dst$$Address, $src$$Register);
10760   %}
10761   ins_pipe(ialu_mem_reg);
10762 %}
10763 
10764 // Or Memory with Immediate
10765 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10766 %{
10767   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10768   effect(KILL cr);
10769 
10770   ins_cost(125);
10771   format %{ "orq     $dst, $src\t# long" %}
10772   ins_encode %{
10773     __ orq($dst$$Address, $src$$constant);
10774   %}
10775   ins_pipe(ialu_mem_imm);
10776 %}
10777 
10778 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10779 %{
10780   // con should be a pure 64-bit power of 2 immediate
10781   // because AND/OR works well enough for 8/32-bit values.
10782   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10783 
10784   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10785   effect(KILL cr);
10786 
10787   ins_cost(125);
10788   format %{ "btsq    $dst, log2($con)\t# long" %}
10789   ins_encode %{
10790     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10791   %}
10792   ins_pipe(ialu_mem_imm);
10793 %}
10794 
10795 // Xor Instructions
10796 // Xor Register with Register
10797 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10798 %{
10799   match(Set dst (XorL dst src));
10800   effect(KILL cr);
10801 
10802   format %{ "xorq    $dst, $src\t# long" %}
10803   ins_encode %{
10804     __ xorq($dst$$Register, $src$$Register);
10805   %}
10806   ins_pipe(ialu_reg_reg);
10807 %}
10808 
10809 // Xor Register with Immediate -1
10810 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10811   match(Set dst (XorL dst imm));
10812 
10813   format %{ "notq   $dst" %}
10814   ins_encode %{
10815      __ notq($dst$$Register);
10816   %}
10817   ins_pipe(ialu_reg);
10818 %}
10819 
10820 // Xor Register with Immediate
10821 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10822 %{
10823   match(Set dst (XorL dst src));
10824   effect(KILL cr);
10825 
10826   format %{ "xorq    $dst, $src\t# long" %}
10827   ins_encode %{
10828     __ xorq($dst$$Register, $src$$constant);
10829   %}
10830   ins_pipe(ialu_reg);
10831 %}
10832 
10833 // Xor Register with Memory
10834 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10835 %{
10836   match(Set dst (XorL dst (LoadL src)));
10837   effect(KILL cr);
10838 
10839   ins_cost(125);
10840   format %{ "xorq    $dst, $src\t# long" %}
10841   ins_encode %{
10842     __ xorq($dst$$Register, $src$$Address);
10843   %}
10844   ins_pipe(ialu_reg_mem);
10845 %}
10846 
10847 // Xor Memory with Register
10848 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10849 %{
10850   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10851   effect(KILL cr);
10852 
10853   ins_cost(150);
10854   format %{ "xorq    $dst, $src\t# long" %}
10855   ins_encode %{
10856     __ xorq($dst$$Address, $src$$Register);
10857   %}
10858   ins_pipe(ialu_mem_reg);
10859 %}
10860 
10861 // Xor Memory with Immediate
10862 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10863 %{
10864   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10865   effect(KILL cr);
10866 
10867   ins_cost(125);
10868   format %{ "xorq    $dst, $src\t# long" %}
10869   ins_encode %{
10870     __ xorq($dst$$Address, $src$$constant);
10871   %}
10872   ins_pipe(ialu_mem_imm);
10873 %}
10874 
10875 // Convert Int to Boolean
10876 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10877 %{
10878   match(Set dst (Conv2B src));
10879   effect(KILL cr);
10880 
10881   format %{ "testl   $src, $src\t# ci2b\n\t"
10882             "setnz   $dst\n\t"
10883             "movzbl  $dst, $dst" %}
10884   ins_encode %{
10885     __ testl($src$$Register, $src$$Register);
10886     __ set_byte_if_not_zero($dst$$Register);
10887     __ movzbl($dst$$Register, $dst$$Register);
10888   %}
10889   ins_pipe(pipe_slow); // XXX
10890 %}
10891 
10892 // Convert Pointer to Boolean
10893 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10894 %{
10895   match(Set dst (Conv2B src));
10896   effect(KILL cr);
10897 
10898   format %{ "testq   $src, $src\t# cp2b\n\t"
10899             "setnz   $dst\n\t"
10900             "movzbl  $dst, $dst" %}
10901   ins_encode %{
10902     __ testq($src$$Register, $src$$Register);
10903     __ set_byte_if_not_zero($dst$$Register);
10904     __ movzbl($dst$$Register, $dst$$Register);
10905   %}
10906   ins_pipe(pipe_slow); // XXX
10907 %}
10908 
10909 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10910 %{
10911   match(Set dst (CmpLTMask p q));
10912   effect(KILL cr);
10913 
10914   ins_cost(400);
10915   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10916             "setlt   $dst\n\t"
10917             "movzbl  $dst, $dst\n\t"
10918             "negl    $dst" %}
10919   ins_encode %{
10920     __ cmpl($p$$Register, $q$$Register);
10921     __ setl($dst$$Register);
10922     __ movzbl($dst$$Register, $dst$$Register);
10923     __ negl($dst$$Register);
10924   %}
10925   ins_pipe(pipe_slow);
10926 %}
10927 
10928 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10929 %{
10930   match(Set dst (CmpLTMask dst zero));
10931   effect(KILL cr);
10932 
10933   ins_cost(100);
10934   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10935   ins_encode %{
10936     __ sarl($dst$$Register, 31);
10937   %}
10938   ins_pipe(ialu_reg);
10939 %}
10940 
10941 /* Better to save a register than avoid a branch */
10942 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10943 %{
10944   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10945   effect(KILL cr);
10946   ins_cost(300);
10947   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10948             "jge     done\n\t"
10949             "addl    $p,$y\n"
10950             "done:   " %}
10951   ins_encode %{
10952     Register Rp = $p$$Register;
10953     Register Rq = $q$$Register;
10954     Register Ry = $y$$Register;
10955     Label done;
10956     __ subl(Rp, Rq);
10957     __ jccb(Assembler::greaterEqual, done);
10958     __ addl(Rp, Ry);
10959     __ bind(done);
10960   %}
10961   ins_pipe(pipe_cmplt);
10962 %}
10963 
10964 /* Better to save a register than avoid a branch */
10965 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10966 %{
10967   match(Set y (AndI (CmpLTMask p q) y));
10968   effect(KILL cr);
10969 
10970   ins_cost(300);
10971 
10972   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10973             "jlt     done\n\t"
10974             "xorl    $y, $y\n"
10975             "done:   " %}
10976   ins_encode %{
10977     Register Rp = $p$$Register;
10978     Register Rq = $q$$Register;
10979     Register Ry = $y$$Register;
10980     Label done;
10981     __ cmpl(Rp, Rq);
10982     __ jccb(Assembler::less, done);
10983     __ xorl(Ry, Ry);
10984     __ bind(done);
10985   %}
10986   ins_pipe(pipe_cmplt);
10987 %}
10988 
10989 
10990 //---------- FP Instructions------------------------------------------------
10991 
10992 // Really expensive, avoid
10993 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10994 %{
10995   match(Set cr (CmpF src1 src2));
10996 
10997   ins_cost(500);
10998   format %{ "ucomiss $src1, $src2\n\t"
10999             "jnp,s   exit\n\t"
11000             "pushfq\t# saw NaN, set CF\n\t"
11001             "andq    [rsp], #0xffffff2b\n\t"
11002             "popfq\n"
11003     "exit:" %}
11004   ins_encode %{
11005     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11006     emit_cmpfp_fixup(_masm);
11007   %}
11008   ins_pipe(pipe_slow);
11009 %}
11010 
11011 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11012   match(Set cr (CmpF src1 src2));
11013 
11014   ins_cost(100);
11015   format %{ "ucomiss $src1, $src2" %}
11016   ins_encode %{
11017     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11018   %}
11019   ins_pipe(pipe_slow);
11020 %}
11021 
11022 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11023   match(Set cr (CmpF src1 (LoadF src2)));
11024 
11025   ins_cost(100);
11026   format %{ "ucomiss $src1, $src2" %}
11027   ins_encode %{
11028     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11029   %}
11030   ins_pipe(pipe_slow);
11031 %}
11032 
11033 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11034   match(Set cr (CmpF src con));
11035   ins_cost(100);
11036   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11037   ins_encode %{
11038     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11039   %}
11040   ins_pipe(pipe_slow);
11041 %}
11042 
11043 // Really expensive, avoid
11044 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11045 %{
11046   match(Set cr (CmpD src1 src2));
11047 
11048   ins_cost(500);
11049   format %{ "ucomisd $src1, $src2\n\t"
11050             "jnp,s   exit\n\t"
11051             "pushfq\t# saw NaN, set CF\n\t"
11052             "andq    [rsp], #0xffffff2b\n\t"
11053             "popfq\n"
11054     "exit:" %}
11055   ins_encode %{
11056     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11057     emit_cmpfp_fixup(_masm);
11058   %}
11059   ins_pipe(pipe_slow);
11060 %}
11061 
11062 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11063   match(Set cr (CmpD src1 src2));
11064 
11065   ins_cost(100);
11066   format %{ "ucomisd $src1, $src2 test" %}
11067   ins_encode %{
11068     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11069   %}
11070   ins_pipe(pipe_slow);
11071 %}
11072 
11073 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11074   match(Set cr (CmpD src1 (LoadD src2)));
11075 
11076   ins_cost(100);
11077   format %{ "ucomisd $src1, $src2" %}
11078   ins_encode %{
11079     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11080   %}
11081   ins_pipe(pipe_slow);
11082 %}
11083 
11084 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11085   match(Set cr (CmpD src con));
11086   ins_cost(100);
11087   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11088   ins_encode %{
11089     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11090   %}
11091   ins_pipe(pipe_slow);
11092 %}
11093 
11094 // Compare into -1,0,1
11095 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11096 %{
11097   match(Set dst (CmpF3 src1 src2));
11098   effect(KILL cr);
11099 
11100   ins_cost(275);
11101   format %{ "ucomiss $src1, $src2\n\t"
11102             "movl    $dst, #-1\n\t"
11103             "jp,s    done\n\t"
11104             "jb,s    done\n\t"
11105             "setne   $dst\n\t"
11106             "movzbl  $dst, $dst\n"
11107     "done:" %}
11108   ins_encode %{
11109     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11110     emit_cmpfp3(_masm, $dst$$Register);
11111   %}
11112   ins_pipe(pipe_slow);
11113 %}
11114 
11115 // Compare into -1,0,1
11116 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11117 %{
11118   match(Set dst (CmpF3 src1 (LoadF src2)));
11119   effect(KILL cr);
11120 
11121   ins_cost(275);
11122   format %{ "ucomiss $src1, $src2\n\t"
11123             "movl    $dst, #-1\n\t"
11124             "jp,s    done\n\t"
11125             "jb,s    done\n\t"
11126             "setne   $dst\n\t"
11127             "movzbl  $dst, $dst\n"
11128     "done:" %}
11129   ins_encode %{
11130     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11131     emit_cmpfp3(_masm, $dst$$Register);
11132   %}
11133   ins_pipe(pipe_slow);
11134 %}
11135 
11136 // Compare into -1,0,1
11137 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11138   match(Set dst (CmpF3 src con));
11139   effect(KILL cr);
11140 
11141   ins_cost(275);
11142   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11143             "movl    $dst, #-1\n\t"
11144             "jp,s    done\n\t"
11145             "jb,s    done\n\t"
11146             "setne   $dst\n\t"
11147             "movzbl  $dst, $dst\n"
11148     "done:" %}
11149   ins_encode %{
11150     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11151     emit_cmpfp3(_masm, $dst$$Register);
11152   %}
11153   ins_pipe(pipe_slow);
11154 %}
11155 
11156 // Compare into -1,0,1
11157 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11158 %{
11159   match(Set dst (CmpD3 src1 src2));
11160   effect(KILL cr);
11161 
11162   ins_cost(275);
11163   format %{ "ucomisd $src1, $src2\n\t"
11164             "movl    $dst, #-1\n\t"
11165             "jp,s    done\n\t"
11166             "jb,s    done\n\t"
11167             "setne   $dst\n\t"
11168             "movzbl  $dst, $dst\n"
11169     "done:" %}
11170   ins_encode %{
11171     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11172     emit_cmpfp3(_masm, $dst$$Register);
11173   %}
11174   ins_pipe(pipe_slow);
11175 %}
11176 
11177 // Compare into -1,0,1
11178 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11179 %{
11180   match(Set dst (CmpD3 src1 (LoadD src2)));
11181   effect(KILL cr);
11182 
11183   ins_cost(275);
11184   format %{ "ucomisd $src1, $src2\n\t"
11185             "movl    $dst, #-1\n\t"
11186             "jp,s    done\n\t"
11187             "jb,s    done\n\t"
11188             "setne   $dst\n\t"
11189             "movzbl  $dst, $dst\n"
11190     "done:" %}
11191   ins_encode %{
11192     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11193     emit_cmpfp3(_masm, $dst$$Register);
11194   %}
11195   ins_pipe(pipe_slow);
11196 %}
11197 
11198 // Compare into -1,0,1
11199 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11200   match(Set dst (CmpD3 src con));
11201   effect(KILL cr);
11202 
11203   ins_cost(275);
11204   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11205             "movl    $dst, #-1\n\t"
11206             "jp,s    done\n\t"
11207             "jb,s    done\n\t"
11208             "setne   $dst\n\t"
11209             "movzbl  $dst, $dst\n"
11210     "done:" %}
11211   ins_encode %{
11212     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11213     emit_cmpfp3(_masm, $dst$$Register);
11214   %}
11215   ins_pipe(pipe_slow);
11216 %}
11217 
11218 //----------Arithmetic Conversion Instructions---------------------------------
11219 
11220 instruct convF2D_reg_reg(regD dst, regF src)
11221 %{
11222   match(Set dst (ConvF2D src));
11223 
11224   format %{ "cvtss2sd $dst, $src" %}
11225   ins_encode %{
11226     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11227   %}
11228   ins_pipe(pipe_slow); // XXX
11229 %}
11230 
11231 instruct convF2D_reg_mem(regD dst, memory src)
11232 %{
11233   match(Set dst (ConvF2D (LoadF src)));
11234 
11235   format %{ "cvtss2sd $dst, $src" %}
11236   ins_encode %{
11237     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11238   %}
11239   ins_pipe(pipe_slow); // XXX
11240 %}
11241 
11242 instruct convD2F_reg_reg(regF dst, regD src)
11243 %{
11244   match(Set dst (ConvD2F src));
11245 
11246   format %{ "cvtsd2ss $dst, $src" %}
11247   ins_encode %{
11248     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11249   %}
11250   ins_pipe(pipe_slow); // XXX
11251 %}
11252 
11253 instruct convD2F_reg_mem(regF dst, memory src)
11254 %{
11255   match(Set dst (ConvD2F (LoadD src)));
11256 
11257   format %{ "cvtsd2ss $dst, $src" %}
11258   ins_encode %{
11259     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11260   %}
11261   ins_pipe(pipe_slow); // XXX
11262 %}
11263 
11264 // XXX do mem variants
11265 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11266 %{
11267   match(Set dst (ConvF2I src));
11268   effect(KILL cr);
11269   format %{ "convert_f2i $dst,$src" %}
11270   ins_encode %{
11271     __ convert_f2i($dst$$Register, $src$$XMMRegister);
11272   %}
11273   ins_pipe(pipe_slow);
11274 %}
11275 
11276 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11277 %{
11278   match(Set dst (ConvF2L src));
11279   effect(KILL cr);
11280   format %{ "convert_f2l $dst,$src"%}
11281   ins_encode %{
11282     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11283   %}
11284   ins_pipe(pipe_slow);
11285 %}
11286 
11287 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11288 %{
11289   match(Set dst (ConvD2I src));
11290   effect(KILL cr);
11291   format %{ "convert_d2i $dst,$src"%}
11292   ins_encode %{
11293     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11294   %}
11295   ins_pipe(pipe_slow);
11296 %}
11297 
11298 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11299 %{
11300   match(Set dst (ConvD2L src));
11301   effect(KILL cr);
11302   format %{ "convert_d2l $dst,$src"%}
11303   ins_encode %{
11304     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11305   %}
11306   ins_pipe(pipe_slow);
11307 %}
11308 
11309 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11310 %{
11311   match(Set dst (RoundD src));
11312   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11313   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11314   ins_encode %{
11315     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11316   %}
11317   ins_pipe(pipe_slow);
11318 %}
11319 
11320 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11321 %{
11322   match(Set dst (RoundF src));
11323   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11324   format %{ "round_float $dst,$src" %}
11325   ins_encode %{
11326     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11327   %}
11328   ins_pipe(pipe_slow);
11329 %}
11330 
11331 instruct convI2F_reg_reg(regF dst, rRegI src)
11332 %{
11333   predicate(!UseXmmI2F);
11334   match(Set dst (ConvI2F src));
11335 
11336   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11337   ins_encode %{
11338     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11339   %}
11340   ins_pipe(pipe_slow); // XXX
11341 %}
11342 
11343 instruct convI2F_reg_mem(regF dst, memory src)
11344 %{
11345   match(Set dst (ConvI2F (LoadI src)));
11346 
11347   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11348   ins_encode %{
11349     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11350   %}
11351   ins_pipe(pipe_slow); // XXX
11352 %}
11353 
11354 instruct convI2D_reg_reg(regD dst, rRegI src)
11355 %{
11356   predicate(!UseXmmI2D);
11357   match(Set dst (ConvI2D src));
11358 
11359   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11360   ins_encode %{
11361     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11362   %}
11363   ins_pipe(pipe_slow); // XXX
11364 %}
11365 
11366 instruct convI2D_reg_mem(regD dst, memory src)
11367 %{
11368   match(Set dst (ConvI2D (LoadI src)));
11369 
11370   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11371   ins_encode %{
11372     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11373   %}
11374   ins_pipe(pipe_slow); // XXX
11375 %}
11376 
11377 instruct convXI2F_reg(regF dst, rRegI src)
11378 %{
11379   predicate(UseXmmI2F);
11380   match(Set dst (ConvI2F src));
11381 
11382   format %{ "movdl $dst, $src\n\t"
11383             "cvtdq2psl $dst, $dst\t# i2f" %}
11384   ins_encode %{
11385     __ movdl($dst$$XMMRegister, $src$$Register);
11386     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11387   %}
11388   ins_pipe(pipe_slow); // XXX
11389 %}
11390 
11391 instruct convXI2D_reg(regD dst, rRegI src)
11392 %{
11393   predicate(UseXmmI2D);
11394   match(Set dst (ConvI2D src));
11395 
11396   format %{ "movdl $dst, $src\n\t"
11397             "cvtdq2pdl $dst, $dst\t# i2d" %}
11398   ins_encode %{
11399     __ movdl($dst$$XMMRegister, $src$$Register);
11400     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11401   %}
11402   ins_pipe(pipe_slow); // XXX
11403 %}
11404 
11405 instruct convL2F_reg_reg(regF dst, rRegL src)
11406 %{
11407   match(Set dst (ConvL2F src));
11408 
11409   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11410   ins_encode %{
11411     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11412   %}
11413   ins_pipe(pipe_slow); // XXX
11414 %}
11415 
11416 instruct convL2F_reg_mem(regF dst, memory src)
11417 %{
11418   match(Set dst (ConvL2F (LoadL src)));
11419 
11420   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11421   ins_encode %{
11422     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11423   %}
11424   ins_pipe(pipe_slow); // XXX
11425 %}
11426 
11427 instruct convL2D_reg_reg(regD dst, rRegL src)
11428 %{
11429   match(Set dst (ConvL2D src));
11430 
11431   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11432   ins_encode %{
11433     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11434   %}
11435   ins_pipe(pipe_slow); // XXX
11436 %}
11437 
11438 instruct convL2D_reg_mem(regD dst, memory src)
11439 %{
11440   match(Set dst (ConvL2D (LoadL src)));
11441 
11442   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11443   ins_encode %{
11444     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11445   %}
11446   ins_pipe(pipe_slow); // XXX
11447 %}
11448 
11449 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11450 %{
11451   match(Set dst (ConvI2L src));
11452 
11453   ins_cost(125);
11454   format %{ "movslq  $dst, $src\t# i2l" %}
11455   ins_encode %{
11456     __ movslq($dst$$Register, $src$$Register);
11457   %}
11458   ins_pipe(ialu_reg_reg);
11459 %}
11460 
11461 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11462 // %{
11463 //   match(Set dst (ConvI2L src));
11464 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11465 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11466 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11467 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11468 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11469 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11470 
11471 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11472 //   ins_encode(enc_copy(dst, src));
11473 // //   opcode(0x63); // needs REX.W
11474 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11475 //   ins_pipe(ialu_reg_reg);
11476 // %}
11477 
11478 // Zero-extend convert int to long
11479 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11480 %{
11481   match(Set dst (AndL (ConvI2L src) mask));
11482 
11483   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11484   ins_encode %{
11485     if ($dst$$reg != $src$$reg) {
11486       __ movl($dst$$Register, $src$$Register);
11487     }
11488   %}
11489   ins_pipe(ialu_reg_reg);
11490 %}
11491 
11492 // Zero-extend convert int to long
11493 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11494 %{
11495   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11496 
11497   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11498   ins_encode %{
11499     __ movl($dst$$Register, $src$$Address);
11500   %}
11501   ins_pipe(ialu_reg_mem);
11502 %}
11503 
11504 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11505 %{
11506   match(Set dst (AndL src mask));
11507 
11508   format %{ "movl    $dst, $src\t# zero-extend long" %}
11509   ins_encode %{
11510     __ movl($dst$$Register, $src$$Register);
11511   %}
11512   ins_pipe(ialu_reg_reg);
11513 %}
11514 
11515 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11516 %{
11517   match(Set dst (ConvL2I src));
11518 
11519   format %{ "movl    $dst, $src\t# l2i" %}
11520   ins_encode %{
11521     __ movl($dst$$Register, $src$$Register);
11522   %}
11523   ins_pipe(ialu_reg_reg);
11524 %}
11525 
11526 
11527 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11528   match(Set dst (MoveF2I src));
11529   effect(DEF dst, USE src);
11530 
11531   ins_cost(125);
11532   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11533   ins_encode %{
11534     __ movl($dst$$Register, Address(rsp, $src$$disp));
11535   %}
11536   ins_pipe(ialu_reg_mem);
11537 %}
11538 
11539 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11540   match(Set dst (MoveI2F src));
11541   effect(DEF dst, USE src);
11542 
11543   ins_cost(125);
11544   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11545   ins_encode %{
11546     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11547   %}
11548   ins_pipe(pipe_slow);
11549 %}
11550 
11551 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11552   match(Set dst (MoveD2L src));
11553   effect(DEF dst, USE src);
11554 
11555   ins_cost(125);
11556   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11557   ins_encode %{
11558     __ movq($dst$$Register, Address(rsp, $src$$disp));
11559   %}
11560   ins_pipe(ialu_reg_mem);
11561 %}
11562 
11563 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11564   predicate(!UseXmmLoadAndClearUpper);
11565   match(Set dst (MoveL2D src));
11566   effect(DEF dst, USE src);
11567 
11568   ins_cost(125);
11569   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11570   ins_encode %{
11571     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11572   %}
11573   ins_pipe(pipe_slow);
11574 %}
11575 
11576 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11577   predicate(UseXmmLoadAndClearUpper);
11578   match(Set dst (MoveL2D src));
11579   effect(DEF dst, USE src);
11580 
11581   ins_cost(125);
11582   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11583   ins_encode %{
11584     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11585   %}
11586   ins_pipe(pipe_slow);
11587 %}
11588 
11589 
11590 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11591   match(Set dst (MoveF2I src));
11592   effect(DEF dst, USE src);
11593 
11594   ins_cost(95); // XXX
11595   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11596   ins_encode %{
11597     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11598   %}
11599   ins_pipe(pipe_slow);
11600 %}
11601 
11602 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11603   match(Set dst (MoveI2F src));
11604   effect(DEF dst, USE src);
11605 
11606   ins_cost(100);
11607   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11608   ins_encode %{
11609     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11610   %}
11611   ins_pipe( ialu_mem_reg );
11612 %}
11613 
11614 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11615   match(Set dst (MoveD2L src));
11616   effect(DEF dst, USE src);
11617 
11618   ins_cost(95); // XXX
11619   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11620   ins_encode %{
11621     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11622   %}
11623   ins_pipe(pipe_slow);
11624 %}
11625 
11626 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11627   match(Set dst (MoveL2D src));
11628   effect(DEF dst, USE src);
11629 
11630   ins_cost(100);
11631   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11632   ins_encode %{
11633     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11634   %}
11635   ins_pipe(ialu_mem_reg);
11636 %}
11637 
11638 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11639   match(Set dst (MoveF2I src));
11640   effect(DEF dst, USE src);
11641   ins_cost(85);
11642   format %{ "movd    $dst,$src\t# MoveF2I" %}
11643   ins_encode %{
11644     __ movdl($dst$$Register, $src$$XMMRegister);
11645   %}
11646   ins_pipe( pipe_slow );
11647 %}
11648 
11649 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11650   match(Set dst (MoveD2L src));
11651   effect(DEF dst, USE src);
11652   ins_cost(85);
11653   format %{ "movd    $dst,$src\t# MoveD2L" %}
11654   ins_encode %{
11655     __ movdq($dst$$Register, $src$$XMMRegister);
11656   %}
11657   ins_pipe( pipe_slow );
11658 %}
11659 
11660 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11661   match(Set dst (MoveI2F src));
11662   effect(DEF dst, USE src);
11663   ins_cost(100);
11664   format %{ "movd    $dst,$src\t# MoveI2F" %}
11665   ins_encode %{
11666     __ movdl($dst$$XMMRegister, $src$$Register);
11667   %}
11668   ins_pipe( pipe_slow );
11669 %}
11670 
11671 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11672   match(Set dst (MoveL2D src));
11673   effect(DEF dst, USE src);
11674   ins_cost(100);
11675   format %{ "movd    $dst,$src\t# MoveL2D" %}
11676   ins_encode %{
11677      __ movdq($dst$$XMMRegister, $src$$Register);
11678   %}
11679   ins_pipe( pipe_slow );
11680 %}
11681 
11682 // Fast clearing of an array
11683 // Small ClearArray non-AVX512.
11684 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11685                   Universe dummy, rFlagsReg cr)
11686 %{
11687   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11688   match(Set dummy (ClearArray cnt base));
11689   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11690 
11691   format %{ $$template
11692     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11693     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11694     $$emit$$"jg      LARGE\n\t"
11695     $$emit$$"dec     rcx\n\t"
11696     $$emit$$"js      DONE\t# Zero length\n\t"
11697     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11698     $$emit$$"dec     rcx\n\t"
11699     $$emit$$"jge     LOOP\n\t"
11700     $$emit$$"jmp     DONE\n\t"
11701     $$emit$$"# LARGE:\n\t"
11702     if (UseFastStosb) {
11703        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11704        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11705     } else if (UseXMMForObjInit) {
11706        $$emit$$"mov     rdi,rax\n\t"
11707        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11708        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11709        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11710        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11711        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11712        $$emit$$"add     0x40,rax\n\t"
11713        $$emit$$"# L_zero_64_bytes:\n\t"
11714        $$emit$$"sub     0x8,rcx\n\t"
11715        $$emit$$"jge     L_loop\n\t"
11716        $$emit$$"add     0x4,rcx\n\t"
11717        $$emit$$"jl      L_tail\n\t"
11718        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11719        $$emit$$"add     0x20,rax\n\t"
11720        $$emit$$"sub     0x4,rcx\n\t"
11721        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11722        $$emit$$"add     0x4,rcx\n\t"
11723        $$emit$$"jle     L_end\n\t"
11724        $$emit$$"dec     rcx\n\t"
11725        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11726        $$emit$$"vmovq   xmm0,(rax)\n\t"
11727        $$emit$$"add     0x8,rax\n\t"
11728        $$emit$$"dec     rcx\n\t"
11729        $$emit$$"jge     L_sloop\n\t"
11730        $$emit$$"# L_end:\n\t"
11731     } else {
11732        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11733     }
11734     $$emit$$"# DONE"
11735   %}
11736   ins_encode %{
11737     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11738                  $tmp$$XMMRegister, false, knoreg);
11739   %}
11740   ins_pipe(pipe_slow);
11741 %}
11742 
11743 // Small ClearArray AVX512 non-constant length.
11744 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11745                        Universe dummy, rFlagsReg cr)
11746 %{
11747   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11748   match(Set dummy (ClearArray cnt base));
11749   ins_cost(125);
11750   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11751 
11752   format %{ $$template
11753     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11754     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11755     $$emit$$"jg      LARGE\n\t"
11756     $$emit$$"dec     rcx\n\t"
11757     $$emit$$"js      DONE\t# Zero length\n\t"
11758     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11759     $$emit$$"dec     rcx\n\t"
11760     $$emit$$"jge     LOOP\n\t"
11761     $$emit$$"jmp     DONE\n\t"
11762     $$emit$$"# LARGE:\n\t"
11763     if (UseFastStosb) {
11764        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11765        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11766     } else if (UseXMMForObjInit) {
11767        $$emit$$"mov     rdi,rax\n\t"
11768        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11769        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11770        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11771        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11772        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11773        $$emit$$"add     0x40,rax\n\t"
11774        $$emit$$"# L_zero_64_bytes:\n\t"
11775        $$emit$$"sub     0x8,rcx\n\t"
11776        $$emit$$"jge     L_loop\n\t"
11777        $$emit$$"add     0x4,rcx\n\t"
11778        $$emit$$"jl      L_tail\n\t"
11779        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11780        $$emit$$"add     0x20,rax\n\t"
11781        $$emit$$"sub     0x4,rcx\n\t"
11782        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11783        $$emit$$"add     0x4,rcx\n\t"
11784        $$emit$$"jle     L_end\n\t"
11785        $$emit$$"dec     rcx\n\t"
11786        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11787        $$emit$$"vmovq   xmm0,(rax)\n\t"
11788        $$emit$$"add     0x8,rax\n\t"
11789        $$emit$$"dec     rcx\n\t"
11790        $$emit$$"jge     L_sloop\n\t"
11791        $$emit$$"# L_end:\n\t"
11792     } else {
11793        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11794     }
11795     $$emit$$"# DONE"
11796   %}
11797   ins_encode %{
11798     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11799                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11800   %}
11801   ins_pipe(pipe_slow);
11802 %}
11803 
11804 // Large ClearArray non-AVX512.
11805 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11806                         Universe dummy, rFlagsReg cr)
11807 %{
11808   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11809   match(Set dummy (ClearArray cnt base));
11810   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11811 
11812   format %{ $$template
11813     if (UseFastStosb) {
11814        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11815        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11816        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11817     } else if (UseXMMForObjInit) {
11818        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11819        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11820        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11821        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11822        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11823        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11824        $$emit$$"add     0x40,rax\n\t"
11825        $$emit$$"# L_zero_64_bytes:\n\t"
11826        $$emit$$"sub     0x8,rcx\n\t"
11827        $$emit$$"jge     L_loop\n\t"
11828        $$emit$$"add     0x4,rcx\n\t"
11829        $$emit$$"jl      L_tail\n\t"
11830        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11831        $$emit$$"add     0x20,rax\n\t"
11832        $$emit$$"sub     0x4,rcx\n\t"
11833        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11834        $$emit$$"add     0x4,rcx\n\t"
11835        $$emit$$"jle     L_end\n\t"
11836        $$emit$$"dec     rcx\n\t"
11837        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11838        $$emit$$"vmovq   xmm0,(rax)\n\t"
11839        $$emit$$"add     0x8,rax\n\t"
11840        $$emit$$"dec     rcx\n\t"
11841        $$emit$$"jge     L_sloop\n\t"
11842        $$emit$$"# L_end:\n\t"
11843     } else {
11844        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11845        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11846     }
11847   %}
11848   ins_encode %{
11849     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11850                  $tmp$$XMMRegister, true, knoreg);
11851   %}
11852   ins_pipe(pipe_slow);
11853 %}
11854 
11855 // Large ClearArray AVX512.
11856 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11857                              Universe dummy, rFlagsReg cr)
11858 %{
11859   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11860   match(Set dummy (ClearArray cnt base));
11861   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11862 
11863   format %{ $$template
11864     if (UseFastStosb) {
11865        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11866        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11867        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11868     } else if (UseXMMForObjInit) {
11869        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11870        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11871        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11872        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11873        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11874        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11875        $$emit$$"add     0x40,rax\n\t"
11876        $$emit$$"# L_zero_64_bytes:\n\t"
11877        $$emit$$"sub     0x8,rcx\n\t"
11878        $$emit$$"jge     L_loop\n\t"
11879        $$emit$$"add     0x4,rcx\n\t"
11880        $$emit$$"jl      L_tail\n\t"
11881        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11882        $$emit$$"add     0x20,rax\n\t"
11883        $$emit$$"sub     0x4,rcx\n\t"
11884        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11885        $$emit$$"add     0x4,rcx\n\t"
11886        $$emit$$"jle     L_end\n\t"
11887        $$emit$$"dec     rcx\n\t"
11888        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11889        $$emit$$"vmovq   xmm0,(rax)\n\t"
11890        $$emit$$"add     0x8,rax\n\t"
11891        $$emit$$"dec     rcx\n\t"
11892        $$emit$$"jge     L_sloop\n\t"
11893        $$emit$$"# L_end:\n\t"
11894     } else {
11895        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11896        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11897     }
11898   %}
11899   ins_encode %{
11900     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11901                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11902   %}
11903   ins_pipe(pipe_slow);
11904 %}
11905 
11906 // Small ClearArray AVX512 constant length.
11907 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11908 %{
11909   predicate(!((ClearArrayNode*)n)->is_large() &&
11910               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11911   match(Set dummy (ClearArray cnt base));
11912   ins_cost(100);
11913   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11914   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11915   ins_encode %{
11916    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11917   %}
11918   ins_pipe(pipe_slow);
11919 %}
11920 
11921 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11922                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11923 %{
11924   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11925   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11926   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11927 
11928   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11929   ins_encode %{
11930     __ string_compare($str1$$Register, $str2$$Register,
11931                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11932                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11933   %}
11934   ins_pipe( pipe_slow );
11935 %}
11936 
11937 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11938                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11939 %{
11940   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11941   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11942   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11943 
11944   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11945   ins_encode %{
11946     __ string_compare($str1$$Register, $str2$$Register,
11947                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11948                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11949   %}
11950   ins_pipe( pipe_slow );
11951 %}
11952 
11953 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11954                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11955 %{
11956   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11957   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11958   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11959 
11960   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11961   ins_encode %{
11962     __ string_compare($str1$$Register, $str2$$Register,
11963                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11964                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11965   %}
11966   ins_pipe( pipe_slow );
11967 %}
11968 
11969 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11970                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11971 %{
11972   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11973   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11974   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11975 
11976   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11977   ins_encode %{
11978     __ string_compare($str1$$Register, $str2$$Register,
11979                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11980                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11981   %}
11982   ins_pipe( pipe_slow );
11983 %}
11984 
11985 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11986                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11987 %{
11988   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11989   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11990   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11991 
11992   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11993   ins_encode %{
11994     __ string_compare($str1$$Register, $str2$$Register,
11995                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11996                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11997   %}
11998   ins_pipe( pipe_slow );
11999 %}
12000 
12001 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12002                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12003 %{
12004   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12005   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12006   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12007 
12008   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12009   ins_encode %{
12010     __ string_compare($str1$$Register, $str2$$Register,
12011                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12012                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12013   %}
12014   ins_pipe( pipe_slow );
12015 %}
12016 
12017 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12018                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12019 %{
12020   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12021   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12022   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12023 
12024   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12025   ins_encode %{
12026     __ string_compare($str2$$Register, $str1$$Register,
12027                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12028                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12029   %}
12030   ins_pipe( pipe_slow );
12031 %}
12032 
12033 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12034                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12035 %{
12036   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12037   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12038   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12039 
12040   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12041   ins_encode %{
12042     __ string_compare($str2$$Register, $str1$$Register,
12043                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12044                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12045   %}
12046   ins_pipe( pipe_slow );
12047 %}
12048 
12049 // fast search of substring with known size.
12050 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12051                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12052 %{
12053   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12054   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12055   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12056 
12057   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12058   ins_encode %{
12059     int icnt2 = (int)$int_cnt2$$constant;
12060     if (icnt2 >= 16) {
12061       // IndexOf for constant substrings with size >= 16 elements
12062       // which don't need to be loaded through stack.
12063       __ string_indexofC8($str1$$Register, $str2$$Register,
12064                           $cnt1$$Register, $cnt2$$Register,
12065                           icnt2, $result$$Register,
12066                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12067     } else {
12068       // Small strings are loaded through stack if they cross page boundary.
12069       __ string_indexof($str1$$Register, $str2$$Register,
12070                         $cnt1$$Register, $cnt2$$Register,
12071                         icnt2, $result$$Register,
12072                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12073     }
12074   %}
12075   ins_pipe( pipe_slow );
12076 %}
12077 
12078 // fast search of substring with known size.
12079 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12080                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12081 %{
12082   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12083   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12084   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12085 
12086   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12087   ins_encode %{
12088     int icnt2 = (int)$int_cnt2$$constant;
12089     if (icnt2 >= 8) {
12090       // IndexOf for constant substrings with size >= 8 elements
12091       // which don't need to be loaded through stack.
12092       __ string_indexofC8($str1$$Register, $str2$$Register,
12093                           $cnt1$$Register, $cnt2$$Register,
12094                           icnt2, $result$$Register,
12095                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12096     } else {
12097       // Small strings are loaded through stack if they cross page boundary.
12098       __ string_indexof($str1$$Register, $str2$$Register,
12099                         $cnt1$$Register, $cnt2$$Register,
12100                         icnt2, $result$$Register,
12101                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12102     }
12103   %}
12104   ins_pipe( pipe_slow );
12105 %}
12106 
12107 // fast search of substring with known size.
12108 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12109                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12110 %{
12111   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12112   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12113   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12114 
12115   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12116   ins_encode %{
12117     int icnt2 = (int)$int_cnt2$$constant;
12118     if (icnt2 >= 8) {
12119       // IndexOf for constant substrings with size >= 8 elements
12120       // which don't need to be loaded through stack.
12121       __ string_indexofC8($str1$$Register, $str2$$Register,
12122                           $cnt1$$Register, $cnt2$$Register,
12123                           icnt2, $result$$Register,
12124                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12125     } else {
12126       // Small strings are loaded through stack if they cross page boundary.
12127       __ string_indexof($str1$$Register, $str2$$Register,
12128                         $cnt1$$Register, $cnt2$$Register,
12129                         icnt2, $result$$Register,
12130                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12131     }
12132   %}
12133   ins_pipe( pipe_slow );
12134 %}
12135 
12136 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12137                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12138 %{
12139   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12140   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12141   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12142 
12143   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12144   ins_encode %{
12145     __ string_indexof($str1$$Register, $str2$$Register,
12146                       $cnt1$$Register, $cnt2$$Register,
12147                       (-1), $result$$Register,
12148                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12149   %}
12150   ins_pipe( pipe_slow );
12151 %}
12152 
12153 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12154                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12155 %{
12156   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12157   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12158   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12159 
12160   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12161   ins_encode %{
12162     __ string_indexof($str1$$Register, $str2$$Register,
12163                       $cnt1$$Register, $cnt2$$Register,
12164                       (-1), $result$$Register,
12165                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12166   %}
12167   ins_pipe( pipe_slow );
12168 %}
12169 
12170 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12171                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12172 %{
12173   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12174   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12175   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12176 
12177   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12178   ins_encode %{
12179     __ string_indexof($str1$$Register, $str2$$Register,
12180                       $cnt1$$Register, $cnt2$$Register,
12181                       (-1), $result$$Register,
12182                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12183   %}
12184   ins_pipe( pipe_slow );
12185 %}
12186 
12187 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12188                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12189 %{
12190   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12191   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12192   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12193   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12194   ins_encode %{
12195     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12196                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12197   %}
12198   ins_pipe( pipe_slow );
12199 %}
12200 
12201 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12202                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12203 %{
12204   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12205   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12206   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12207   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12208   ins_encode %{
12209     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12210                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12211   %}
12212   ins_pipe( pipe_slow );
12213 %}
12214 
12215 // fast string equals
12216 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12217                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12218 %{
12219   predicate(!VM_Version::supports_avx512vlbw());
12220   match(Set result (StrEquals (Binary str1 str2) cnt));
12221   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12222 
12223   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12224   ins_encode %{
12225     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12226                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12227                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12228   %}
12229   ins_pipe( pipe_slow );
12230 %}
12231 
12232 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12233                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12234 %{
12235   predicate(VM_Version::supports_avx512vlbw());
12236   match(Set result (StrEquals (Binary str1 str2) cnt));
12237   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12238 
12239   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12240   ins_encode %{
12241     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12242                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12243                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12244   %}
12245   ins_pipe( pipe_slow );
12246 %}
12247 
12248 // fast array equals
12249 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12250                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12251 %{
12252   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12253   match(Set result (AryEq ary1 ary2));
12254   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12255 
12256   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12257   ins_encode %{
12258     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12259                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12260                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12261   %}
12262   ins_pipe( pipe_slow );
12263 %}
12264 
12265 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12266                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12267 %{
12268   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12269   match(Set result (AryEq ary1 ary2));
12270   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12271 
12272   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12273   ins_encode %{
12274     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12275                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12276                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12277   %}
12278   ins_pipe( pipe_slow );
12279 %}
12280 
12281 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12282                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12283 %{
12284   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12285   match(Set result (AryEq ary1 ary2));
12286   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12287 
12288   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12289   ins_encode %{
12290     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12291                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12292                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12293   %}
12294   ins_pipe( pipe_slow );
12295 %}
12296 
12297 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12298                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12299 %{
12300   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12301   match(Set result (AryEq ary1 ary2));
12302   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12303 
12304   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12305   ins_encode %{
12306     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12307                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12308                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12309   %}
12310   ins_pipe( pipe_slow );
12311 %}
12312 
12313 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12314                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12315 %{
12316   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12317   match(Set result (CountPositives ary1 len));
12318   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12319 
12320   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12321   ins_encode %{
12322     __ count_positives($ary1$$Register, $len$$Register,
12323                        $result$$Register, $tmp3$$Register,
12324                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12325   %}
12326   ins_pipe( pipe_slow );
12327 %}
12328 
12329 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12330                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12331 %{
12332   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12333   match(Set result (CountPositives ary1 len));
12334   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12335 
12336   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12337   ins_encode %{
12338     __ count_positives($ary1$$Register, $len$$Register,
12339                        $result$$Register, $tmp3$$Register,
12340                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12341   %}
12342   ins_pipe( pipe_slow );
12343 %}
12344 
12345 // fast char[] to byte[] compression
12346 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12347                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12348   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12349   match(Set result (StrCompressedCopy src (Binary dst len)));
12350   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12351          USE_KILL len, KILL tmp5, KILL cr);
12352 
12353   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12354   ins_encode %{
12355     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12356                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12357                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12358                            knoreg, knoreg);
12359   %}
12360   ins_pipe( pipe_slow );
12361 %}
12362 
12363 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12364                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12365   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12366   match(Set result (StrCompressedCopy src (Binary dst len)));
12367   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12368          USE_KILL len, KILL tmp5, KILL cr);
12369 
12370   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12371   ins_encode %{
12372     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12373                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12374                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12375                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12376   %}
12377   ins_pipe( pipe_slow );
12378 %}
12379 // fast byte[] to char[] inflation
12380 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12381                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12382   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12383   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12384   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12385 
12386   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12387   ins_encode %{
12388     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12389                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12390   %}
12391   ins_pipe( pipe_slow );
12392 %}
12393 
12394 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12395                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12396   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12397   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12398   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12399 
12400   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12401   ins_encode %{
12402     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12403                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12404   %}
12405   ins_pipe( pipe_slow );
12406 %}
12407 
12408 // encode char[] to byte[] in ISO_8859_1
12409 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12410                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12411                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12412   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12413   match(Set result (EncodeISOArray src (Binary dst len)));
12414   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12415 
12416   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12417   ins_encode %{
12418     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12419                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12420                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12421   %}
12422   ins_pipe( pipe_slow );
12423 %}
12424 
12425 // encode char[] to byte[] in ASCII
12426 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12427                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12428                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12429   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12430   match(Set result (EncodeISOArray src (Binary dst len)));
12431   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12432 
12433   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12434   ins_encode %{
12435     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12436                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12437                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12438   %}
12439   ins_pipe( pipe_slow );
12440 %}
12441 
12442 //----------Overflow Math Instructions-----------------------------------------
12443 
12444 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12445 %{
12446   match(Set cr (OverflowAddI op1 op2));
12447   effect(DEF cr, USE_KILL op1, USE op2);
12448 
12449   format %{ "addl    $op1, $op2\t# overflow check int" %}
12450 
12451   ins_encode %{
12452     __ addl($op1$$Register, $op2$$Register);
12453   %}
12454   ins_pipe(ialu_reg_reg);
12455 %}
12456 
12457 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12458 %{
12459   match(Set cr (OverflowAddI op1 op2));
12460   effect(DEF cr, USE_KILL op1, USE op2);
12461 
12462   format %{ "addl    $op1, $op2\t# overflow check int" %}
12463 
12464   ins_encode %{
12465     __ addl($op1$$Register, $op2$$constant);
12466   %}
12467   ins_pipe(ialu_reg_reg);
12468 %}
12469 
12470 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12471 %{
12472   match(Set cr (OverflowAddL op1 op2));
12473   effect(DEF cr, USE_KILL op1, USE op2);
12474 
12475   format %{ "addq    $op1, $op2\t# overflow check long" %}
12476   ins_encode %{
12477     __ addq($op1$$Register, $op2$$Register);
12478   %}
12479   ins_pipe(ialu_reg_reg);
12480 %}
12481 
12482 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12483 %{
12484   match(Set cr (OverflowAddL op1 op2));
12485   effect(DEF cr, USE_KILL op1, USE op2);
12486 
12487   format %{ "addq    $op1, $op2\t# overflow check long" %}
12488   ins_encode %{
12489     __ addq($op1$$Register, $op2$$constant);
12490   %}
12491   ins_pipe(ialu_reg_reg);
12492 %}
12493 
12494 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12495 %{
12496   match(Set cr (OverflowSubI op1 op2));
12497 
12498   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12499   ins_encode %{
12500     __ cmpl($op1$$Register, $op2$$Register);
12501   %}
12502   ins_pipe(ialu_reg_reg);
12503 %}
12504 
12505 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12506 %{
12507   match(Set cr (OverflowSubI op1 op2));
12508 
12509   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12510   ins_encode %{
12511     __ cmpl($op1$$Register, $op2$$constant);
12512   %}
12513   ins_pipe(ialu_reg_reg);
12514 %}
12515 
12516 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12517 %{
12518   match(Set cr (OverflowSubL op1 op2));
12519 
12520   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12521   ins_encode %{
12522     __ cmpq($op1$$Register, $op2$$Register);
12523   %}
12524   ins_pipe(ialu_reg_reg);
12525 %}
12526 
12527 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12528 %{
12529   match(Set cr (OverflowSubL op1 op2));
12530 
12531   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12532   ins_encode %{
12533     __ cmpq($op1$$Register, $op2$$constant);
12534   %}
12535   ins_pipe(ialu_reg_reg);
12536 %}
12537 
12538 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12539 %{
12540   match(Set cr (OverflowSubI zero op2));
12541   effect(DEF cr, USE_KILL op2);
12542 
12543   format %{ "negl    $op2\t# overflow check int" %}
12544   ins_encode %{
12545     __ negl($op2$$Register);
12546   %}
12547   ins_pipe(ialu_reg_reg);
12548 %}
12549 
12550 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12551 %{
12552   match(Set cr (OverflowSubL zero op2));
12553   effect(DEF cr, USE_KILL op2);
12554 
12555   format %{ "negq    $op2\t# overflow check long" %}
12556   ins_encode %{
12557     __ negq($op2$$Register);
12558   %}
12559   ins_pipe(ialu_reg_reg);
12560 %}
12561 
12562 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12563 %{
12564   match(Set cr (OverflowMulI op1 op2));
12565   effect(DEF cr, USE_KILL op1, USE op2);
12566 
12567   format %{ "imull    $op1, $op2\t# overflow check int" %}
12568   ins_encode %{
12569     __ imull($op1$$Register, $op2$$Register);
12570   %}
12571   ins_pipe(ialu_reg_reg_alu0);
12572 %}
12573 
12574 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12575 %{
12576   match(Set cr (OverflowMulI op1 op2));
12577   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12578 
12579   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12580   ins_encode %{
12581     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12582   %}
12583   ins_pipe(ialu_reg_reg_alu0);
12584 %}
12585 
12586 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12587 %{
12588   match(Set cr (OverflowMulL op1 op2));
12589   effect(DEF cr, USE_KILL op1, USE op2);
12590 
12591   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12592   ins_encode %{
12593     __ imulq($op1$$Register, $op2$$Register);
12594   %}
12595   ins_pipe(ialu_reg_reg_alu0);
12596 %}
12597 
12598 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12599 %{
12600   match(Set cr (OverflowMulL op1 op2));
12601   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12602 
12603   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12604   ins_encode %{
12605     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12606   %}
12607   ins_pipe(ialu_reg_reg_alu0);
12608 %}
12609 
12610 
12611 //----------Control Flow Instructions------------------------------------------
12612 // Signed compare Instructions
12613 
12614 // XXX more variants!!
12615 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12616 %{
12617   match(Set cr (CmpI op1 op2));
12618   effect(DEF cr, USE op1, USE op2);
12619 
12620   format %{ "cmpl    $op1, $op2" %}
12621   ins_encode %{
12622     __ cmpl($op1$$Register, $op2$$Register);
12623   %}
12624   ins_pipe(ialu_cr_reg_reg);
12625 %}
12626 
12627 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12628 %{
12629   match(Set cr (CmpI op1 op2));
12630 
12631   format %{ "cmpl    $op1, $op2" %}
12632   ins_encode %{
12633     __ cmpl($op1$$Register, $op2$$constant);
12634   %}
12635   ins_pipe(ialu_cr_reg_imm);
12636 %}
12637 
12638 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12639 %{
12640   match(Set cr (CmpI op1 (LoadI op2)));
12641 
12642   ins_cost(500); // XXX
12643   format %{ "cmpl    $op1, $op2" %}
12644   ins_encode %{
12645     __ cmpl($op1$$Register, $op2$$Address);
12646   %}
12647   ins_pipe(ialu_cr_reg_mem);
12648 %}
12649 
12650 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12651 %{
12652   match(Set cr (CmpI src zero));
12653 
12654   format %{ "testl   $src, $src" %}
12655   ins_encode %{
12656     __ testl($src$$Register, $src$$Register);
12657   %}
12658   ins_pipe(ialu_cr_reg_imm);
12659 %}
12660 
12661 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12662 %{
12663   match(Set cr (CmpI (AndI src con) zero));
12664 
12665   format %{ "testl   $src, $con" %}
12666   ins_encode %{
12667     __ testl($src$$Register, $con$$constant);
12668   %}
12669   ins_pipe(ialu_cr_reg_imm);
12670 %}
12671 
12672 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12673 %{
12674   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12675 
12676   format %{ "testl   $src, $mem" %}
12677   ins_encode %{
12678     __ testl($src$$Register, $mem$$Address);
12679   %}
12680   ins_pipe(ialu_cr_reg_mem);
12681 %}
12682 
12683 // Unsigned compare Instructions; really, same as signed except they
12684 // produce an rFlagsRegU instead of rFlagsReg.
12685 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12686 %{
12687   match(Set cr (CmpU op1 op2));
12688 
12689   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12690   ins_encode %{
12691     __ cmpl($op1$$Register, $op2$$Register);
12692   %}
12693   ins_pipe(ialu_cr_reg_reg);
12694 %}
12695 
12696 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12697 %{
12698   match(Set cr (CmpU op1 op2));
12699 
12700   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12701   ins_encode %{
12702     __ cmpl($op1$$Register, $op2$$constant);
12703   %}
12704   ins_pipe(ialu_cr_reg_imm);
12705 %}
12706 
12707 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12708 %{
12709   match(Set cr (CmpU op1 (LoadI op2)));
12710 
12711   ins_cost(500); // XXX
12712   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12713   ins_encode %{
12714     __ cmpl($op1$$Register, $op2$$Address);
12715   %}
12716   ins_pipe(ialu_cr_reg_mem);
12717 %}
12718 
12719 // // // Cisc-spilled version of cmpU_rReg
12720 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12721 // //%{
12722 // //  match(Set cr (CmpU (LoadI op1) op2));
12723 // //
12724 // //  format %{ "CMPu   $op1,$op2" %}
12725 // //  ins_cost(500);
12726 // //  opcode(0x39);  /* Opcode 39 /r */
12727 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12728 // //%}
12729 
12730 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12731 %{
12732   match(Set cr (CmpU src zero));
12733 
12734   format %{ "testl   $src, $src\t# unsigned" %}
12735   ins_encode %{
12736     __ testl($src$$Register, $src$$Register);
12737   %}
12738   ins_pipe(ialu_cr_reg_imm);
12739 %}
12740 
12741 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12742 %{
12743   match(Set cr (CmpP op1 op2));
12744 
12745   format %{ "cmpq    $op1, $op2\t# ptr" %}
12746   ins_encode %{
12747     __ cmpq($op1$$Register, $op2$$Register);
12748   %}
12749   ins_pipe(ialu_cr_reg_reg);
12750 %}
12751 
12752 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12753 %{
12754   match(Set cr (CmpP op1 (LoadP op2)));
12755   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12756 
12757   ins_cost(500); // XXX
12758   format %{ "cmpq    $op1, $op2\t# ptr" %}
12759   ins_encode %{
12760     __ cmpq($op1$$Register, $op2$$Address);
12761   %}
12762   ins_pipe(ialu_cr_reg_mem);
12763 %}
12764 
12765 // // // Cisc-spilled version of cmpP_rReg
12766 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12767 // //%{
12768 // //  match(Set cr (CmpP (LoadP op1) op2));
12769 // //
12770 // //  format %{ "CMPu   $op1,$op2" %}
12771 // //  ins_cost(500);
12772 // //  opcode(0x39);  /* Opcode 39 /r */
12773 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12774 // //%}
12775 
12776 // XXX this is generalized by compP_rReg_mem???
12777 // Compare raw pointer (used in out-of-heap check).
12778 // Only works because non-oop pointers must be raw pointers
12779 // and raw pointers have no anti-dependencies.
12780 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12781 %{
12782   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12783             n->in(2)->as_Load()->barrier_data() == 0);
12784   match(Set cr (CmpP op1 (LoadP op2)));
12785 
12786   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12787   ins_encode %{
12788     __ cmpq($op1$$Register, $op2$$Address);
12789   %}
12790   ins_pipe(ialu_cr_reg_mem);
12791 %}
12792 
12793 // This will generate a signed flags result. This should be OK since
12794 // any compare to a zero should be eq/neq.
12795 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12796 %{
12797   match(Set cr (CmpP src zero));
12798 
12799   format %{ "testq   $src, $src\t# ptr" %}
12800   ins_encode %{
12801     __ testq($src$$Register, $src$$Register);
12802   %}
12803   ins_pipe(ialu_cr_reg_imm);
12804 %}
12805 
12806 // This will generate a signed flags result. This should be OK since
12807 // any compare to a zero should be eq/neq.
12808 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12809 %{
12810   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12811             n->in(1)->as_Load()->barrier_data() == 0);
12812   match(Set cr (CmpP (LoadP op) zero));
12813 
12814   ins_cost(500); // XXX
12815   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12816   ins_encode %{
12817     __ testq($op$$Address, 0xFFFFFFFF);
12818   %}
12819   ins_pipe(ialu_cr_reg_imm);
12820 %}
12821 
12822 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12823 %{
12824   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12825             n->in(1)->as_Load()->barrier_data() == 0);
12826   match(Set cr (CmpP (LoadP mem) zero));
12827 
12828   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12829   ins_encode %{
12830     __ cmpq(r12, $mem$$Address);
12831   %}
12832   ins_pipe(ialu_cr_reg_mem);
12833 %}
12834 
12835 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12836 %{
12837   match(Set cr (CmpN op1 op2));
12838 
12839   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12840   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12841   ins_pipe(ialu_cr_reg_reg);
12842 %}
12843 
12844 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12845 %{
12846   match(Set cr (CmpN src (LoadN mem)));
12847 
12848   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12849   ins_encode %{
12850     __ cmpl($src$$Register, $mem$$Address);
12851   %}
12852   ins_pipe(ialu_cr_reg_mem);
12853 %}
12854 
12855 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12856   match(Set cr (CmpN op1 op2));
12857 
12858   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12859   ins_encode %{
12860     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12861   %}
12862   ins_pipe(ialu_cr_reg_imm);
12863 %}
12864 
12865 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12866 %{
12867   match(Set cr (CmpN src (LoadN mem)));
12868 
12869   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12870   ins_encode %{
12871     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12872   %}
12873   ins_pipe(ialu_cr_reg_mem);
12874 %}
12875 
12876 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12877   match(Set cr (CmpN op1 op2));
12878 
12879   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12880   ins_encode %{
12881     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12882   %}
12883   ins_pipe(ialu_cr_reg_imm);
12884 %}
12885 
12886 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12887 %{
12888   match(Set cr (CmpN src (LoadNKlass mem)));
12889 
12890   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12891   ins_encode %{
12892     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12893   %}
12894   ins_pipe(ialu_cr_reg_mem);
12895 %}
12896 
12897 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12898   match(Set cr (CmpN src zero));
12899 
12900   format %{ "testl   $src, $src\t# compressed ptr" %}
12901   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12902   ins_pipe(ialu_cr_reg_imm);
12903 %}
12904 
12905 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12906 %{
12907   predicate(CompressedOops::base() != NULL);
12908   match(Set cr (CmpN (LoadN mem) zero));
12909 
12910   ins_cost(500); // XXX
12911   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12912   ins_encode %{
12913     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12914   %}
12915   ins_pipe(ialu_cr_reg_mem);
12916 %}
12917 
12918 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12919 %{
12920   predicate(CompressedOops::base() == NULL);
12921   match(Set cr (CmpN (LoadN mem) zero));
12922 
12923   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12924   ins_encode %{
12925     __ cmpl(r12, $mem$$Address);
12926   %}
12927   ins_pipe(ialu_cr_reg_mem);
12928 %}
12929 
12930 // Yanked all unsigned pointer compare operations.
12931 // Pointer compares are done with CmpP which is already unsigned.
12932 
12933 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12934 %{
12935   match(Set cr (CmpL op1 op2));
12936 
12937   format %{ "cmpq    $op1, $op2" %}
12938   ins_encode %{
12939     __ cmpq($op1$$Register, $op2$$Register);
12940   %}
12941   ins_pipe(ialu_cr_reg_reg);
12942 %}
12943 
12944 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12945 %{
12946   match(Set cr (CmpL op1 op2));
12947 
12948   format %{ "cmpq    $op1, $op2" %}
12949   ins_encode %{
12950     __ cmpq($op1$$Register, $op2$$constant);
12951   %}
12952   ins_pipe(ialu_cr_reg_imm);
12953 %}
12954 
12955 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12956 %{
12957   match(Set cr (CmpL op1 (LoadL op2)));
12958 
12959   format %{ "cmpq    $op1, $op2" %}
12960   ins_encode %{
12961     __ cmpq($op1$$Register, $op2$$Address);
12962   %}
12963   ins_pipe(ialu_cr_reg_mem);
12964 %}
12965 
12966 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12967 %{
12968   match(Set cr (CmpL src zero));
12969 
12970   format %{ "testq   $src, $src" %}
12971   ins_encode %{
12972     __ testq($src$$Register, $src$$Register);
12973   %}
12974   ins_pipe(ialu_cr_reg_imm);
12975 %}
12976 
12977 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12978 %{
12979   match(Set cr (CmpL (AndL src con) zero));
12980 
12981   format %{ "testq   $src, $con\t# long" %}
12982   ins_encode %{
12983     __ testq($src$$Register, $con$$constant);
12984   %}
12985   ins_pipe(ialu_cr_reg_imm);
12986 %}
12987 
12988 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12989 %{
12990   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12991 
12992   format %{ "testq   $src, $mem" %}
12993   ins_encode %{
12994     __ testq($src$$Register, $mem$$Address);
12995   %}
12996   ins_pipe(ialu_cr_reg_mem);
12997 %}
12998 
12999 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13000 %{
13001   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13002 
13003   format %{ "testq   $src, $mem" %}
13004   ins_encode %{
13005     __ testq($src$$Register, $mem$$Address);
13006   %}
13007   ins_pipe(ialu_cr_reg_mem);
13008 %}
13009 
13010 // Manifest a CmpU result in an integer register.  Very painful.
13011 // This is the test to avoid.
13012 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13013 %{
13014   match(Set dst (CmpU3 src1 src2));
13015   effect(KILL flags);
13016 
13017   ins_cost(275); // XXX
13018   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13019             "movl    $dst, -1\n\t"
13020             "jb,u    done\n\t"
13021             "setne   $dst\n\t"
13022             "movzbl  $dst, $dst\n\t"
13023     "done:" %}
13024   ins_encode %{
13025     Label done;
13026     __ cmpl($src1$$Register, $src2$$Register);
13027     __ movl($dst$$Register, -1);
13028     __ jccb(Assembler::below, done);
13029     __ setne($dst$$Register);
13030     __ movzbl($dst$$Register, $dst$$Register);
13031     __ bind(done);
13032   %}
13033   ins_pipe(pipe_slow);
13034 %}
13035 
13036 // Manifest a CmpL result in an integer register.  Very painful.
13037 // This is the test to avoid.
13038 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13039 %{
13040   match(Set dst (CmpL3 src1 src2));
13041   effect(KILL flags);
13042 
13043   ins_cost(275); // XXX
13044   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13045             "movl    $dst, -1\n\t"
13046             "jl,s    done\n\t"
13047             "setne   $dst\n\t"
13048             "movzbl  $dst, $dst\n\t"
13049     "done:" %}
13050   ins_encode %{
13051     Label done;
13052     __ cmpq($src1$$Register, $src2$$Register);
13053     __ movl($dst$$Register, -1);
13054     __ jccb(Assembler::less, done);
13055     __ setne($dst$$Register);
13056     __ movzbl($dst$$Register, $dst$$Register);
13057     __ bind(done);
13058   %}
13059   ins_pipe(pipe_slow);
13060 %}
13061 
13062 // Manifest a CmpUL result in an integer register.  Very painful.
13063 // This is the test to avoid.
13064 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13065 %{
13066   match(Set dst (CmpUL3 src1 src2));
13067   effect(KILL flags);
13068 
13069   ins_cost(275); // XXX
13070   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13071             "movl    $dst, -1\n\t"
13072             "jb,u    done\n\t"
13073             "setne   $dst\n\t"
13074             "movzbl  $dst, $dst\n\t"
13075     "done:" %}
13076   ins_encode %{
13077     Label done;
13078     __ cmpq($src1$$Register, $src2$$Register);
13079     __ movl($dst$$Register, -1);
13080     __ jccb(Assembler::below, done);
13081     __ setne($dst$$Register);
13082     __ movzbl($dst$$Register, $dst$$Register);
13083     __ bind(done);
13084   %}
13085   ins_pipe(pipe_slow);
13086 %}
13087 
13088 // Unsigned long compare Instructions; really, same as signed long except they
13089 // produce an rFlagsRegU instead of rFlagsReg.
13090 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13091 %{
13092   match(Set cr (CmpUL op1 op2));
13093 
13094   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13095   ins_encode %{
13096     __ cmpq($op1$$Register, $op2$$Register);
13097   %}
13098   ins_pipe(ialu_cr_reg_reg);
13099 %}
13100 
13101 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13102 %{
13103   match(Set cr (CmpUL op1 op2));
13104 
13105   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13106   ins_encode %{
13107     __ cmpq($op1$$Register, $op2$$constant);
13108   %}
13109   ins_pipe(ialu_cr_reg_imm);
13110 %}
13111 
13112 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13113 %{
13114   match(Set cr (CmpUL op1 (LoadL op2)));
13115 
13116   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13117   ins_encode %{
13118     __ cmpq($op1$$Register, $op2$$Address);
13119   %}
13120   ins_pipe(ialu_cr_reg_mem);
13121 %}
13122 
13123 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13124 %{
13125   match(Set cr (CmpUL src zero));
13126 
13127   format %{ "testq   $src, $src\t# unsigned" %}
13128   ins_encode %{
13129     __ testq($src$$Register, $src$$Register);
13130   %}
13131   ins_pipe(ialu_cr_reg_imm);
13132 %}
13133 
13134 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13135 %{
13136   match(Set cr (CmpI (LoadB mem) imm));
13137 
13138   ins_cost(125);
13139   format %{ "cmpb    $mem, $imm" %}
13140   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13141   ins_pipe(ialu_cr_reg_mem);
13142 %}
13143 
13144 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13145 %{
13146   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13147 
13148   ins_cost(125);
13149   format %{ "testb   $mem, $imm\t# ubyte" %}
13150   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13151   ins_pipe(ialu_cr_reg_mem);
13152 %}
13153 
13154 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13155 %{
13156   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13157 
13158   ins_cost(125);
13159   format %{ "testb   $mem, $imm\t# byte" %}
13160   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13161   ins_pipe(ialu_cr_reg_mem);
13162 %}
13163 
13164 //----------Max and Min--------------------------------------------------------
13165 // Min Instructions
13166 
13167 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13168 %{
13169   effect(USE_DEF dst, USE src, USE cr);
13170 
13171   format %{ "cmovlgt $dst, $src\t# min" %}
13172   ins_encode %{
13173     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13174   %}
13175   ins_pipe(pipe_cmov_reg);
13176 %}
13177 
13178 
13179 instruct minI_rReg(rRegI dst, rRegI src)
13180 %{
13181   match(Set dst (MinI dst src));
13182 
13183   ins_cost(200);
13184   expand %{
13185     rFlagsReg cr;
13186     compI_rReg(cr, dst, src);
13187     cmovI_reg_g(dst, src, cr);
13188   %}
13189 %}
13190 
13191 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13192 %{
13193   effect(USE_DEF dst, USE src, USE cr);
13194 
13195   format %{ "cmovllt $dst, $src\t# max" %}
13196   ins_encode %{
13197     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13198   %}
13199   ins_pipe(pipe_cmov_reg);
13200 %}
13201 
13202 
13203 instruct maxI_rReg(rRegI dst, rRegI src)
13204 %{
13205   match(Set dst (MaxI dst src));
13206 
13207   ins_cost(200);
13208   expand %{
13209     rFlagsReg cr;
13210     compI_rReg(cr, dst, src);
13211     cmovI_reg_l(dst, src, cr);
13212   %}
13213 %}
13214 
13215 // ============================================================================
13216 // Branch Instructions
13217 
13218 // Jump Direct - Label defines a relative address from JMP+1
13219 instruct jmpDir(label labl)
13220 %{
13221   match(Goto);
13222   effect(USE labl);
13223 
13224   ins_cost(300);
13225   format %{ "jmp     $labl" %}
13226   size(5);
13227   ins_encode %{
13228     Label* L = $labl$$label;
13229     __ jmp(*L, false); // Always long jump
13230   %}
13231   ins_pipe(pipe_jmp);
13232 %}
13233 
13234 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13235 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13236 %{
13237   match(If cop cr);
13238   effect(USE labl);
13239 
13240   ins_cost(300);
13241   format %{ "j$cop     $labl" %}
13242   size(6);
13243   ins_encode %{
13244     Label* L = $labl$$label;
13245     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13246   %}
13247   ins_pipe(pipe_jcc);
13248 %}
13249 
13250 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13251 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13252 %{
13253   match(CountedLoopEnd cop cr);
13254   effect(USE labl);
13255 
13256   ins_cost(300);
13257   format %{ "j$cop     $labl\t# loop end" %}
13258   size(6);
13259   ins_encode %{
13260     Label* L = $labl$$label;
13261     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13262   %}
13263   ins_pipe(pipe_jcc);
13264 %}
13265 
13266 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13267 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13268   match(CountedLoopEnd cop cmp);
13269   effect(USE labl);
13270 
13271   ins_cost(300);
13272   format %{ "j$cop,u   $labl\t# loop end" %}
13273   size(6);
13274   ins_encode %{
13275     Label* L = $labl$$label;
13276     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13277   %}
13278   ins_pipe(pipe_jcc);
13279 %}
13280 
13281 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13282   match(CountedLoopEnd cop cmp);
13283   effect(USE labl);
13284 
13285   ins_cost(200);
13286   format %{ "j$cop,u   $labl\t# loop end" %}
13287   size(6);
13288   ins_encode %{
13289     Label* L = $labl$$label;
13290     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13291   %}
13292   ins_pipe(pipe_jcc);
13293 %}
13294 
13295 // Jump Direct Conditional - using unsigned comparison
13296 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13297   match(If cop cmp);
13298   effect(USE labl);
13299 
13300   ins_cost(300);
13301   format %{ "j$cop,u   $labl" %}
13302   size(6);
13303   ins_encode %{
13304     Label* L = $labl$$label;
13305     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13306   %}
13307   ins_pipe(pipe_jcc);
13308 %}
13309 
13310 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13311   match(If cop cmp);
13312   effect(USE labl);
13313 
13314   ins_cost(200);
13315   format %{ "j$cop,u   $labl" %}
13316   size(6);
13317   ins_encode %{
13318     Label* L = $labl$$label;
13319     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13320   %}
13321   ins_pipe(pipe_jcc);
13322 %}
13323 
13324 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13325   match(If cop cmp);
13326   effect(USE labl);
13327 
13328   ins_cost(200);
13329   format %{ $$template
13330     if ($cop$$cmpcode == Assembler::notEqual) {
13331       $$emit$$"jp,u    $labl\n\t"
13332       $$emit$$"j$cop,u   $labl"
13333     } else {
13334       $$emit$$"jp,u    done\n\t"
13335       $$emit$$"j$cop,u   $labl\n\t"
13336       $$emit$$"done:"
13337     }
13338   %}
13339   ins_encode %{
13340     Label* l = $labl$$label;
13341     if ($cop$$cmpcode == Assembler::notEqual) {
13342       __ jcc(Assembler::parity, *l, false);
13343       __ jcc(Assembler::notEqual, *l, false);
13344     } else if ($cop$$cmpcode == Assembler::equal) {
13345       Label done;
13346       __ jccb(Assembler::parity, done);
13347       __ jcc(Assembler::equal, *l, false);
13348       __ bind(done);
13349     } else {
13350        ShouldNotReachHere();
13351     }
13352   %}
13353   ins_pipe(pipe_jcc);
13354 %}
13355 
13356 // ============================================================================
13357 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13358 // superklass array for an instance of the superklass.  Set a hidden
13359 // internal cache on a hit (cache is checked with exposed code in
13360 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13361 // encoding ALSO sets flags.
13362 
13363 instruct partialSubtypeCheck(rdi_RegP result,
13364                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13365                              rFlagsReg cr)
13366 %{
13367   match(Set result (PartialSubtypeCheck sub super));
13368   effect(KILL rcx, KILL cr);
13369 
13370   ins_cost(1100);  // slightly larger than the next version
13371   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13372             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13373             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13374             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13375             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13376             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13377             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13378     "miss:\t" %}
13379 
13380   opcode(0x1); // Force a XOR of RDI
13381   ins_encode(enc_PartialSubtypeCheck());
13382   ins_pipe(pipe_slow);
13383 %}
13384 
13385 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13386                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13387                                      immP0 zero,
13388                                      rdi_RegP result)
13389 %{
13390   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13391   effect(KILL rcx, KILL result);
13392 
13393   ins_cost(1000);
13394   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13395             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13396             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13397             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13398             "jne,s   miss\t\t# Missed: flags nz\n\t"
13399             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13400     "miss:\t" %}
13401 
13402   opcode(0x0); // No need to XOR RDI
13403   ins_encode(enc_PartialSubtypeCheck());
13404   ins_pipe(pipe_slow);
13405 %}
13406 
13407 // ============================================================================
13408 // Branch Instructions -- short offset versions
13409 //
13410 // These instructions are used to replace jumps of a long offset (the default
13411 // match) with jumps of a shorter offset.  These instructions are all tagged
13412 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13413 // match rules in general matching.  Instead, the ADLC generates a conversion
13414 // method in the MachNode which can be used to do in-place replacement of the
13415 // long variant with the shorter variant.  The compiler will determine if a
13416 // branch can be taken by the is_short_branch_offset() predicate in the machine
13417 // specific code section of the file.
13418 
13419 // Jump Direct - Label defines a relative address from JMP+1
13420 instruct jmpDir_short(label labl) %{
13421   match(Goto);
13422   effect(USE labl);
13423 
13424   ins_cost(300);
13425   format %{ "jmp,s   $labl" %}
13426   size(2);
13427   ins_encode %{
13428     Label* L = $labl$$label;
13429     __ jmpb(*L);
13430   %}
13431   ins_pipe(pipe_jmp);
13432   ins_short_branch(1);
13433 %}
13434 
13435 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13436 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13437   match(If cop cr);
13438   effect(USE labl);
13439 
13440   ins_cost(300);
13441   format %{ "j$cop,s   $labl" %}
13442   size(2);
13443   ins_encode %{
13444     Label* L = $labl$$label;
13445     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13446   %}
13447   ins_pipe(pipe_jcc);
13448   ins_short_branch(1);
13449 %}
13450 
13451 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13452 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13453   match(CountedLoopEnd cop cr);
13454   effect(USE labl);
13455 
13456   ins_cost(300);
13457   format %{ "j$cop,s   $labl\t# loop end" %}
13458   size(2);
13459   ins_encode %{
13460     Label* L = $labl$$label;
13461     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13462   %}
13463   ins_pipe(pipe_jcc);
13464   ins_short_branch(1);
13465 %}
13466 
13467 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13468 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13469   match(CountedLoopEnd cop cmp);
13470   effect(USE labl);
13471 
13472   ins_cost(300);
13473   format %{ "j$cop,us  $labl\t# loop end" %}
13474   size(2);
13475   ins_encode %{
13476     Label* L = $labl$$label;
13477     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13478   %}
13479   ins_pipe(pipe_jcc);
13480   ins_short_branch(1);
13481 %}
13482 
13483 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13484   match(CountedLoopEnd cop cmp);
13485   effect(USE labl);
13486 
13487   ins_cost(300);
13488   format %{ "j$cop,us  $labl\t# loop end" %}
13489   size(2);
13490   ins_encode %{
13491     Label* L = $labl$$label;
13492     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13493   %}
13494   ins_pipe(pipe_jcc);
13495   ins_short_branch(1);
13496 %}
13497 
13498 // Jump Direct Conditional - using unsigned comparison
13499 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13500   match(If cop cmp);
13501   effect(USE labl);
13502 
13503   ins_cost(300);
13504   format %{ "j$cop,us  $labl" %}
13505   size(2);
13506   ins_encode %{
13507     Label* L = $labl$$label;
13508     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13509   %}
13510   ins_pipe(pipe_jcc);
13511   ins_short_branch(1);
13512 %}
13513 
13514 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13515   match(If cop cmp);
13516   effect(USE labl);
13517 
13518   ins_cost(300);
13519   format %{ "j$cop,us  $labl" %}
13520   size(2);
13521   ins_encode %{
13522     Label* L = $labl$$label;
13523     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13524   %}
13525   ins_pipe(pipe_jcc);
13526   ins_short_branch(1);
13527 %}
13528 
13529 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13530   match(If cop cmp);
13531   effect(USE labl);
13532 
13533   ins_cost(300);
13534   format %{ $$template
13535     if ($cop$$cmpcode == Assembler::notEqual) {
13536       $$emit$$"jp,u,s  $labl\n\t"
13537       $$emit$$"j$cop,u,s  $labl"
13538     } else {
13539       $$emit$$"jp,u,s  done\n\t"
13540       $$emit$$"j$cop,u,s  $labl\n\t"
13541       $$emit$$"done:"
13542     }
13543   %}
13544   size(4);
13545   ins_encode %{
13546     Label* l = $labl$$label;
13547     if ($cop$$cmpcode == Assembler::notEqual) {
13548       __ jccb(Assembler::parity, *l);
13549       __ jccb(Assembler::notEqual, *l);
13550     } else if ($cop$$cmpcode == Assembler::equal) {
13551       Label done;
13552       __ jccb(Assembler::parity, done);
13553       __ jccb(Assembler::equal, *l);
13554       __ bind(done);
13555     } else {
13556        ShouldNotReachHere();
13557     }
13558   %}
13559   ins_pipe(pipe_jcc);
13560   ins_short_branch(1);
13561 %}
13562 
13563 // ============================================================================
13564 // inlined locking and unlocking
13565 
13566 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13567   predicate(Compile::current()->use_rtm());
13568   match(Set cr (FastLock object box));
13569   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13570   ins_cost(300);
13571   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13572   ins_encode %{
13573     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13574                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13575                  _rtm_counters, _stack_rtm_counters,
13576                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13577                  true, ra_->C->profile_rtm());
13578   %}
13579   ins_pipe(pipe_slow);
13580 %}
13581 
13582 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13583   predicate(!Compile::current()->use_rtm());
13584   match(Set cr (FastLock object box));
13585   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13586   ins_cost(300);
13587   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13588   ins_encode %{
13589     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13590                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13591   %}
13592   ins_pipe(pipe_slow);
13593 %}
13594 
13595 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13596   match(Set cr (FastUnlock object box));
13597   effect(TEMP tmp, USE_KILL box);
13598   ins_cost(300);
13599   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13600   ins_encode %{
13601     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13602   %}
13603   ins_pipe(pipe_slow);
13604 %}
13605 
13606 
13607 // ============================================================================
13608 // Safepoint Instructions
13609 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13610 %{
13611   match(SafePoint poll);
13612   effect(KILL cr, USE poll);
13613 
13614   format %{ "testl   rax, [$poll]\t"
13615             "# Safepoint: poll for GC" %}
13616   ins_cost(125);
13617   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13618   ins_encode %{
13619     __ relocate(relocInfo::poll_type);
13620     address pre_pc = __ pc();
13621     __ testl(rax, Address($poll$$Register, 0));
13622     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13623   %}
13624   ins_pipe(ialu_reg_mem);
13625 %}
13626 
13627 instruct mask_all_evexL(kReg dst, rRegL src) %{
13628   match(Set dst (MaskAll src));
13629   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13630   ins_encode %{
13631     int mask_len = Matcher::vector_length(this);
13632     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13633   %}
13634   ins_pipe( pipe_slow );
13635 %}
13636 
13637 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13638   predicate(Matcher::vector_length(n) > 32);
13639   match(Set dst (MaskAll src));
13640   effect(TEMP tmp);
13641   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13642   ins_encode %{
13643     int mask_len = Matcher::vector_length(this);
13644     __ movslq($tmp$$Register, $src$$Register);
13645     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13646   %}
13647   ins_pipe( pipe_slow );
13648 %}
13649 
13650 // ============================================================================
13651 // Procedure Call/Return Instructions
13652 // Call Java Static Instruction
13653 // Note: If this code changes, the corresponding ret_addr_offset() and
13654 //       compute_padding() functions will have to be adjusted.
13655 instruct CallStaticJavaDirect(method meth) %{
13656   match(CallStaticJava);
13657   effect(USE meth);
13658 
13659   ins_cost(300);
13660   format %{ "call,static " %}
13661   opcode(0xE8); /* E8 cd */
13662   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13663   ins_pipe(pipe_slow);
13664   ins_alignment(4);
13665 %}
13666 
13667 // Call Java Dynamic Instruction
13668 // Note: If this code changes, the corresponding ret_addr_offset() and
13669 //       compute_padding() functions will have to be adjusted.
13670 instruct CallDynamicJavaDirect(method meth)
13671 %{
13672   match(CallDynamicJava);
13673   effect(USE meth);
13674 
13675   ins_cost(300);
13676   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13677             "call,dynamic " %}
13678   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13679   ins_pipe(pipe_slow);
13680   ins_alignment(4);
13681 %}
13682 
13683 // Call Runtime Instruction
13684 instruct CallRuntimeDirect(method meth)
13685 %{
13686   match(CallRuntime);
13687   effect(USE meth);
13688 
13689   ins_cost(300);
13690   format %{ "call,runtime " %}
13691   ins_encode(clear_avx, Java_To_Runtime(meth));
13692   ins_pipe(pipe_slow);
13693 %}
13694 
13695 // Call runtime without safepoint
13696 instruct CallLeafDirect(method meth)
13697 %{
13698   match(CallLeaf);
13699   effect(USE meth);
13700 
13701   ins_cost(300);
13702   format %{ "call_leaf,runtime " %}
13703   ins_encode(clear_avx, Java_To_Runtime(meth));
13704   ins_pipe(pipe_slow);
13705 %}
13706 
13707 // Call runtime without safepoint and with vector arguments
13708 instruct CallLeafDirectVector(method meth)
13709 %{
13710   match(CallLeafVector);
13711   effect(USE meth);
13712 
13713   ins_cost(300);
13714   format %{ "call_leaf,vector " %}
13715   ins_encode(Java_To_Runtime(meth));
13716   ins_pipe(pipe_slow);
13717 %}
13718 
13719 // Call runtime without safepoint
13720 instruct CallLeafNoFPDirect(method meth)
13721 %{
13722   match(CallLeafNoFP);
13723   effect(USE meth);
13724 
13725   ins_cost(300);
13726   format %{ "call_leaf_nofp,runtime " %}
13727   ins_encode(clear_avx, Java_To_Runtime(meth));
13728   ins_pipe(pipe_slow);
13729 %}
13730 
13731 // Return Instruction
13732 // Remove the return address & jump to it.
13733 // Notice: We always emit a nop after a ret to make sure there is room
13734 // for safepoint patching
13735 instruct Ret()
13736 %{
13737   match(Return);
13738 
13739   format %{ "ret" %}
13740   ins_encode %{
13741     __ ret(0);
13742   %}
13743   ins_pipe(pipe_jmp);
13744 %}
13745 
13746 // Tail Call; Jump from runtime stub to Java code.
13747 // Also known as an 'interprocedural jump'.
13748 // Target of jump will eventually return to caller.
13749 // TailJump below removes the return address.
13750 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13751 %{
13752   match(TailCall jump_target method_ptr);
13753 
13754   ins_cost(300);
13755   format %{ "jmp     $jump_target\t# rbx holds method" %}
13756   ins_encode %{
13757     __ jmp($jump_target$$Register);
13758   %}
13759   ins_pipe(pipe_jmp);
13760 %}
13761 
13762 // Tail Jump; remove the return address; jump to target.
13763 // TailCall above leaves the return address around.
13764 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13765 %{
13766   match(TailJump jump_target ex_oop);
13767 
13768   ins_cost(300);
13769   format %{ "popq    rdx\t# pop return address\n\t"
13770             "jmp     $jump_target" %}
13771   ins_encode %{
13772     __ popq(as_Register(RDX_enc));
13773     __ jmp($jump_target$$Register);
13774   %}
13775   ins_pipe(pipe_jmp);
13776 %}
13777 
13778 // Create exception oop: created by stack-crawling runtime code.
13779 // Created exception is now available to this handler, and is setup
13780 // just prior to jumping to this handler.  No code emitted.
13781 instruct CreateException(rax_RegP ex_oop)
13782 %{
13783   match(Set ex_oop (CreateEx));
13784 
13785   size(0);
13786   // use the following format syntax
13787   format %{ "# exception oop is in rax; no code emitted" %}
13788   ins_encode();
13789   ins_pipe(empty);
13790 %}
13791 
13792 // Rethrow exception:
13793 // The exception oop will come in the first argument position.
13794 // Then JUMP (not call) to the rethrow stub code.
13795 instruct RethrowException()
13796 %{
13797   match(Rethrow);
13798 
13799   // use the following format syntax
13800   format %{ "jmp     rethrow_stub" %}
13801   ins_encode(enc_rethrow);
13802   ins_pipe(pipe_jmp);
13803 %}
13804 
13805 // ============================================================================
13806 // This name is KNOWN by the ADLC and cannot be changed.
13807 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13808 // for this guy.
13809 instruct tlsLoadP(r15_RegP dst) %{
13810   match(Set dst (ThreadLocal));
13811   effect(DEF dst);
13812 
13813   size(0);
13814   format %{ "# TLS is in R15" %}
13815   ins_encode( /*empty encoding*/ );
13816   ins_pipe(ialu_reg_reg);
13817 %}
13818 
13819 instruct rdtsc(rax_RegL dst, rdx_RegL tmp) %{
13820   match(Set dst (Timestamp));
13821   effect(DEF dst, KILL tmp);
13822 
13823   format %{ "RDTSC" %}
13824   ins_encode %{
13825     __ rdtsc();
13826     __ shlptr(rdx, 32);
13827     __ orptr(rax, rdx);
13828   %}
13829   ins_pipe(ialu_reg_reg);
13830 %}
13831 
13832 instruct rdtscp(rax_RegL dst, rdx_RegL tmp, rcx_RegL dead) %{
13833   match(Set dst (TimestampSerial));
13834   effect(DEF dst, KILL tmp, KILL dead);
13835 
13836   format %{ "RDTSCP" %}
13837   ins_encode %{
13838     __ rdtscp();
13839     __ shlptr(rdx, 32);
13840     __ orptr(rax, rdx);
13841   %}
13842   ins_pipe(ialu_reg_reg);
13843 %}
13844 
13845 
13846 //----------PEEPHOLE RULES-----------------------------------------------------
13847 // These must follow all instruction definitions as they use the names
13848 // defined in the instructions definitions.
13849 //
13850 // peepmatch ( root_instr_name [preceding_instruction]* );
13851 //
13852 // peepconstraint %{
13853 // (instruction_number.operand_name relational_op instruction_number.operand_name
13854 //  [, ...] );
13855 // // instruction numbers are zero-based using left to right order in peepmatch
13856 //
13857 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13858 // // provide an instruction_number.operand_name for each operand that appears
13859 // // in the replacement instruction's match rule
13860 //
13861 // ---------VM FLAGS---------------------------------------------------------
13862 //
13863 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13864 //
13865 // Each peephole rule is given an identifying number starting with zero and
13866 // increasing by one in the order seen by the parser.  An individual peephole
13867 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13868 // on the command-line.
13869 //
13870 // ---------CURRENT LIMITATIONS----------------------------------------------
13871 //
13872 // Only match adjacent instructions in same basic block
13873 // Only equality constraints
13874 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13875 // Only one replacement instruction
13876 //
13877 // ---------EXAMPLE----------------------------------------------------------
13878 //
13879 // // pertinent parts of existing instructions in architecture description
13880 // instruct movI(rRegI dst, rRegI src)
13881 // %{
13882 //   match(Set dst (CopyI src));
13883 // %}
13884 //
13885 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13886 // %{
13887 //   match(Set dst (AddI dst src));
13888 //   effect(KILL cr);
13889 // %}
13890 //
13891 // // Change (inc mov) to lea
13892 // peephole %{
13893 //   // increment preceded by register-register move
13894 //   peepmatch ( incI_rReg movI );
13895 //   // require that the destination register of the increment
13896 //   // match the destination register of the move
13897 //   peepconstraint ( 0.dst == 1.dst );
13898 //   // construct a replacement instruction that sets
13899 //   // the destination to ( move's source register + one )
13900 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13901 // %}
13902 //
13903 
13904 // Implementation no longer uses movX instructions since
13905 // machine-independent system no longer uses CopyX nodes.
13906 //
13907 // peephole
13908 // %{
13909 //   peepmatch (incI_rReg movI);
13910 //   peepconstraint (0.dst == 1.dst);
13911 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13912 // %}
13913 
13914 // peephole
13915 // %{
13916 //   peepmatch (decI_rReg movI);
13917 //   peepconstraint (0.dst == 1.dst);
13918 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13919 // %}
13920 
13921 // peephole
13922 // %{
13923 //   peepmatch (addI_rReg_imm movI);
13924 //   peepconstraint (0.dst == 1.dst);
13925 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13926 // %}
13927 
13928 // peephole
13929 // %{
13930 //   peepmatch (incL_rReg movL);
13931 //   peepconstraint (0.dst == 1.dst);
13932 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13933 // %}
13934 
13935 // peephole
13936 // %{
13937 //   peepmatch (decL_rReg movL);
13938 //   peepconstraint (0.dst == 1.dst);
13939 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13940 // %}
13941 
13942 // peephole
13943 // %{
13944 //   peepmatch (addL_rReg_imm movL);
13945 //   peepconstraint (0.dst == 1.dst);
13946 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13947 // %}
13948 
13949 // peephole
13950 // %{
13951 //   peepmatch (addP_rReg_imm movP);
13952 //   peepconstraint (0.dst == 1.dst);
13953 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13954 // %}
13955 
13956 // // Change load of spilled value to only a spill
13957 // instruct storeI(memory mem, rRegI src)
13958 // %{
13959 //   match(Set mem (StoreI mem src));
13960 // %}
13961 //
13962 // instruct loadI(rRegI dst, memory mem)
13963 // %{
13964 //   match(Set dst (LoadI mem));
13965 // %}
13966 //
13967 
13968 peephole
13969 %{
13970   peepmatch (loadI storeI);
13971   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13972   peepreplace (storeI(1.mem 1.mem 1.src));
13973 %}
13974 
13975 peephole
13976 %{
13977   peepmatch (loadL storeL);
13978   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13979   peepreplace (storeL(1.mem 1.mem 1.src));
13980 %}
13981 
13982 //----------SMARTSPILL RULES---------------------------------------------------
13983 // These must follow all instruction definitions as they use the names
13984 // defined in the instructions definitions.