1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 }
  458 
  459 static bool generate_vzeroupper(Compile* C) {
  460   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  461 }
  462 
  463 static int clear_avx_size() {
  464   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  465 }
  466 
  467 // !!!!! Special hack to get all types of calls to specify the byte offset
  468 //       from the start of the call to the point where the return address
  469 //       will point.
  470 int MachCallStaticJavaNode::ret_addr_offset()
  471 {
  472   int offset = 5; // 5 bytes from start of call to where return address points
  473   offset += clear_avx_size();
  474   return offset;
  475 }
  476 
  477 int MachCallDynamicJavaNode::ret_addr_offset()
  478 {
  479   int offset = 15; // 15 bytes from start of call to where return address points
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 
  484 int MachCallRuntimeNode::ret_addr_offset() {
  485   int offset = 13; // movq r10,#addr; callq (r10)
  486   if (this->ideal_Opcode() != Op_CallLeafVector) {
  487     offset += clear_avx_size();
  488   }
  489   return offset;
  490 }
  491 //
  492 // Compute padding required for nodes which need alignment
  493 //
  494 
  495 // The address of the call instruction needs to be 4-byte aligned to
  496 // ensure that it does not span a cache line so that it can be patched.
  497 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  498 {
  499   current_offset += clear_avx_size(); // skip vzeroupper
  500   current_offset += 1; // skip call opcode byte
  501   return align_up(current_offset, alignment_required()) - current_offset;
  502 }
  503 
  504 // The address of the call instruction needs to be 4-byte aligned to
  505 // ensure that it does not span a cache line so that it can be patched.
  506 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  507 {
  508   current_offset += clear_avx_size(); // skip vzeroupper
  509   current_offset += 11; // skip movq instruction + call opcode byte
  510   return align_up(current_offset, alignment_required()) - current_offset;
  511 }
  512 
  513 // EMIT_RM()
  514 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  515   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  516   cbuf.insts()->emit_int8(c);
  517 }
  518 
  519 // EMIT_CC()
  520 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  521   unsigned char c = (unsigned char) (f1 | f2);
  522   cbuf.insts()->emit_int8(c);
  523 }
  524 
  525 // EMIT_OPCODE()
  526 void emit_opcode(CodeBuffer &cbuf, int code) {
  527   cbuf.insts()->emit_int8((unsigned char) code);
  528 }
  529 
  530 // EMIT_OPCODE() w/ relocation information
  531 void emit_opcode(CodeBuffer &cbuf,
  532                  int code, relocInfo::relocType reloc, int offset, int format)
  533 {
  534   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  535   emit_opcode(cbuf, code);
  536 }
  537 
  538 // EMIT_D8()
  539 void emit_d8(CodeBuffer &cbuf, int d8) {
  540   cbuf.insts()->emit_int8((unsigned char) d8);
  541 }
  542 
  543 // EMIT_D16()
  544 void emit_d16(CodeBuffer &cbuf, int d16) {
  545   cbuf.insts()->emit_int16(d16);
  546 }
  547 
  548 // EMIT_D32()
  549 void emit_d32(CodeBuffer &cbuf, int d32) {
  550   cbuf.insts()->emit_int32(d32);
  551 }
  552 
  553 // EMIT_D64()
  554 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  555   cbuf.insts()->emit_int64(d64);
  556 }
  557 
  558 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  559 void emit_d32_reloc(CodeBuffer& cbuf,
  560                     int d32,
  561                     relocInfo::relocType reloc,
  562                     int format)
  563 {
  564   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  565   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  566   cbuf.insts()->emit_int32(d32);
  567 }
  568 
  569 // emit 32 bit value and construct relocation entry from RelocationHolder
  570 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  571 #ifdef ASSERT
  572   if (rspec.reloc()->type() == relocInfo::oop_type &&
  573       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  574     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  575     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  576   }
  577 #endif
  578   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  579   cbuf.insts()->emit_int32(d32);
  580 }
  581 
  582 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  583   address next_ip = cbuf.insts_end() + 4;
  584   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  585                  external_word_Relocation::spec(addr),
  586                  RELOC_DISP32);
  587 }
  588 
  589 
  590 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  591 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  592   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  593   cbuf.insts()->emit_int64(d64);
  594 }
  595 
  596 // emit 64 bit value and construct relocation entry from RelocationHolder
  597 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  598 #ifdef ASSERT
  599   if (rspec.reloc()->type() == relocInfo::oop_type &&
  600       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  601     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  602     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  603   }
  604 #endif
  605   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  606   cbuf.insts()->emit_int64(d64);
  607 }
  608 
  609 // Access stack slot for load or store
  610 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  611 {
  612   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  613   if (-0x80 <= disp && disp < 0x80) {
  614     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  615     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  616     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  617   } else {
  618     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  619     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  620     emit_d32(cbuf, disp);     // Displacement // R/M byte
  621   }
  622 }
  623 
  624    // rRegI ereg, memory mem) %{    // emit_reg_mem
  625 void encode_RegMem(CodeBuffer &cbuf,
  626                    int reg,
  627                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  628 {
  629   assert(disp_reloc == relocInfo::none, "cannot have disp");
  630   int regenc = reg & 7;
  631   int baseenc = base & 7;
  632   int indexenc = index & 7;
  633 
  634   // There is no index & no scale, use form without SIB byte
  635   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  636     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  637     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  638       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  639     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  640       // If 8-bit displacement, mode 0x1
  641       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  642       emit_d8(cbuf, disp);
  643     } else {
  644       // If 32-bit displacement
  645       if (base == -1) { // Special flag for absolute address
  646         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       } else {
  653         // Normal base + offset
  654         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       }
  661     }
  662   } else {
  663     // Else, encode with the SIB byte
  664     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  665     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  666       // If no displacement
  667       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  668       emit_rm(cbuf, scale, indexenc, baseenc);
  669     } else {
  670       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  671         // If 8-bit displacement, mode 0x1
  672         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  673         emit_rm(cbuf, scale, indexenc, baseenc);
  674         emit_d8(cbuf, disp);
  675       } else {
  676         // If 32-bit displacement
  677         if (base == 0x04 ) {
  678           emit_rm(cbuf, 0x2, regenc, 0x4);
  679           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  680         } else {
  681           emit_rm(cbuf, 0x2, regenc, 0x4);
  682           emit_rm(cbuf, scale, indexenc, baseenc); // *
  683         }
  684         if (disp_reloc != relocInfo::none) {
  685           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  686         } else {
  687           emit_d32(cbuf, disp);
  688         }
  689       }
  690     }
  691   }
  692 }
  693 
  694 // This could be in MacroAssembler but it's fairly C2 specific
  695 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  696   Label exit;
  697   __ jccb(Assembler::noParity, exit);
  698   __ pushf();
  699   //
  700   // comiss/ucomiss instructions set ZF,PF,CF flags and
  701   // zero OF,AF,SF for NaN values.
  702   // Fixup flags by zeroing ZF,PF so that compare of NaN
  703   // values returns 'less than' result (CF is set).
  704   // Leave the rest of flags unchanged.
  705   //
  706   //    7 6 5 4 3 2 1 0
  707   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  708   //    0 0 1 0 1 0 1 1   (0x2B)
  709   //
  710   __ andq(Address(rsp, 0), 0xffffff2b);
  711   __ popf();
  712   __ bind(exit);
  713 }
  714 
  715 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  716   Label done;
  717   __ movl(dst, -1);
  718   __ jcc(Assembler::parity, done);
  719   __ jcc(Assembler::below, done);
  720   __ setb(Assembler::notEqual, dst);
  721   __ movzbl(dst, dst);
  722   __ bind(done);
  723 }
  724 
  725 // Math.min()    # Math.max()
  726 // --------------------------
  727 // ucomis[s/d]   #
  728 // ja   -> b     # a
  729 // jp   -> NaN   # NaN
  730 // jb   -> a     # b
  731 // je            #
  732 // |-jz -> a | b # a & b
  733 // |    -> a     #
  734 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  735                      XMMRegister a, XMMRegister b,
  736                      XMMRegister xmmt, Register rt,
  737                      bool min, bool single) {
  738 
  739   Label nan, zero, below, above, done;
  740 
  741   if (single)
  742     __ ucomiss(a, b);
  743   else
  744     __ ucomisd(a, b);
  745 
  746   if (dst->encoding() != (min ? b : a)->encoding())
  747     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  748   else
  749     __ jccb(Assembler::above, done);
  750 
  751   __ jccb(Assembler::parity, nan);  // PF=1
  752   __ jccb(Assembler::below, below); // CF=1
  753 
  754   // equal
  755   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  756   if (single) {
  757     __ ucomiss(a, xmmt);
  758     __ jccb(Assembler::equal, zero);
  759 
  760     __ movflt(dst, a);
  761     __ jmp(done);
  762   }
  763   else {
  764     __ ucomisd(a, xmmt);
  765     __ jccb(Assembler::equal, zero);
  766 
  767     __ movdbl(dst, a);
  768     __ jmp(done);
  769   }
  770 
  771   __ bind(zero);
  772   if (min)
  773     __ vpor(dst, a, b, Assembler::AVX_128bit);
  774   else
  775     __ vpand(dst, a, b, Assembler::AVX_128bit);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(above);
  780   if (single)
  781     __ movflt(dst, min ? b : a);
  782   else
  783     __ movdbl(dst, min ? b : a);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(nan);
  788   if (single) {
  789     __ movl(rt, 0x7fc00000); // Float.NaN
  790     __ movdl(dst, rt);
  791   }
  792   else {
  793     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  794     __ movdq(dst, rt);
  795   }
  796   __ jmp(done);
  797 
  798   __ bind(below);
  799   if (single)
  800     __ movflt(dst, min ? a : b);
  801   else
  802     __ movdbl(dst, min ? a : b);
  803 
  804   __ bind(done);
  805 }
  806 
  807 //=============================================================================
  808 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  809 
  810 int ConstantTable::calculate_table_base_offset() const {
  811   return 0;  // absolute addressing, no offset
  812 }
  813 
  814 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  815 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  816   ShouldNotReachHere();
  817 }
  818 
  819 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  820   // Empty encoding
  821 }
  822 
  823 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  824   return 0;
  825 }
  826 
  827 #ifndef PRODUCT
  828 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   st->print("# MachConstantBaseNode (empty encoding)");
  830 }
  831 #endif
  832 
  833 
  834 //=============================================================================
  835 #ifndef PRODUCT
  836 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   Compile* C = ra_->C;
  838 
  839   int framesize = C->output()->frame_size_in_bytes();
  840   int bangsize = C->output()->bang_size_in_bytes();
  841   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  842   // Remove wordSize for return addr which is already pushed.
  843   framesize -= wordSize;
  844 
  845   if (C->output()->need_stack_bang(bangsize)) {
  846     framesize -= wordSize;
  847     st->print("# stack bang (%d bytes)", bangsize);
  848     st->print("\n\t");
  849     st->print("pushq   rbp\t# Save rbp");
  850     if (PreserveFramePointer) {
  851         st->print("\n\t");
  852         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  853     }
  854     if (framesize) {
  855       st->print("\n\t");
  856       st->print("subq    rsp, #%d\t# Create frame",framesize);
  857     }
  858   } else {
  859     st->print("subq    rsp, #%d\t# Create frame",framesize);
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  863     if (PreserveFramePointer) {
  864       st->print("\n\t");
  865       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  866       if (framesize > 0) {
  867         st->print("\n\t");
  868         st->print("addq    rbp, #%d", framesize);
  869       }
  870     }
  871   }
  872 
  873   if (VerifyStackAtCalls) {
  874     st->print("\n\t");
  875     framesize -= wordSize;
  876     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  877 #ifdef ASSERT
  878     st->print("\n\t");
  879     st->print("# stack alignment check");
  880 #endif
  881   }
  882   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  883     st->print("\n\t");
  884     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  885     st->print("\n\t");
  886     st->print("je      fast_entry\t");
  887     st->print("\n\t");
  888     st->print("call    #nmethod_entry_barrier_stub\t");
  889     st->print("\n\tfast_entry:");
  890   }
  891   st->cr();
  892 }
  893 #endif
  894 
  895 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  896   Compile* C = ra_->C;
  897   C2_MacroAssembler _masm(&cbuf);
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   int bangsize = C->output()->bang_size_in_bytes();
  901 
  902   if (C->clinit_barrier_on_entry()) {
  903     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  904     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  905 
  906     Label L_skip_barrier;
  907     Register klass = rscratch1;
  908 
  909     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  910     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  911 
  912     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  913 
  914     __ bind(L_skip_barrier);
  915   }
  916 
  917   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  918 
  919   C->output()->set_frame_complete(cbuf.insts_size());
  920 
  921   if (C->has_mach_constant_base_node()) {
  922     // NOTE: We set the table base offset here because users might be
  923     // emitted before MachConstantBaseNode.
  924     ConstantTable& constant_table = C->output()->constant_table();
  925     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  926   }
  927 }
  928 
  929 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  930 {
  931   return MachNode::size(ra_); // too many variables; just compute it
  932                               // the hard way
  933 }
  934 
  935 int MachPrologNode::reloc() const
  936 {
  937   return 0; // a large enough number
  938 }
  939 
  940 //=============================================================================
  941 #ifndef PRODUCT
  942 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  943 {
  944   Compile* C = ra_->C;
  945   if (generate_vzeroupper(C)) {
  946     st->print("vzeroupper");
  947     st->cr(); st->print("\t");
  948   }
  949 
  950   int framesize = C->output()->frame_size_in_bytes();
  951   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  952   // Remove word for return adr already pushed
  953   // and RBP
  954   framesize -= 2*wordSize;
  955 
  956   if (framesize) {
  957     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  958     st->print("\t");
  959   }
  960 
  961   st->print_cr("popq    rbp");
  962   if (do_polling() && C->is_method_compilation()) {
  963     st->print("\t");
  964     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  965                  "ja       #safepoint_stub\t"
  966                  "# Safepoint: poll for GC");
  967   }
  968 }
  969 #endif
  970 
  971 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  972 {
  973   Compile* C = ra_->C;
  974   MacroAssembler _masm(&cbuf);
  975 
  976   if (generate_vzeroupper(C)) {
  977     // Clear upper bits of YMM registers when current compiled code uses
  978     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  979     __ vzeroupper();
  980   }
  981 
  982   int framesize = C->output()->frame_size_in_bytes();
  983   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  984   // Remove word for return adr already pushed
  985   // and RBP
  986   framesize -= 2*wordSize;
  987 
  988   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  989 
  990   if (framesize) {
  991     emit_opcode(cbuf, Assembler::REX_W);
  992     if (framesize < 0x80) {
  993       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  995       emit_d8(cbuf, framesize);
  996     } else {
  997       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  998       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  999       emit_d32(cbuf, framesize);
 1000     }
 1001   }
 1002 
 1003   // popq rbp
 1004   emit_opcode(cbuf, 0x58 | RBP_enc);
 1005 
 1006   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1007     __ reserved_stack_check();
 1008   }
 1009 
 1010   if (do_polling() && C->is_method_compilation()) {
 1011     MacroAssembler _masm(&cbuf);
 1012     Label dummy_label;
 1013     Label* code_stub = &dummy_label;
 1014     if (!C->output()->in_scratch_emit_size()) {
 1015       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1016     }
 1017     __ relocate(relocInfo::poll_return_type);
 1018     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1019   }
 1020 }
 1021 
 1022 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1023 {
 1024   return MachNode::size(ra_); // too many variables; just compute it
 1025                               // the hard way
 1026 }
 1027 
 1028 int MachEpilogNode::reloc() const
 1029 {
 1030   return 2; // a large enough number
 1031 }
 1032 
 1033 const Pipeline* MachEpilogNode::pipeline() const
 1034 {
 1035   return MachNode::pipeline_class();
 1036 }
 1037 
 1038 //=============================================================================
 1039 
 1040 enum RC {
 1041   rc_bad,
 1042   rc_int,
 1043   rc_kreg,
 1044   rc_float,
 1045   rc_stack
 1046 };
 1047 
 1048 static enum RC rc_class(OptoReg::Name reg)
 1049 {
 1050   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1051 
 1052   if (OptoReg::is_stack(reg)) return rc_stack;
 1053 
 1054   VMReg r = OptoReg::as_VMReg(reg);
 1055 
 1056   if (r->is_Register()) return rc_int;
 1057 
 1058   if (r->is_KRegister()) return rc_kreg;
 1059 
 1060   assert(r->is_XMMRegister(), "must be");
 1061   return rc_float;
 1062 }
 1063 
 1064 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1065 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1066                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1067 
 1068 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1069                      int stack_offset, int reg, uint ireg, outputStream* st);
 1070 
 1071 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1072                                       int dst_offset, uint ireg, outputStream* st) {
 1073   if (cbuf) {
 1074     MacroAssembler _masm(cbuf);
 1075     switch (ireg) {
 1076     case Op_VecS:
 1077       __ movq(Address(rsp, -8), rax);
 1078       __ movl(rax, Address(rsp, src_offset));
 1079       __ movl(Address(rsp, dst_offset), rax);
 1080       __ movq(rax, Address(rsp, -8));
 1081       break;
 1082     case Op_VecD:
 1083       __ pushq(Address(rsp, src_offset));
 1084       __ popq (Address(rsp, dst_offset));
 1085       break;
 1086     case Op_VecX:
 1087       __ pushq(Address(rsp, src_offset));
 1088       __ popq (Address(rsp, dst_offset));
 1089       __ pushq(Address(rsp, src_offset+8));
 1090       __ popq (Address(rsp, dst_offset+8));
 1091       break;
 1092     case Op_VecY:
 1093       __ vmovdqu(Address(rsp, -32), xmm0);
 1094       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1095       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1096       __ vmovdqu(xmm0, Address(rsp, -32));
 1097       break;
 1098     case Op_VecZ:
 1099       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1100       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1101       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1102       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1103       break;
 1104     default:
 1105       ShouldNotReachHere();
 1106     }
 1107 #ifndef PRODUCT
 1108   } else {
 1109     switch (ireg) {
 1110     case Op_VecS:
 1111       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1112                 "movl    rax, [rsp + #%d]\n\t"
 1113                 "movl    [rsp + #%d], rax\n\t"
 1114                 "movq    rax, [rsp - #8]",
 1115                 src_offset, dst_offset);
 1116       break;
 1117     case Op_VecD:
 1118       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1119                 "popq    [rsp + #%d]",
 1120                 src_offset, dst_offset);
 1121       break;
 1122      case Op_VecX:
 1123       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1124                 "popq    [rsp + #%d]\n\t"
 1125                 "pushq   [rsp + #%d]\n\t"
 1126                 "popq    [rsp + #%d]",
 1127                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1128       break;
 1129     case Op_VecY:
 1130       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1133                 "vmovdqu xmm0, [rsp - #32]",
 1134                 src_offset, dst_offset);
 1135       break;
 1136     case Op_VecZ:
 1137       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1138                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1139                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1140                 "vmovdqu xmm0, [rsp - #64]",
 1141                 src_offset, dst_offset);
 1142       break;
 1143     default:
 1144       ShouldNotReachHere();
 1145     }
 1146 #endif
 1147   }
 1148 }
 1149 
 1150 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1151                                        PhaseRegAlloc* ra_,
 1152                                        bool do_size,
 1153                                        outputStream* st) const {
 1154   assert(cbuf != NULL || st  != NULL, "sanity");
 1155   // Get registers to move
 1156   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1157   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1158   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1159   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1160 
 1161   enum RC src_second_rc = rc_class(src_second);
 1162   enum RC src_first_rc = rc_class(src_first);
 1163   enum RC dst_second_rc = rc_class(dst_second);
 1164   enum RC dst_first_rc = rc_class(dst_first);
 1165 
 1166   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1167          "must move at least 1 register" );
 1168 
 1169   if (src_first == dst_first && src_second == dst_second) {
 1170     // Self copy, no move
 1171     return 0;
 1172   }
 1173   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1174     uint ireg = ideal_reg();
 1175     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1176     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1177     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1178       // mem -> mem
 1179       int src_offset = ra_->reg2offset(src_first);
 1180       int dst_offset = ra_->reg2offset(dst_first);
 1181       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1182     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1183       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1184     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1185       int stack_offset = ra_->reg2offset(dst_first);
 1186       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1187     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1188       int stack_offset = ra_->reg2offset(src_first);
 1189       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1190     } else {
 1191       ShouldNotReachHere();
 1192     }
 1193     return 0;
 1194   }
 1195   if (src_first_rc == rc_stack) {
 1196     // mem ->
 1197     if (dst_first_rc == rc_stack) {
 1198       // mem -> mem
 1199       assert(src_second != dst_first, "overlap");
 1200       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1201           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1202         // 64-bit
 1203         int src_offset = ra_->reg2offset(src_first);
 1204         int dst_offset = ra_->reg2offset(dst_first);
 1205         if (cbuf) {
 1206           MacroAssembler _masm(cbuf);
 1207           __ pushq(Address(rsp, src_offset));
 1208           __ popq (Address(rsp, dst_offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1212                     "popq    [rsp + #%d]",
 1213                      src_offset, dst_offset);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         // No pushl/popl, so:
 1221         int src_offset = ra_->reg2offset(src_first);
 1222         int dst_offset = ra_->reg2offset(dst_first);
 1223         if (cbuf) {
 1224           MacroAssembler _masm(cbuf);
 1225           __ movq(Address(rsp, -8), rax);
 1226           __ movl(rax, Address(rsp, src_offset));
 1227           __ movl(Address(rsp, dst_offset), rax);
 1228           __ movq(rax, Address(rsp, -8));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1232                     "movl    rax, [rsp + #%d]\n\t"
 1233                     "movl    [rsp + #%d], rax\n\t"
 1234                     "movq    rax, [rsp - #8]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_int) {
 1241       // mem -> gpr
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (cbuf) {
 1247           MacroAssembler _masm(cbuf);
 1248           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1249 #ifndef PRODUCT
 1250         } else {
 1251           st->print("movq    %s, [rsp + #%d]\t# spill",
 1252                      Matcher::regName[dst_first],
 1253                      offset);
 1254 #endif
 1255         }
 1256       } else {
 1257         // 32-bit
 1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1260         int offset = ra_->reg2offset(src_first);
 1261         if (cbuf) {
 1262           MacroAssembler _masm(cbuf);
 1263           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1264 #ifndef PRODUCT
 1265         } else {
 1266           st->print("movl    %s, [rsp + #%d]\t# spill",
 1267                      Matcher::regName[dst_first],
 1268                      offset);
 1269 #endif
 1270         }
 1271       }
 1272       return 0;
 1273     } else if (dst_first_rc == rc_float) {
 1274       // mem-> xmm
 1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1277         // 64-bit
 1278         int offset = ra_->reg2offset(src_first);
 1279         if (cbuf) {
 1280           MacroAssembler _masm(cbuf);
 1281           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("%s  %s, [rsp + #%d]\t# spill",
 1285                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1286                      Matcher::regName[dst_first],
 1287                      offset);
 1288 #endif
 1289         }
 1290       } else {
 1291         // 32-bit
 1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1294         int offset = ra_->reg2offset(src_first);
 1295         if (cbuf) {
 1296           MacroAssembler _masm(cbuf);
 1297           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movss   %s, [rsp + #%d]\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      offset);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_kreg) {
 1308       // mem -> kreg
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         int offset = ra_->reg2offset(src_first);
 1313         if (cbuf) {
 1314           MacroAssembler _masm(cbuf);
 1315           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1316 #ifndef PRODUCT
 1317         } else {
 1318           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1319                      Matcher::regName[dst_first],
 1320                      offset);
 1321 #endif
 1322         }
 1323       }
 1324       return 0;
 1325     }
 1326   } else if (src_first_rc == rc_int) {
 1327     // gpr ->
 1328     if (dst_first_rc == rc_stack) {
 1329       // gpr -> mem
 1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1332         // 64-bit
 1333         int offset = ra_->reg2offset(dst_first);
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("movq    [rsp + #%d], %s\t# spill",
 1340                      offset,
 1341                      Matcher::regName[src_first]);
 1342 #endif
 1343         }
 1344       } else {
 1345         // 32-bit
 1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("movl    [rsp + #%d], %s\t# spill",
 1355                      offset,
 1356                      Matcher::regName[src_first]);
 1357 #endif
 1358         }
 1359       }
 1360       return 0;
 1361     } else if (dst_first_rc == rc_int) {
 1362       // gpr -> gpr
 1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1365         // 64-bit
 1366         if (cbuf) {
 1367           MacroAssembler _masm(cbuf);
 1368           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1369                   as_Register(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movq    %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377         return 0;
 1378       } else {
 1379         // 32-bit
 1380         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1381         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1382         if (cbuf) {
 1383           MacroAssembler _masm(cbuf);
 1384           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1385                   as_Register(Matcher::_regEncode[src_first]));
 1386 #ifndef PRODUCT
 1387         } else {
 1388           st->print("movl    %s, %s\t# spill",
 1389                      Matcher::regName[dst_first],
 1390                      Matcher::regName[src_first]);
 1391 #endif
 1392         }
 1393         return 0;
 1394       }
 1395     } else if (dst_first_rc == rc_float) {
 1396       // gpr -> xmm
 1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1399         // 64-bit
 1400         if (cbuf) {
 1401           MacroAssembler _masm(cbuf);
 1402           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1403 #ifndef PRODUCT
 1404         } else {
 1405           st->print("movdq   %s, %s\t# spill",
 1406                      Matcher::regName[dst_first],
 1407                      Matcher::regName[src_first]);
 1408 #endif
 1409         }
 1410       } else {
 1411         // 32-bit
 1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1414         if (cbuf) {
 1415           MacroAssembler _masm(cbuf);
 1416           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1417 #ifndef PRODUCT
 1418         } else {
 1419           st->print("movdl   %s, %s\t# spill",
 1420                      Matcher::regName[dst_first],
 1421                      Matcher::regName[src_first]);
 1422 #endif
 1423         }
 1424       }
 1425       return 0;
 1426     } else if (dst_first_rc == rc_kreg) {
 1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1429         // 64-bit
 1430         if (cbuf) {
 1431           MacroAssembler _masm(cbuf);
 1432           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1433   #ifndef PRODUCT
 1434         } else {
 1435            st->print("kmovq   %s, %s\t# spill",
 1436                        Matcher::regName[dst_first],
 1437                        Matcher::regName[src_first]);
 1438   #endif
 1439         }
 1440       }
 1441       Unimplemented();
 1442       return 0;
 1443     }
 1444   } else if (src_first_rc == rc_float) {
 1445     // xmm ->
 1446     if (dst_first_rc == rc_stack) {
 1447       // xmm -> mem
 1448       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1449           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1450         // 64-bit
 1451         int offset = ra_->reg2offset(dst_first);
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1455 #ifndef PRODUCT
 1456         } else {
 1457           st->print("movsd   [rsp + #%d], %s\t# spill",
 1458                      offset,
 1459                      Matcher::regName[src_first]);
 1460 #endif
 1461         }
 1462       } else {
 1463         // 32-bit
 1464         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1465         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1466         int offset = ra_->reg2offset(dst_first);
 1467         if (cbuf) {
 1468           MacroAssembler _masm(cbuf);
 1469           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1470 #ifndef PRODUCT
 1471         } else {
 1472           st->print("movss   [rsp + #%d], %s\t# spill",
 1473                      offset,
 1474                      Matcher::regName[src_first]);
 1475 #endif
 1476         }
 1477       }
 1478       return 0;
 1479     } else if (dst_first_rc == rc_int) {
 1480       // xmm -> gpr
 1481       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1482           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1483         // 64-bit
 1484         if (cbuf) {
 1485           MacroAssembler _masm(cbuf);
 1486           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1487 #ifndef PRODUCT
 1488         } else {
 1489           st->print("movdq   %s, %s\t# spill",
 1490                      Matcher::regName[dst_first],
 1491                      Matcher::regName[src_first]);
 1492 #endif
 1493         }
 1494       } else {
 1495         // 32-bit
 1496         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1497         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1498         if (cbuf) {
 1499           MacroAssembler _masm(cbuf);
 1500           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1501 #ifndef PRODUCT
 1502         } else {
 1503           st->print("movdl   %s, %s\t# spill",
 1504                      Matcher::regName[dst_first],
 1505                      Matcher::regName[src_first]);
 1506 #endif
 1507         }
 1508       }
 1509       return 0;
 1510     } else if (dst_first_rc == rc_float) {
 1511       // xmm -> xmm
 1512       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1513           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1514         // 64-bit
 1515         if (cbuf) {
 1516           MacroAssembler _masm(cbuf);
 1517           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1518 #ifndef PRODUCT
 1519         } else {
 1520           st->print("%s  %s, %s\t# spill",
 1521                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1522                      Matcher::regName[dst_first],
 1523                      Matcher::regName[src_first]);
 1524 #endif
 1525         }
 1526       } else {
 1527         // 32-bit
 1528         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1529         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1530         if (cbuf) {
 1531           MacroAssembler _masm(cbuf);
 1532           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1533 #ifndef PRODUCT
 1534         } else {
 1535           st->print("%s  %s, %s\t# spill",
 1536                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1537                      Matcher::regName[dst_first],
 1538                      Matcher::regName[src_first]);
 1539 #endif
 1540         }
 1541       }
 1542       return 0;
 1543     } else if (dst_first_rc == rc_kreg) {
 1544       assert(false, "Illegal spilling");
 1545       return 0;
 1546     }
 1547   } else if (src_first_rc == rc_kreg) {
 1548     if (dst_first_rc == rc_stack) {
 1549       // mem -> kreg
 1550       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1551           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1552         // 64-bit
 1553         int offset = ra_->reg2offset(dst_first);
 1554         if (cbuf) {
 1555           MacroAssembler _masm(cbuf);
 1556           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1557 #ifndef PRODUCT
 1558         } else {
 1559           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1560                      offset,
 1561                      Matcher::regName[src_first]);
 1562 #endif
 1563         }
 1564       }
 1565       return 0;
 1566     } else if (dst_first_rc == rc_int) {
 1567       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1568           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1569         // 64-bit
 1570         if (cbuf) {
 1571           MacroAssembler _masm(cbuf);
 1572           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1573 #ifndef PRODUCT
 1574         } else {
 1575          st->print("kmovq   %s, %s\t# spill",
 1576                      Matcher::regName[dst_first],
 1577                      Matcher::regName[src_first]);
 1578 #endif
 1579         }
 1580       }
 1581       Unimplemented();
 1582       return 0;
 1583     } else if (dst_first_rc == rc_kreg) {
 1584       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1585           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1586         // 64-bit
 1587         if (cbuf) {
 1588           MacroAssembler _masm(cbuf);
 1589           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1590 #ifndef PRODUCT
 1591         } else {
 1592          st->print("kmovq   %s, %s\t# spill",
 1593                      Matcher::regName[dst_first],
 1594                      Matcher::regName[src_first]);
 1595 #endif
 1596         }
 1597       }
 1598       return 0;
 1599     } else if (dst_first_rc == rc_float) {
 1600       assert(false, "Illegal spill");
 1601       return 0;
 1602     }
 1603   }
 1604 
 1605   assert(0," foo ");
 1606   Unimplemented();
 1607   return 0;
 1608 }
 1609 
 1610 #ifndef PRODUCT
 1611 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1612   implementation(NULL, ra_, false, st);
 1613 }
 1614 #endif
 1615 
 1616 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1617   implementation(&cbuf, ra_, false, NULL);
 1618 }
 1619 
 1620 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1621   return MachNode::size(ra_);
 1622 }
 1623 
 1624 //=============================================================================
 1625 #ifndef PRODUCT
 1626 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1627 {
 1628   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1629   int reg = ra_->get_reg_first(this);
 1630   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1631             Matcher::regName[reg], offset);
 1632 }
 1633 #endif
 1634 
 1635 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1636 {
 1637   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1638   int reg = ra_->get_encode(this);
 1639   if (offset >= 0x80) {
 1640     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1641     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1642     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1643     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1644     emit_d32(cbuf, offset);
 1645   } else {
 1646     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1647     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1648     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1649     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1650     emit_d8(cbuf, offset);
 1651   }
 1652 }
 1653 
 1654 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1655 {
 1656   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1657   return (offset < 0x80) ? 5 : 8; // REX
 1658 }
 1659 
 1660 //=============================================================================
 1661 #ifndef PRODUCT
 1662 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1663 {
 1664   if (UseCompressedClassPointers) {
 1665     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1666     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1667     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1668   } else {
 1669     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1670                  "# Inline cache check");
 1671   }
 1672   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1673   st->print_cr("\tnop\t# nops to align entry point");
 1674 }
 1675 #endif
 1676 
 1677 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1678 {
 1679   MacroAssembler masm(&cbuf);
 1680   uint insts_size = cbuf.insts_size();
 1681   if (UseCompressedClassPointers) {
 1682     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1683     masm.cmpptr(rax, rscratch1);
 1684   } else {
 1685     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1686   }
 1687 
 1688   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1689 
 1690   /* WARNING these NOPs are critical so that verified entry point is properly
 1691      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1692   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1693   if (OptoBreakpoint) {
 1694     // Leave space for int3
 1695     nops_cnt -= 1;
 1696   }
 1697   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1698   if (nops_cnt > 0)
 1699     masm.nop(nops_cnt);
 1700 }
 1701 
 1702 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1703 {
 1704   return MachNode::size(ra_); // too many variables; just compute it
 1705                               // the hard way
 1706 }
 1707 
 1708 
 1709 //=============================================================================
 1710 
 1711 const bool Matcher::supports_vector_calling_convention(void) {
 1712   if (EnableVectorSupport && UseVectorStubs) {
 1713     return true;
 1714   }
 1715   return false;
 1716 }
 1717 
 1718 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1719   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1720   int lo = XMM0_num;
 1721   int hi = XMM0b_num;
 1722   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1723   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1724   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1725   return OptoRegPair(hi, lo);
 1726 }
 1727 
 1728 // Is this branch offset short enough that a short branch can be used?
 1729 //
 1730 // NOTE: If the platform does not provide any short branch variants, then
 1731 //       this method should return false for offset 0.
 1732 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1733   // The passed offset is relative to address of the branch.
 1734   // On 86 a branch displacement is calculated relative to address
 1735   // of a next instruction.
 1736   offset -= br_size;
 1737 
 1738   // the short version of jmpConUCF2 contains multiple branches,
 1739   // making the reach slightly less
 1740   if (rule == jmpConUCF2_rule)
 1741     return (-126 <= offset && offset <= 125);
 1742   return (-128 <= offset && offset <= 127);
 1743 }
 1744 
 1745 // Return whether or not this register is ever used as an argument.
 1746 // This function is used on startup to build the trampoline stubs in
 1747 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1748 // call in the trampoline, and arguments in those registers not be
 1749 // available to the callee.
 1750 bool Matcher::can_be_java_arg(int reg)
 1751 {
 1752   return
 1753     reg ==  RDI_num || reg == RDI_H_num ||
 1754     reg ==  RSI_num || reg == RSI_H_num ||
 1755     reg ==  RDX_num || reg == RDX_H_num ||
 1756     reg ==  RCX_num || reg == RCX_H_num ||
 1757     reg ==   R8_num || reg ==  R8_H_num ||
 1758     reg ==   R9_num || reg ==  R9_H_num ||
 1759     reg ==  R12_num || reg == R12_H_num ||
 1760     reg == XMM0_num || reg == XMM0b_num ||
 1761     reg == XMM1_num || reg == XMM1b_num ||
 1762     reg == XMM2_num || reg == XMM2b_num ||
 1763     reg == XMM3_num || reg == XMM3b_num ||
 1764     reg == XMM4_num || reg == XMM4b_num ||
 1765     reg == XMM5_num || reg == XMM5b_num ||
 1766     reg == XMM6_num || reg == XMM6b_num ||
 1767     reg == XMM7_num || reg == XMM7b_num;
 1768 }
 1769 
 1770 bool Matcher::is_spillable_arg(int reg)
 1771 {
 1772   return can_be_java_arg(reg);
 1773 }
 1774 
 1775 uint Matcher::int_pressure_limit()
 1776 {
 1777   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1778 }
 1779 
 1780 uint Matcher::float_pressure_limit()
 1781 {
 1782   // After experiment around with different values, the following default threshold
 1783   // works best for LCM's register pressure scheduling on x64.
 1784   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1785   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1786   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1787 }
 1788 
 1789 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1790   // In 64 bit mode a code which use multiply when
 1791   // devisor is constant is faster than hardware
 1792   // DIV instruction (it uses MulHiL).
 1793   return false;
 1794 }
 1795 
 1796 // Register for DIVI projection of divmodI
 1797 RegMask Matcher::divI_proj_mask() {
 1798   return INT_RAX_REG_mask();
 1799 }
 1800 
 1801 // Register for MODI projection of divmodI
 1802 RegMask Matcher::modI_proj_mask() {
 1803   return INT_RDX_REG_mask();
 1804 }
 1805 
 1806 // Register for DIVL projection of divmodL
 1807 RegMask Matcher::divL_proj_mask() {
 1808   return LONG_RAX_REG_mask();
 1809 }
 1810 
 1811 // Register for MODL projection of divmodL
 1812 RegMask Matcher::modL_proj_mask() {
 1813   return LONG_RDX_REG_mask();
 1814 }
 1815 
 1816 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1817 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1818     return NO_REG_mask();
 1819 }
 1820 
 1821 %}
 1822 
 1823 //----------ENCODING BLOCK-----------------------------------------------------
 1824 // This block specifies the encoding classes used by the compiler to
 1825 // output byte streams.  Encoding classes are parameterized macros
 1826 // used by Machine Instruction Nodes in order to generate the bit
 1827 // encoding of the instruction.  Operands specify their base encoding
 1828 // interface with the interface keyword.  There are currently
 1829 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1830 // COND_INTER.  REG_INTER causes an operand to generate a function
 1831 // which returns its register number when queried.  CONST_INTER causes
 1832 // an operand to generate a function which returns the value of the
 1833 // constant when queried.  MEMORY_INTER causes an operand to generate
 1834 // four functions which return the Base Register, the Index Register,
 1835 // the Scale Value, and the Offset Value of the operand when queried.
 1836 // COND_INTER causes an operand to generate six functions which return
 1837 // the encoding code (ie - encoding bits for the instruction)
 1838 // associated with each basic boolean condition for a conditional
 1839 // instruction.
 1840 //
 1841 // Instructions specify two basic values for encoding.  Again, a
 1842 // function is available to check if the constant displacement is an
 1843 // oop. They use the ins_encode keyword to specify their encoding
 1844 // classes (which must be a sequence of enc_class names, and their
 1845 // parameters, specified in the encoding block), and they use the
 1846 // opcode keyword to specify, in order, their primary, secondary, and
 1847 // tertiary opcode.  Only the opcode sections which a particular
 1848 // instruction needs for encoding need to be specified.
 1849 encode %{
 1850   // Build emit functions for each basic byte or larger field in the
 1851   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1852   // from C++ code in the enc_class source block.  Emit functions will
 1853   // live in the main source block for now.  In future, we can
 1854   // generalize this by adding a syntax that specifies the sizes of
 1855   // fields in an order, so that the adlc can build the emit functions
 1856   // automagically
 1857 
 1858   // Emit primary opcode
 1859   enc_class OpcP
 1860   %{
 1861     emit_opcode(cbuf, $primary);
 1862   %}
 1863 
 1864   // Emit secondary opcode
 1865   enc_class OpcS
 1866   %{
 1867     emit_opcode(cbuf, $secondary);
 1868   %}
 1869 
 1870   // Emit tertiary opcode
 1871   enc_class OpcT
 1872   %{
 1873     emit_opcode(cbuf, $tertiary);
 1874   %}
 1875 
 1876   // Emit opcode directly
 1877   enc_class Opcode(immI d8)
 1878   %{
 1879     emit_opcode(cbuf, $d8$$constant);
 1880   %}
 1881 
 1882   // Emit size prefix
 1883   enc_class SizePrefix
 1884   %{
 1885     emit_opcode(cbuf, 0x66);
 1886   %}
 1887 
 1888   enc_class reg(rRegI reg)
 1889   %{
 1890     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1891   %}
 1892 
 1893   enc_class reg_reg(rRegI dst, rRegI src)
 1894   %{
 1895     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1896   %}
 1897 
 1898   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1899   %{
 1900     emit_opcode(cbuf, $opcode$$constant);
 1901     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1902   %}
 1903 
 1904   enc_class cdql_enc(no_rax_rdx_RegI div)
 1905   %{
 1906     // Full implementation of Java idiv and irem; checks for
 1907     // special case as described in JVM spec., p.243 & p.271.
 1908     //
 1909     //         normal case                           special case
 1910     //
 1911     // input : rax: dividend                         min_int
 1912     //         reg: divisor                          -1
 1913     //
 1914     // output: rax: quotient  (= rax idiv reg)       min_int
 1915     //         rdx: remainder (= rax irem reg)       0
 1916     //
 1917     //  Code sequnce:
 1918     //
 1919     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1920     //    5:   75 07/08                jne    e <normal>
 1921     //    7:   33 d2                   xor    %edx,%edx
 1922     //  [div >= 8 -> offset + 1]
 1923     //  [REX_B]
 1924     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1925     //    c:   74 03/04                je     11 <done>
 1926     // 000000000000000e <normal>:
 1927     //    e:   99                      cltd
 1928     //  [div >= 8 -> offset + 1]
 1929     //  [REX_B]
 1930     //    f:   f7 f9                   idiv   $div
 1931     // 0000000000000011 <done>:
 1932     MacroAssembler _masm(&cbuf);
 1933     Label normal;
 1934     Label done;
 1935 
 1936     // cmp    $0x80000000,%eax
 1937     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1938 
 1939     // jne    e <normal>
 1940     __ jccb(Assembler::notEqual, normal);
 1941 
 1942     // xor    %edx,%edx
 1943     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1944 
 1945     // cmp    $0xffffffffffffffff,%ecx
 1946     __ cmpl($div$$Register, -1);
 1947 
 1948     // je     11 <done>
 1949     __ jccb(Assembler::equal, done);
 1950 
 1951     // <normal>
 1952     // cltd
 1953     __ bind(normal);
 1954     __ cdql();
 1955 
 1956     // idivl
 1957     // <done>
 1958     __ idivl($div$$Register);
 1959     __ bind(done);
 1960   %}
 1961 
 1962   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1963   %{
 1964     // Full implementation of Java ldiv and lrem; checks for
 1965     // special case as described in JVM spec., p.243 & p.271.
 1966     //
 1967     //         normal case                           special case
 1968     //
 1969     // input : rax: dividend                         min_long
 1970     //         reg: divisor                          -1
 1971     //
 1972     // output: rax: quotient  (= rax idiv reg)       min_long
 1973     //         rdx: remainder (= rax irem reg)       0
 1974     //
 1975     //  Code sequnce:
 1976     //
 1977     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1978     //    7:   00 00 80
 1979     //    a:   48 39 d0                cmp    %rdx,%rax
 1980     //    d:   75 08                   jne    17 <normal>
 1981     //    f:   33 d2                   xor    %edx,%edx
 1982     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1983     //   15:   74 05                   je     1c <done>
 1984     // 0000000000000017 <normal>:
 1985     //   17:   48 99                   cqto
 1986     //   19:   48 f7 f9                idiv   $div
 1987     // 000000000000001c <done>:
 1988     MacroAssembler _masm(&cbuf);
 1989     Label normal;
 1990     Label done;
 1991 
 1992     // mov    $0x8000000000000000,%rdx
 1993     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1994 
 1995     // cmp    %rdx,%rax
 1996     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1997 
 1998     // jne    17 <normal>
 1999     __ jccb(Assembler::notEqual, normal);
 2000 
 2001     // xor    %edx,%edx
 2002     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2003 
 2004     // cmp    $0xffffffffffffffff,$div
 2005     __ cmpq($div$$Register, -1);
 2006 
 2007     // je     1e <done>
 2008     __ jccb(Assembler::equal, done);
 2009 
 2010     // <normal>
 2011     // cqto
 2012     __ bind(normal);
 2013     __ cdqq();
 2014 
 2015     // idivq (note: must be emitted by the user of this rule)
 2016     // <done>
 2017     __ idivq($div$$Register);
 2018     __ bind(done);
 2019   %}
 2020 
 2021   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2022   enc_class OpcSE(immI imm)
 2023   %{
 2024     // Emit primary opcode and set sign-extend bit
 2025     // Check for 8-bit immediate, and set sign extend bit in opcode
 2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2027       emit_opcode(cbuf, $primary | 0x02);
 2028     } else {
 2029       // 32-bit immediate
 2030       emit_opcode(cbuf, $primary);
 2031     }
 2032   %}
 2033 
 2034   enc_class OpcSErm(rRegI dst, immI imm)
 2035   %{
 2036     // OpcSEr/m
 2037     int dstenc = $dst$$reg;
 2038     if (dstenc >= 8) {
 2039       emit_opcode(cbuf, Assembler::REX_B);
 2040       dstenc -= 8;
 2041     }
 2042     // Emit primary opcode and set sign-extend bit
 2043     // Check for 8-bit immediate, and set sign extend bit in opcode
 2044     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2045       emit_opcode(cbuf, $primary | 0x02);
 2046     } else {
 2047       // 32-bit immediate
 2048       emit_opcode(cbuf, $primary);
 2049     }
 2050     // Emit r/m byte with secondary opcode, after primary opcode.
 2051     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2052   %}
 2053 
 2054   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2055   %{
 2056     // OpcSEr/m
 2057     int dstenc = $dst$$reg;
 2058     if (dstenc < 8) {
 2059       emit_opcode(cbuf, Assembler::REX_W);
 2060     } else {
 2061       emit_opcode(cbuf, Assembler::REX_WB);
 2062       dstenc -= 8;
 2063     }
 2064     // Emit primary opcode and set sign-extend bit
 2065     // Check for 8-bit immediate, and set sign extend bit in opcode
 2066     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2067       emit_opcode(cbuf, $primary | 0x02);
 2068     } else {
 2069       // 32-bit immediate
 2070       emit_opcode(cbuf, $primary);
 2071     }
 2072     // Emit r/m byte with secondary opcode, after primary opcode.
 2073     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2074   %}
 2075 
 2076   enc_class Con8or32(immI imm)
 2077   %{
 2078     // Check for 8-bit immediate, and set sign extend bit in opcode
 2079     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2080       $$$emit8$imm$$constant;
 2081     } else {
 2082       // 32-bit immediate
 2083       $$$emit32$imm$$constant;
 2084     }
 2085   %}
 2086 
 2087   enc_class opc2_reg(rRegI dst)
 2088   %{
 2089     // BSWAP
 2090     emit_cc(cbuf, $secondary, $dst$$reg);
 2091   %}
 2092 
 2093   enc_class opc3_reg(rRegI dst)
 2094   %{
 2095     // BSWAP
 2096     emit_cc(cbuf, $tertiary, $dst$$reg);
 2097   %}
 2098 
 2099   enc_class reg_opc(rRegI div)
 2100   %{
 2101     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2102     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2103   %}
 2104 
 2105   enc_class enc_cmov(cmpOp cop)
 2106   %{
 2107     // CMOV
 2108     $$$emit8$primary;
 2109     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2110   %}
 2111 
 2112   enc_class enc_PartialSubtypeCheck()
 2113   %{
 2114     Register Rrdi = as_Register(RDI_enc); // result register
 2115     Register Rrax = as_Register(RAX_enc); // super class
 2116     Register Rrcx = as_Register(RCX_enc); // killed
 2117     Register Rrsi = as_Register(RSI_enc); // sub class
 2118     Label miss;
 2119     const bool set_cond_codes = true;
 2120 
 2121     MacroAssembler _masm(&cbuf);
 2122     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2123                                      NULL, &miss,
 2124                                      /*set_cond_codes:*/ true);
 2125     if ($primary) {
 2126       __ xorptr(Rrdi, Rrdi);
 2127     }
 2128     __ bind(miss);
 2129   %}
 2130 
 2131   enc_class clear_avx %{
 2132     debug_only(int off0 = cbuf.insts_size());
 2133     if (generate_vzeroupper(Compile::current())) {
 2134       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2135       // Clear upper bits of YMM registers when current compiled code uses
 2136       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2137       MacroAssembler _masm(&cbuf);
 2138       __ vzeroupper();
 2139     }
 2140     debug_only(int off1 = cbuf.insts_size());
 2141     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2142   %}
 2143 
 2144   enc_class Java_To_Runtime(method meth) %{
 2145     // No relocation needed
 2146     MacroAssembler _masm(&cbuf);
 2147     __ mov64(r10, (int64_t) $meth$$method);
 2148     __ call(r10);
 2149     __ post_call_nop();
 2150   %}
 2151 
 2152   enc_class Java_Static_Call(method meth)
 2153   %{
 2154     // JAVA STATIC CALL
 2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2156     // determine who we intended to call.
 2157     MacroAssembler _masm(&cbuf);
 2158     cbuf.set_insts_mark();
 2159     $$$emit8$primary;
 2160 
 2161     if (!_method) {
 2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2163                      runtime_call_Relocation::spec(),
 2164                      RELOC_DISP32);
 2165     } else {
 2166       int method_index = resolved_method_index(cbuf);
 2167       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2168                                                   : static_call_Relocation::spec(method_index);
 2169       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2170                      rspec, RELOC_DISP32);
 2171       address mark = cbuf.insts_mark();
 2172       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2173         // Calls of the same statically bound method can share
 2174         // a stub to the interpreter.
 2175         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2176       } else {
 2177         // Emit stubs for static call.
 2178         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2179         if (stub == NULL) {
 2180           ciEnv::current()->record_failure("CodeCache is full");
 2181           return;
 2182         }
 2183       }
 2184     }
 2185     _masm.clear_inst_mark();
 2186     __ post_call_nop();
 2187   %}
 2188 
 2189   enc_class Java_Dynamic_Call(method meth) %{
 2190     MacroAssembler _masm(&cbuf);
 2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2192     __ post_call_nop();
 2193   %}
 2194 
 2195   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2196   %{
 2197     // SAL, SAR, SHR
 2198     int dstenc = $dst$$reg;
 2199     if (dstenc >= 8) {
 2200       emit_opcode(cbuf, Assembler::REX_B);
 2201       dstenc -= 8;
 2202     }
 2203     $$$emit8$primary;
 2204     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2205     $$$emit8$shift$$constant;
 2206   %}
 2207 
 2208   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2209   %{
 2210     // SAL, SAR, SHR
 2211     int dstenc = $dst$$reg;
 2212     if (dstenc < 8) {
 2213       emit_opcode(cbuf, Assembler::REX_W);
 2214     } else {
 2215       emit_opcode(cbuf, Assembler::REX_WB);
 2216       dstenc -= 8;
 2217     }
 2218     $$$emit8$primary;
 2219     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2220     $$$emit8$shift$$constant;
 2221   %}
 2222 
 2223   enc_class load_immI(rRegI dst, immI src)
 2224   %{
 2225     int dstenc = $dst$$reg;
 2226     if (dstenc >= 8) {
 2227       emit_opcode(cbuf, Assembler::REX_B);
 2228       dstenc -= 8;
 2229     }
 2230     emit_opcode(cbuf, 0xB8 | dstenc);
 2231     $$$emit32$src$$constant;
 2232   %}
 2233 
 2234   enc_class load_immL(rRegL dst, immL src)
 2235   %{
 2236     int dstenc = $dst$$reg;
 2237     if (dstenc < 8) {
 2238       emit_opcode(cbuf, Assembler::REX_W);
 2239     } else {
 2240       emit_opcode(cbuf, Assembler::REX_WB);
 2241       dstenc -= 8;
 2242     }
 2243     emit_opcode(cbuf, 0xB8 | dstenc);
 2244     emit_d64(cbuf, $src$$constant);
 2245   %}
 2246 
 2247   enc_class load_immUL32(rRegL dst, immUL32 src)
 2248   %{
 2249     // same as load_immI, but this time we care about zeroes in the high word
 2250     int dstenc = $dst$$reg;
 2251     if (dstenc >= 8) {
 2252       emit_opcode(cbuf, Assembler::REX_B);
 2253       dstenc -= 8;
 2254     }
 2255     emit_opcode(cbuf, 0xB8 | dstenc);
 2256     $$$emit32$src$$constant;
 2257   %}
 2258 
 2259   enc_class load_immL32(rRegL dst, immL32 src)
 2260   %{
 2261     int dstenc = $dst$$reg;
 2262     if (dstenc < 8) {
 2263       emit_opcode(cbuf, Assembler::REX_W);
 2264     } else {
 2265       emit_opcode(cbuf, Assembler::REX_WB);
 2266       dstenc -= 8;
 2267     }
 2268     emit_opcode(cbuf, 0xC7);
 2269     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2270     $$$emit32$src$$constant;
 2271   %}
 2272 
 2273   enc_class load_immP31(rRegP dst, immP32 src)
 2274   %{
 2275     // same as load_immI, but this time we care about zeroes in the high word
 2276     int dstenc = $dst$$reg;
 2277     if (dstenc >= 8) {
 2278       emit_opcode(cbuf, Assembler::REX_B);
 2279       dstenc -= 8;
 2280     }
 2281     emit_opcode(cbuf, 0xB8 | dstenc);
 2282     $$$emit32$src$$constant;
 2283   %}
 2284 
 2285   enc_class load_immP(rRegP dst, immP src)
 2286   %{
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc < 8) {
 2289       emit_opcode(cbuf, Assembler::REX_W);
 2290     } else {
 2291       emit_opcode(cbuf, Assembler::REX_WB);
 2292       dstenc -= 8;
 2293     }
 2294     emit_opcode(cbuf, 0xB8 | dstenc);
 2295     // This next line should be generated from ADLC
 2296     if ($src->constant_reloc() != relocInfo::none) {
 2297       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2298     } else {
 2299       emit_d64(cbuf, $src$$constant);
 2300     }
 2301   %}
 2302 
 2303   enc_class Con32(immI src)
 2304   %{
 2305     // Output immediate
 2306     $$$emit32$src$$constant;
 2307   %}
 2308 
 2309   enc_class Con32F_as_bits(immF src)
 2310   %{
 2311     // Output Float immediate bits
 2312     jfloat jf = $src$$constant;
 2313     jint jf_as_bits = jint_cast(jf);
 2314     emit_d32(cbuf, jf_as_bits);
 2315   %}
 2316 
 2317   enc_class Con16(immI src)
 2318   %{
 2319     // Output immediate
 2320     $$$emit16$src$$constant;
 2321   %}
 2322 
 2323   // How is this different from Con32??? XXX
 2324   enc_class Con_d32(immI src)
 2325   %{
 2326     emit_d32(cbuf,$src$$constant);
 2327   %}
 2328 
 2329   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2330     // Output immediate memory reference
 2331     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2332     emit_d32(cbuf, 0x00);
 2333   %}
 2334 
 2335   enc_class lock_prefix()
 2336   %{
 2337     emit_opcode(cbuf, 0xF0); // lock
 2338   %}
 2339 
 2340   enc_class REX_mem(memory mem)
 2341   %{
 2342     if ($mem$$base >= 8) {
 2343       if ($mem$$index < 8) {
 2344         emit_opcode(cbuf, Assembler::REX_B);
 2345       } else {
 2346         emit_opcode(cbuf, Assembler::REX_XB);
 2347       }
 2348     } else {
 2349       if ($mem$$index >= 8) {
 2350         emit_opcode(cbuf, Assembler::REX_X);
 2351       }
 2352     }
 2353   %}
 2354 
 2355   enc_class REX_mem_wide(memory mem)
 2356   %{
 2357     if ($mem$$base >= 8) {
 2358       if ($mem$$index < 8) {
 2359         emit_opcode(cbuf, Assembler::REX_WB);
 2360       } else {
 2361         emit_opcode(cbuf, Assembler::REX_WXB);
 2362       }
 2363     } else {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_W);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_WX);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   // for byte regs
 2373   enc_class REX_breg(rRegI reg)
 2374   %{
 2375     if ($reg$$reg >= 4) {
 2376       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2382   %{
 2383     if ($dst$$reg < 8) {
 2384       if ($src$$reg >= 4) {
 2385         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2386       }
 2387     } else {
 2388       if ($src$$reg < 8) {
 2389         emit_opcode(cbuf, Assembler::REX_R);
 2390       } else {
 2391         emit_opcode(cbuf, Assembler::REX_RB);
 2392       }
 2393     }
 2394   %}
 2395 
 2396   // for byte regs
 2397   enc_class REX_breg_mem(rRegI reg, memory mem)
 2398   %{
 2399     if ($reg$$reg < 8) {
 2400       if ($mem$$base < 8) {
 2401         if ($mem$$index >= 8) {
 2402           emit_opcode(cbuf, Assembler::REX_X);
 2403         } else if ($reg$$reg >= 4) {
 2404           emit_opcode(cbuf, Assembler::REX);
 2405         }
 2406       } else {
 2407         if ($mem$$index < 8) {
 2408           emit_opcode(cbuf, Assembler::REX_B);
 2409         } else {
 2410           emit_opcode(cbuf, Assembler::REX_XB);
 2411         }
 2412       }
 2413     } else {
 2414       if ($mem$$base < 8) {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_R);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_RX);
 2419         }
 2420       } else {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_RB);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RXB);
 2425         }
 2426       }
 2427     }
 2428   %}
 2429 
 2430   enc_class REX_reg(rRegI reg)
 2431   %{
 2432     if ($reg$$reg >= 8) {
 2433       emit_opcode(cbuf, Assembler::REX_B);
 2434     }
 2435   %}
 2436 
 2437   enc_class REX_reg_wide(rRegI reg)
 2438   %{
 2439     if ($reg$$reg < 8) {
 2440       emit_opcode(cbuf, Assembler::REX_W);
 2441     } else {
 2442       emit_opcode(cbuf, Assembler::REX_WB);
 2443     }
 2444   %}
 2445 
 2446   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2447   %{
 2448     if ($dst$$reg < 8) {
 2449       if ($src$$reg >= 8) {
 2450         emit_opcode(cbuf, Assembler::REX_B);
 2451       }
 2452     } else {
 2453       if ($src$$reg < 8) {
 2454         emit_opcode(cbuf, Assembler::REX_R);
 2455       } else {
 2456         emit_opcode(cbuf, Assembler::REX_RB);
 2457       }
 2458     }
 2459   %}
 2460 
 2461   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2462   %{
 2463     if ($dst$$reg < 8) {
 2464       if ($src$$reg < 8) {
 2465         emit_opcode(cbuf, Assembler::REX_W);
 2466       } else {
 2467         emit_opcode(cbuf, Assembler::REX_WB);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_WR);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_WRB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_mem(rRegI reg, memory mem)
 2479   %{
 2480     if ($reg$$reg < 8) {
 2481       if ($mem$$base < 8) {
 2482         if ($mem$$index >= 8) {
 2483           emit_opcode(cbuf, Assembler::REX_X);
 2484         }
 2485       } else {
 2486         if ($mem$$index < 8) {
 2487           emit_opcode(cbuf, Assembler::REX_B);
 2488         } else {
 2489           emit_opcode(cbuf, Assembler::REX_XB);
 2490         }
 2491       }
 2492     } else {
 2493       if ($mem$$base < 8) {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_R);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_RX);
 2498         }
 2499       } else {
 2500         if ($mem$$index < 8) {
 2501           emit_opcode(cbuf, Assembler::REX_RB);
 2502         } else {
 2503           emit_opcode(cbuf, Assembler::REX_RXB);
 2504         }
 2505       }
 2506     }
 2507   %}
 2508 
 2509   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2510   %{
 2511     if ($reg$$reg < 8) {
 2512       if ($mem$$base < 8) {
 2513         if ($mem$$index < 8) {
 2514           emit_opcode(cbuf, Assembler::REX_W);
 2515         } else {
 2516           emit_opcode(cbuf, Assembler::REX_WX);
 2517         }
 2518       } else {
 2519         if ($mem$$index < 8) {
 2520           emit_opcode(cbuf, Assembler::REX_WB);
 2521         } else {
 2522           emit_opcode(cbuf, Assembler::REX_WXB);
 2523         }
 2524       }
 2525     } else {
 2526       if ($mem$$base < 8) {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WR);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WRX);
 2531         }
 2532       } else {
 2533         if ($mem$$index < 8) {
 2534           emit_opcode(cbuf, Assembler::REX_WRB);
 2535         } else {
 2536           emit_opcode(cbuf, Assembler::REX_WRXB);
 2537         }
 2538       }
 2539     }
 2540   %}
 2541 
 2542   enc_class reg_mem(rRegI ereg, memory mem)
 2543   %{
 2544     // High registers handle in encode_RegMem
 2545     int reg = $ereg$$reg;
 2546     int base = $mem$$base;
 2547     int index = $mem$$index;
 2548     int scale = $mem$$scale;
 2549     int disp = $mem$$disp;
 2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2551 
 2552     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2553   %}
 2554 
 2555   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2556   %{
 2557     int rm_byte_opcode = $rm_opcode$$constant;
 2558 
 2559     // High registers handle in encode_RegMem
 2560     int base = $mem$$base;
 2561     int index = $mem$$index;
 2562     int scale = $mem$$scale;
 2563     int displace = $mem$$disp;
 2564 
 2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2566                                             // working with static
 2567                                             // globals
 2568     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2569                   disp_reloc);
 2570   %}
 2571 
 2572   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2573   %{
 2574     int reg_encoding = $dst$$reg;
 2575     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2576     int index        = 0x04;            // 0x04 indicates no index
 2577     int scale        = 0x00;            // 0x00 indicates no scale
 2578     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2579     relocInfo::relocType disp_reloc = relocInfo::none;
 2580     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2581                   disp_reloc);
 2582   %}
 2583 
 2584   enc_class neg_reg(rRegI dst)
 2585   %{
 2586     int dstenc = $dst$$reg;
 2587     if (dstenc >= 8) {
 2588       emit_opcode(cbuf, Assembler::REX_B);
 2589       dstenc -= 8;
 2590     }
 2591     // NEG $dst
 2592     emit_opcode(cbuf, 0xF7);
 2593     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2594   %}
 2595 
 2596   enc_class neg_reg_wide(rRegI dst)
 2597   %{
 2598     int dstenc = $dst$$reg;
 2599     if (dstenc < 8) {
 2600       emit_opcode(cbuf, Assembler::REX_W);
 2601     } else {
 2602       emit_opcode(cbuf, Assembler::REX_WB);
 2603       dstenc -= 8;
 2604     }
 2605     // NEG $dst
 2606     emit_opcode(cbuf, 0xF7);
 2607     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2608   %}
 2609 
 2610   enc_class setLT_reg(rRegI dst)
 2611   %{
 2612     int dstenc = $dst$$reg;
 2613     if (dstenc >= 8) {
 2614       emit_opcode(cbuf, Assembler::REX_B);
 2615       dstenc -= 8;
 2616     } else if (dstenc >= 4) {
 2617       emit_opcode(cbuf, Assembler::REX);
 2618     }
 2619     // SETLT $dst
 2620     emit_opcode(cbuf, 0x0F);
 2621     emit_opcode(cbuf, 0x9C);
 2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2623   %}
 2624 
 2625   enc_class setNZ_reg(rRegI dst)
 2626   %{
 2627     int dstenc = $dst$$reg;
 2628     if (dstenc >= 8) {
 2629       emit_opcode(cbuf, Assembler::REX_B);
 2630       dstenc -= 8;
 2631     } else if (dstenc >= 4) {
 2632       emit_opcode(cbuf, Assembler::REX);
 2633     }
 2634     // SETNZ $dst
 2635     emit_opcode(cbuf, 0x0F);
 2636     emit_opcode(cbuf, 0x95);
 2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2638   %}
 2639 
 2640 
 2641   // Compare the lonogs and set -1, 0, or 1 into dst
 2642   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2643   %{
 2644     int src1enc = $src1$$reg;
 2645     int src2enc = $src2$$reg;
 2646     int dstenc = $dst$$reg;
 2647 
 2648     // cmpq $src1, $src2
 2649     if (src1enc < 8) {
 2650       if (src2enc < 8) {
 2651         emit_opcode(cbuf, Assembler::REX_W);
 2652       } else {
 2653         emit_opcode(cbuf, Assembler::REX_WB);
 2654       }
 2655     } else {
 2656       if (src2enc < 8) {
 2657         emit_opcode(cbuf, Assembler::REX_WR);
 2658       } else {
 2659         emit_opcode(cbuf, Assembler::REX_WRB);
 2660       }
 2661     }
 2662     emit_opcode(cbuf, 0x3B);
 2663     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2664 
 2665     // movl $dst, -1
 2666     if (dstenc >= 8) {
 2667       emit_opcode(cbuf, Assembler::REX_B);
 2668     }
 2669     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2670     emit_d32(cbuf, -1);
 2671 
 2672     // jl,s done
 2673     emit_opcode(cbuf, 0x7C);
 2674     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2675 
 2676     // setne $dst
 2677     if (dstenc >= 4) {
 2678       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2679     }
 2680     emit_opcode(cbuf, 0x0F);
 2681     emit_opcode(cbuf, 0x95);
 2682     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2683 
 2684     // movzbl $dst, $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0xB6);
 2690     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2691   %}
 2692 
 2693   enc_class Push_ResultXD(regD dst) %{
 2694     MacroAssembler _masm(&cbuf);
 2695     __ fstp_d(Address(rsp, 0));
 2696     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2697     __ addptr(rsp, 8);
 2698   %}
 2699 
 2700   enc_class Push_SrcXD(regD src) %{
 2701     MacroAssembler _masm(&cbuf);
 2702     __ subptr(rsp, 8);
 2703     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2704     __ fld_d(Address(rsp, 0));
 2705   %}
 2706 
 2707 
 2708   enc_class enc_rethrow()
 2709   %{
 2710     cbuf.set_insts_mark();
 2711     emit_opcode(cbuf, 0xE9); // jmp entry
 2712     emit_d32_reloc(cbuf,
 2713                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2714                    runtime_call_Relocation::spec(),
 2715                    RELOC_DISP32);
 2716   %}
 2717 
 2718 %}
 2719 
 2720 
 2721 
 2722 //----------FRAME--------------------------------------------------------------
 2723 // Definition of frame structure and management information.
 2724 //
 2725 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2726 //                             |   (to get allocators register number
 2727 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2728 //  r   CALLER     |        |
 2729 //  o     |        +--------+      pad to even-align allocators stack-slot
 2730 //  w     V        |  pad0  |        numbers; owned by CALLER
 2731 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2732 //  h     ^        |   in   |  5
 2733 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2734 //  |     |        |        |  3
 2735 //  |     |        +--------+
 2736 //  V     |        | old out|      Empty on Intel, window on Sparc
 2737 //        |    old |preserve|      Must be even aligned.
 2738 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2739 //        |        |   in   |  3   area for Intel ret address
 2740 //     Owned by    |preserve|      Empty on Sparc.
 2741 //       SELF      +--------+
 2742 //        |        |  pad2  |  2   pad to align old SP
 2743 //        |        +--------+  1
 2744 //        |        | locks  |  0
 2745 //        |        +--------+----> OptoReg::stack0(), even aligned
 2746 //        |        |  pad1  | 11   pad to align new SP
 2747 //        |        +--------+
 2748 //        |        |        | 10
 2749 //        |        | spills |  9   spills
 2750 //        V        |        |  8   (pad0 slot for callee)
 2751 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2752 //        ^        |  out   |  7
 2753 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2754 //     Owned by    +--------+
 2755 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2756 //        |    new |preserve|      Must be even-aligned.
 2757 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2758 //        |        |        |
 2759 //
 2760 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2761 //         known from SELF's arguments and the Java calling convention.
 2762 //         Region 6-7 is determined per call site.
 2763 // Note 2: If the calling convention leaves holes in the incoming argument
 2764 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2765 //         are owned by the CALLEE.  Holes should not be necessary in the
 2766 //         incoming area, as the Java calling convention is completely under
 2767 //         the control of the AD file.  Doubles can be sorted and packed to
 2768 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2769 //         varargs C calling conventions.
 2770 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2771 //         even aligned with pad0 as needed.
 2772 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2773 //         region 6-11 is even aligned; it may be padded out more so that
 2774 //         the region from SP to FP meets the minimum stack alignment.
 2775 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2776 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2777 //         SP meets the minimum alignment.
 2778 
 2779 frame
 2780 %{
 2781   // These three registers define part of the calling convention
 2782   // between compiled code and the interpreter.
 2783   inline_cache_reg(RAX);                // Inline Cache Register
 2784 
 2785   // Optional: name the operand used by cisc-spilling to access
 2786   // [stack_pointer + offset]
 2787   cisc_spilling_operand_name(indOffset32);
 2788 
 2789   // Number of stack slots consumed by locking an object
 2790   sync_stack_slots(2);
 2791 
 2792   // Compiled code's Frame Pointer
 2793   frame_pointer(RSP);
 2794 
 2795   // Interpreter stores its frame pointer in a register which is
 2796   // stored to the stack by I2CAdaptors.
 2797   // I2CAdaptors convert from interpreted java to compiled java.
 2798   interpreter_frame_pointer(RBP);
 2799 
 2800   // Stack alignment requirement
 2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2802 
 2803   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2804   // for calls to C.  Supports the var-args backing area for register parms.
 2805   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2806 
 2807   // The after-PROLOG location of the return address.  Location of
 2808   // return address specifies a type (REG or STACK) and a number
 2809   // representing the register number (i.e. - use a register name) or
 2810   // stack slot.
 2811   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2812   // Otherwise, it is above the locks and verification slot and alignment word
 2813   return_addr(STACK - 2 +
 2814               align_up((Compile::current()->in_preserve_stack_slots() +
 2815                         Compile::current()->fixed_slots()),
 2816                        stack_alignment_in_slots()));
 2817 
 2818   // Location of compiled Java return values.  Same as C for now.
 2819   return_value
 2820   %{
 2821     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2822            "only return normal values");
 2823 
 2824     static const int lo[Op_RegL + 1] = {
 2825       0,
 2826       0,
 2827       RAX_num,  // Op_RegN
 2828       RAX_num,  // Op_RegI
 2829       RAX_num,  // Op_RegP
 2830       XMM0_num, // Op_RegF
 2831       XMM0_num, // Op_RegD
 2832       RAX_num   // Op_RegL
 2833     };
 2834     static const int hi[Op_RegL + 1] = {
 2835       0,
 2836       0,
 2837       OptoReg::Bad, // Op_RegN
 2838       OptoReg::Bad, // Op_RegI
 2839       RAX_H_num,    // Op_RegP
 2840       OptoReg::Bad, // Op_RegF
 2841       XMM0b_num,    // Op_RegD
 2842       RAX_H_num     // Op_RegL
 2843     };
 2844     // Excluded flags and vector registers.
 2845     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2846     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2847   %}
 2848 %}
 2849 
 2850 //----------ATTRIBUTES---------------------------------------------------------
 2851 //----------Operand Attributes-------------------------------------------------
 2852 op_attrib op_cost(0);        // Required cost attribute
 2853 
 2854 //----------Instruction Attributes---------------------------------------------
 2855 ins_attrib ins_cost(100);       // Required cost attribute
 2856 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2857 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2858                                 // a non-matching short branch variant
 2859                                 // of some long branch?
 2860 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2861                                 // be a power of 2) specifies the
 2862                                 // alignment that some part of the
 2863                                 // instruction (not necessarily the
 2864                                 // start) requires.  If > 1, a
 2865                                 // compute_padding() function must be
 2866                                 // provided for the instruction
 2867 
 2868 //----------OPERANDS-----------------------------------------------------------
 2869 // Operand definitions must precede instruction definitions for correct parsing
 2870 // in the ADLC because operands constitute user defined types which are used in
 2871 // instruction definitions.
 2872 
 2873 //----------Simple Operands----------------------------------------------------
 2874 // Immediate Operands
 2875 // Integer Immediate
 2876 operand immI()
 2877 %{
 2878   match(ConI);
 2879 
 2880   op_cost(10);
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 // Constant for test vs zero
 2886 operand immI_0()
 2887 %{
 2888   predicate(n->get_int() == 0);
 2889   match(ConI);
 2890 
 2891   op_cost(0);
 2892   format %{ %}
 2893   interface(CONST_INTER);
 2894 %}
 2895 
 2896 // Constant for increment
 2897 operand immI_1()
 2898 %{
 2899   predicate(n->get_int() == 1);
 2900   match(ConI);
 2901 
 2902   op_cost(0);
 2903   format %{ %}
 2904   interface(CONST_INTER);
 2905 %}
 2906 
 2907 // Constant for decrement
 2908 operand immI_M1()
 2909 %{
 2910   predicate(n->get_int() == -1);
 2911   match(ConI);
 2912 
 2913   op_cost(0);
 2914   format %{ %}
 2915   interface(CONST_INTER);
 2916 %}
 2917 
 2918 operand immI_2()
 2919 %{
 2920   predicate(n->get_int() == 2);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 operand immI_4()
 2929 %{
 2930   predicate(n->get_int() == 4);
 2931   match(ConI);
 2932 
 2933   op_cost(0);
 2934   format %{ %}
 2935   interface(CONST_INTER);
 2936 %}
 2937 
 2938 operand immI_8()
 2939 %{
 2940   predicate(n->get_int() == 8);
 2941   match(ConI);
 2942 
 2943   op_cost(0);
 2944   format %{ %}
 2945   interface(CONST_INTER);
 2946 %}
 2947 
 2948 // Valid scale values for addressing modes
 2949 operand immI2()
 2950 %{
 2951   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2952   match(ConI);
 2953 
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 operand immU7()
 2959 %{
 2960   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2961   match(ConI);
 2962 
 2963   op_cost(5);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 operand immI8()
 2969 %{
 2970   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2971   match(ConI);
 2972 
 2973   op_cost(5);
 2974   format %{ %}
 2975   interface(CONST_INTER);
 2976 %}
 2977 
 2978 operand immU8()
 2979 %{
 2980   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2981   match(ConI);
 2982 
 2983   op_cost(5);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immI16()
 2989 %{
 2990   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2991   match(ConI);
 2992 
 2993   op_cost(10);
 2994   format %{ %}
 2995   interface(CONST_INTER);
 2996 %}
 2997 
 2998 // Int Immediate non-negative
 2999 operand immU31()
 3000 %{
 3001   predicate(n->get_int() >= 0);
 3002   match(ConI);
 3003 
 3004   op_cost(0);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 // Constant for long shifts
 3010 operand immI_32()
 3011 %{
 3012   predicate( n->get_int() == 32 );
 3013   match(ConI);
 3014 
 3015   op_cost(0);
 3016   format %{ %}
 3017   interface(CONST_INTER);
 3018 %}
 3019 
 3020 // Constant for long shifts
 3021 operand immI_64()
 3022 %{
 3023   predicate( n->get_int() == 64 );
 3024   match(ConI);
 3025 
 3026   op_cost(0);
 3027   format %{ %}
 3028   interface(CONST_INTER);
 3029 %}
 3030 
 3031 // Pointer Immediate
 3032 operand immP()
 3033 %{
 3034   match(ConP);
 3035 
 3036   op_cost(10);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // NULL Pointer Immediate
 3042 operand immP0()
 3043 %{
 3044   predicate(n->get_ptr() == 0);
 3045   match(ConP);
 3046 
 3047   op_cost(5);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immN() %{
 3054   match(ConN);
 3055 
 3056   op_cost(10);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immNKlass() %{
 3062   match(ConNKlass);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // NULL Pointer Immediate
 3070 operand immN0() %{
 3071   predicate(n->get_narrowcon() == 0);
 3072   match(ConN);
 3073 
 3074   op_cost(5);
 3075   format %{ %}
 3076   interface(CONST_INTER);
 3077 %}
 3078 
 3079 operand immP31()
 3080 %{
 3081   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3082             && (n->get_ptr() >> 31) == 0);
 3083   match(ConP);
 3084 
 3085   op_cost(5);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 
 3091 // Long Immediate
 3092 operand immL()
 3093 %{
 3094   match(ConL);
 3095 
 3096   op_cost(20);
 3097   format %{ %}
 3098   interface(CONST_INTER);
 3099 %}
 3100 
 3101 // Long Immediate 8-bit
 3102 operand immL8()
 3103 %{
 3104   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3105   match(ConL);
 3106 
 3107   op_cost(5);
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate 32-bit unsigned
 3113 operand immUL32()
 3114 %{
 3115   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3116   match(ConL);
 3117 
 3118   op_cost(10);
 3119   format %{ %}
 3120   interface(CONST_INTER);
 3121 %}
 3122 
 3123 // Long Immediate 32-bit signed
 3124 operand immL32()
 3125 %{
 3126   predicate(n->get_long() == (int) (n->get_long()));
 3127   match(ConL);
 3128 
 3129   op_cost(15);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 operand immL_Pow2()
 3135 %{
 3136   predicate(is_power_of_2((julong)n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(15);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 operand immL_NotPow2()
 3145 %{
 3146   predicate(is_power_of_2((julong)~n->get_long()));
 3147   match(ConL);
 3148 
 3149   op_cost(15);
 3150   format %{ %}
 3151   interface(CONST_INTER);
 3152 %}
 3153 
 3154 // Long Immediate zero
 3155 operand immL0()
 3156 %{
 3157   predicate(n->get_long() == 0L);
 3158   match(ConL);
 3159 
 3160   op_cost(10);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 // Constant for increment
 3166 operand immL1()
 3167 %{
 3168   predicate(n->get_long() == 1);
 3169   match(ConL);
 3170 
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Constant for decrement
 3176 operand immL_M1()
 3177 %{
 3178   predicate(n->get_long() == -1);
 3179   match(ConL);
 3180 
 3181   format %{ %}
 3182   interface(CONST_INTER);
 3183 %}
 3184 
 3185 // Long Immediate: the value 10
 3186 operand immL10()
 3187 %{
 3188   predicate(n->get_long() == 10);
 3189   match(ConL);
 3190 
 3191   format %{ %}
 3192   interface(CONST_INTER);
 3193 %}
 3194 
 3195 // Long immediate from 0 to 127.
 3196 // Used for a shorter form of long mul by 10.
 3197 operand immL_127()
 3198 %{
 3199   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3200   match(ConL);
 3201 
 3202   op_cost(10);
 3203   format %{ %}
 3204   interface(CONST_INTER);
 3205 %}
 3206 
 3207 // Long Immediate: low 32-bit mask
 3208 operand immL_32bits()
 3209 %{
 3210   predicate(n->get_long() == 0xFFFFFFFFL);
 3211   match(ConL);
 3212   op_cost(20);
 3213 
 3214   format %{ %}
 3215   interface(CONST_INTER);
 3216 %}
 3217 
 3218 // Int Immediate: 2^n-1, positive
 3219 operand immI_Pow2M1()
 3220 %{
 3221   predicate((n->get_int() > 0)
 3222             && is_power_of_2(n->get_int() + 1));
 3223   match(ConI);
 3224 
 3225   op_cost(20);
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Float Immediate zero
 3231 operand immF0()
 3232 %{
 3233   predicate(jint_cast(n->getf()) == 0);
 3234   match(ConF);
 3235 
 3236   op_cost(5);
 3237   format %{ %}
 3238   interface(CONST_INTER);
 3239 %}
 3240 
 3241 // Float Immediate
 3242 operand immF()
 3243 %{
 3244   match(ConF);
 3245 
 3246   op_cost(15);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Double Immediate zero
 3252 operand immD0()
 3253 %{
 3254   predicate(jlong_cast(n->getd()) == 0);
 3255   match(ConD);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Double Immediate
 3263 operand immD()
 3264 %{
 3265   match(ConD);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Immediates for special shifts (sign extend)
 3273 
 3274 // Constants for increment
 3275 operand immI_16()
 3276 %{
 3277   predicate(n->get_int() == 16);
 3278   match(ConI);
 3279 
 3280   format %{ %}
 3281   interface(CONST_INTER);
 3282 %}
 3283 
 3284 operand immI_24()
 3285 %{
 3286   predicate(n->get_int() == 24);
 3287   match(ConI);
 3288 
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Constant for byte-wide masking
 3294 operand immI_255()
 3295 %{
 3296   predicate(n->get_int() == 255);
 3297   match(ConI);
 3298 
 3299   format %{ %}
 3300   interface(CONST_INTER);
 3301 %}
 3302 
 3303 // Constant for short-wide masking
 3304 operand immI_65535()
 3305 %{
 3306   predicate(n->get_int() == 65535);
 3307   match(ConI);
 3308 
 3309   format %{ %}
 3310   interface(CONST_INTER);
 3311 %}
 3312 
 3313 // Constant for byte-wide masking
 3314 operand immL_255()
 3315 %{
 3316   predicate(n->get_long() == 255);
 3317   match(ConL);
 3318 
 3319   format %{ %}
 3320   interface(CONST_INTER);
 3321 %}
 3322 
 3323 // Constant for short-wide masking
 3324 operand immL_65535()
 3325 %{
 3326   predicate(n->get_long() == 65535);
 3327   match(ConL);
 3328 
 3329   format %{ %}
 3330   interface(CONST_INTER);
 3331 %}
 3332 
 3333 operand kReg()
 3334 %{
 3335   constraint(ALLOC_IN_RC(vectmask_reg));
 3336   match(RegVectMask);
 3337   format %{%}
 3338   interface(REG_INTER);
 3339 %}
 3340 
 3341 operand kReg_K1()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K2()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 // Special Registers
 3358 operand kReg_K3()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K4()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K5()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K6()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 // Special Registers
 3391 operand kReg_K7()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 // Register Operands
 3400 // Integer Register
 3401 operand rRegI()
 3402 %{
 3403   constraint(ALLOC_IN_RC(int_reg));
 3404   match(RegI);
 3405 
 3406   match(rax_RegI);
 3407   match(rbx_RegI);
 3408   match(rcx_RegI);
 3409   match(rdx_RegI);
 3410   match(rdi_RegI);
 3411 
 3412   format %{ %}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Special Registers
 3417 operand rax_RegI()
 3418 %{
 3419   constraint(ALLOC_IN_RC(int_rax_reg));
 3420   match(RegI);
 3421   match(rRegI);
 3422 
 3423   format %{ "RAX" %}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand rbx_RegI()
 3429 %{
 3430   constraint(ALLOC_IN_RC(int_rbx_reg));
 3431   match(RegI);
 3432   match(rRegI);
 3433 
 3434   format %{ "RBX" %}
 3435   interface(REG_INTER);
 3436 %}
 3437 
 3438 operand rcx_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rcx_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RCX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 operand rdx_RegI()
 3449 %{
 3450   constraint(ALLOC_IN_RC(int_rdx_reg));
 3451   match(RegI);
 3452   match(rRegI);
 3453 
 3454   format %{ "RDX" %}
 3455   interface(REG_INTER);
 3456 %}
 3457 
 3458 operand rdi_RegI()
 3459 %{
 3460   constraint(ALLOC_IN_RC(int_rdi_reg));
 3461   match(RegI);
 3462   match(rRegI);
 3463 
 3464   format %{ "RDI" %}
 3465   interface(REG_INTER);
 3466 %}
 3467 
 3468 operand no_rax_rdx_RegI()
 3469 %{
 3470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3471   match(RegI);
 3472   match(rbx_RegI);
 3473   match(rcx_RegI);
 3474   match(rdi_RegI);
 3475 
 3476   format %{ %}
 3477   interface(REG_INTER);
 3478 %}
 3479 
 3480 operand no_rbp_r13_RegI()
 3481 %{
 3482   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3483   match(RegI);
 3484   match(rRegI);
 3485   match(rax_RegI);
 3486   match(rbx_RegI);
 3487   match(rcx_RegI);
 3488   match(rdx_RegI);
 3489   match(rdi_RegI);
 3490 
 3491   format %{ %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 // Pointer Register
 3496 operand any_RegP()
 3497 %{
 3498   constraint(ALLOC_IN_RC(any_reg));
 3499   match(RegP);
 3500   match(rax_RegP);
 3501   match(rbx_RegP);
 3502   match(rdi_RegP);
 3503   match(rsi_RegP);
 3504   match(rbp_RegP);
 3505   match(r15_RegP);
 3506   match(rRegP);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 operand rRegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_reg));
 3515   match(RegP);
 3516   match(rax_RegP);
 3517   match(rbx_RegP);
 3518   match(rdi_RegP);
 3519   match(rsi_RegP);
 3520   match(rbp_RegP);  // See Q&A below about
 3521   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3522 
 3523   format %{ %}
 3524   interface(REG_INTER);
 3525 %}
 3526 
 3527 operand rRegN() %{
 3528   constraint(ALLOC_IN_RC(int_reg));
 3529   match(RegN);
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3536 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3537 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3538 // The output of an instruction is controlled by the allocator, which respects
 3539 // register class masks, not match rules.  Unless an instruction mentions
 3540 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3541 // by the allocator as an input.
 3542 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3543 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3544 // result, RBP is not included in the output of the instruction either.
 3545 
 3546 operand no_rax_RegP()
 3547 %{
 3548   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3549   match(RegP);
 3550   match(rbx_RegP);
 3551   match(rsi_RegP);
 3552   match(rdi_RegP);
 3553 
 3554   format %{ %}
 3555   interface(REG_INTER);
 3556 %}
 3557 
 3558 // This operand is not allowed to use RBP even if
 3559 // RBP is not used to hold the frame pointer.
 3560 operand no_rbp_RegP()
 3561 %{
 3562   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3563   match(RegP);
 3564   match(rbx_RegP);
 3565   match(rsi_RegP);
 3566   match(rdi_RegP);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 operand no_rax_rbx_RegP()
 3573 %{
 3574   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3575   match(RegP);
 3576   match(rsi_RegP);
 3577   match(rdi_RegP);
 3578 
 3579   format %{ %}
 3580   interface(REG_INTER);
 3581 %}
 3582 
 3583 // Special Registers
 3584 // Return a pointer value
 3585 operand rax_RegP()
 3586 %{
 3587   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3588   match(RegP);
 3589   match(rRegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // Special Registers
 3596 // Return a compressed pointer value
 3597 operand rax_RegN()
 3598 %{
 3599   constraint(ALLOC_IN_RC(int_rax_reg));
 3600   match(RegN);
 3601   match(rRegN);
 3602 
 3603   format %{ %}
 3604   interface(REG_INTER);
 3605 %}
 3606 
 3607 // Used in AtomicAdd
 3608 operand rbx_RegP()
 3609 %{
 3610   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3611   match(RegP);
 3612   match(rRegP);
 3613 
 3614   format %{ %}
 3615   interface(REG_INTER);
 3616 %}
 3617 
 3618 operand rsi_RegP()
 3619 %{
 3620   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3621   match(RegP);
 3622   match(rRegP);
 3623 
 3624   format %{ %}
 3625   interface(REG_INTER);
 3626 %}
 3627 
 3628 operand rbp_RegP()
 3629 %{
 3630   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3631   match(RegP);
 3632   match(rRegP);
 3633 
 3634   format %{ %}
 3635   interface(REG_INTER);
 3636 %}
 3637 
 3638 // Used in rep stosq
 3639 operand rdi_RegP()
 3640 %{
 3641   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3642   match(RegP);
 3643   match(rRegP);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand r15_RegP()
 3650 %{
 3651   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3652   match(RegP);
 3653   match(rRegP);
 3654 
 3655   format %{ %}
 3656   interface(REG_INTER);
 3657 %}
 3658 
 3659 operand rRegL()
 3660 %{
 3661   constraint(ALLOC_IN_RC(long_reg));
 3662   match(RegL);
 3663   match(rax_RegL);
 3664   match(rdx_RegL);
 3665 
 3666   format %{ %}
 3667   interface(REG_INTER);
 3668 %}
 3669 
 3670 // Special Registers
 3671 operand no_rax_rdx_RegL()
 3672 %{
 3673   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3674   match(RegL);
 3675   match(rRegL);
 3676 
 3677   format %{ %}
 3678   interface(REG_INTER);
 3679 %}
 3680 
 3681 operand no_rax_RegL()
 3682 %{
 3683   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3684   match(RegL);
 3685   match(rRegL);
 3686   match(rdx_RegL);
 3687 
 3688   format %{ %}
 3689   interface(REG_INTER);
 3690 %}
 3691 
 3692 operand rax_RegL()
 3693 %{
 3694   constraint(ALLOC_IN_RC(long_rax_reg));
 3695   match(RegL);
 3696   match(rRegL);
 3697 
 3698   format %{ "RAX" %}
 3699   interface(REG_INTER);
 3700 %}
 3701 
 3702 operand rcx_RegL()
 3703 %{
 3704   constraint(ALLOC_IN_RC(long_rcx_reg));
 3705   match(RegL);
 3706   match(rRegL);
 3707 
 3708   format %{ %}
 3709   interface(REG_INTER);
 3710 %}
 3711 
 3712 operand rdx_RegL()
 3713 %{
 3714   constraint(ALLOC_IN_RC(long_rdx_reg));
 3715   match(RegL);
 3716   match(rRegL);
 3717 
 3718   format %{ %}
 3719   interface(REG_INTER);
 3720 %}
 3721 
 3722 operand no_rbp_r13_RegL()
 3723 %{
 3724   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3725   match(RegL);
 3726   match(rRegL);
 3727   match(rax_RegL);
 3728   match(rcx_RegL);
 3729   match(rdx_RegL);
 3730 
 3731   format %{ %}
 3732   interface(REG_INTER);
 3733 %}
 3734 
 3735 // Flags register, used as output of compare instructions
 3736 operand rFlagsReg()
 3737 %{
 3738   constraint(ALLOC_IN_RC(int_flags));
 3739   match(RegFlags);
 3740 
 3741   format %{ "RFLAGS" %}
 3742   interface(REG_INTER);
 3743 %}
 3744 
 3745 // Flags register, used as output of FLOATING POINT compare instructions
 3746 operand rFlagsRegU()
 3747 %{
 3748   constraint(ALLOC_IN_RC(int_flags));
 3749   match(RegFlags);
 3750 
 3751   format %{ "RFLAGS_U" %}
 3752   interface(REG_INTER);
 3753 %}
 3754 
 3755 operand rFlagsRegUCF() %{
 3756   constraint(ALLOC_IN_RC(int_flags));
 3757   match(RegFlags);
 3758   predicate(false);
 3759 
 3760   format %{ "RFLAGS_U_CF" %}
 3761   interface(REG_INTER);
 3762 %}
 3763 
 3764 // Float register operands
 3765 operand regF() %{
 3766    constraint(ALLOC_IN_RC(float_reg));
 3767    match(RegF);
 3768 
 3769    format %{ %}
 3770    interface(REG_INTER);
 3771 %}
 3772 
 3773 // Float register operands
 3774 operand legRegF() %{
 3775    constraint(ALLOC_IN_RC(float_reg_legacy));
 3776    match(RegF);
 3777 
 3778    format %{ %}
 3779    interface(REG_INTER);
 3780 %}
 3781 
 3782 // Float register operands
 3783 operand vlRegF() %{
 3784    constraint(ALLOC_IN_RC(float_reg_vl));
 3785    match(RegF);
 3786 
 3787    format %{ %}
 3788    interface(REG_INTER);
 3789 %}
 3790 
 3791 // Double register operands
 3792 operand regD() %{
 3793    constraint(ALLOC_IN_RC(double_reg));
 3794    match(RegD);
 3795 
 3796    format %{ %}
 3797    interface(REG_INTER);
 3798 %}
 3799 
 3800 // Double register operands
 3801 operand legRegD() %{
 3802    constraint(ALLOC_IN_RC(double_reg_legacy));
 3803    match(RegD);
 3804 
 3805    format %{ %}
 3806    interface(REG_INTER);
 3807 %}
 3808 
 3809 // Double register operands
 3810 operand vlRegD() %{
 3811    constraint(ALLOC_IN_RC(double_reg_vl));
 3812    match(RegD);
 3813 
 3814    format %{ %}
 3815    interface(REG_INTER);
 3816 %}
 3817 
 3818 //----------Memory Operands----------------------------------------------------
 3819 // Direct Memory Operand
 3820 // operand direct(immP addr)
 3821 // %{
 3822 //   match(addr);
 3823 
 3824 //   format %{ "[$addr]" %}
 3825 //   interface(MEMORY_INTER) %{
 3826 //     base(0xFFFFFFFF);
 3827 //     index(0x4);
 3828 //     scale(0x0);
 3829 //     disp($addr);
 3830 //   %}
 3831 // %}
 3832 
 3833 // Indirect Memory Operand
 3834 operand indirect(any_RegP reg)
 3835 %{
 3836   constraint(ALLOC_IN_RC(ptr_reg));
 3837   match(reg);
 3838 
 3839   format %{ "[$reg]" %}
 3840   interface(MEMORY_INTER) %{
 3841     base($reg);
 3842     index(0x4);
 3843     scale(0x0);
 3844     disp(0x0);
 3845   %}
 3846 %}
 3847 
 3848 // Indirect Memory Plus Short Offset Operand
 3849 operand indOffset8(any_RegP reg, immL8 off)
 3850 %{
 3851   constraint(ALLOC_IN_RC(ptr_reg));
 3852   match(AddP reg off);
 3853 
 3854   format %{ "[$reg + $off (8-bit)]" %}
 3855   interface(MEMORY_INTER) %{
 3856     base($reg);
 3857     index(0x4);
 3858     scale(0x0);
 3859     disp($off);
 3860   %}
 3861 %}
 3862 
 3863 // Indirect Memory Plus Long Offset Operand
 3864 operand indOffset32(any_RegP reg, immL32 off)
 3865 %{
 3866   constraint(ALLOC_IN_RC(ptr_reg));
 3867   match(AddP reg off);
 3868 
 3869   format %{ "[$reg + $off (32-bit)]" %}
 3870   interface(MEMORY_INTER) %{
 3871     base($reg);
 3872     index(0x4);
 3873     scale(0x0);
 3874     disp($off);
 3875   %}
 3876 %}
 3877 
 3878 // Indirect Memory Plus Index Register Plus Offset Operand
 3879 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3880 %{
 3881   constraint(ALLOC_IN_RC(ptr_reg));
 3882   match(AddP (AddP reg lreg) off);
 3883 
 3884   op_cost(10);
 3885   format %{"[$reg + $off + $lreg]" %}
 3886   interface(MEMORY_INTER) %{
 3887     base($reg);
 3888     index($lreg);
 3889     scale(0x0);
 3890     disp($off);
 3891   %}
 3892 %}
 3893 
 3894 // Indirect Memory Plus Index Register Plus Offset Operand
 3895 operand indIndex(any_RegP reg, rRegL lreg)
 3896 %{
 3897   constraint(ALLOC_IN_RC(ptr_reg));
 3898   match(AddP reg lreg);
 3899 
 3900   op_cost(10);
 3901   format %{"[$reg + $lreg]" %}
 3902   interface(MEMORY_INTER) %{
 3903     base($reg);
 3904     index($lreg);
 3905     scale(0x0);
 3906     disp(0x0);
 3907   %}
 3908 %}
 3909 
 3910 // Indirect Memory Times Scale Plus Index Register
 3911 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3912 %{
 3913   constraint(ALLOC_IN_RC(ptr_reg));
 3914   match(AddP reg (LShiftL lreg scale));
 3915 
 3916   op_cost(10);
 3917   format %{"[$reg + $lreg << $scale]" %}
 3918   interface(MEMORY_INTER) %{
 3919     base($reg);
 3920     index($lreg);
 3921     scale($scale);
 3922     disp(0x0);
 3923   %}
 3924 %}
 3925 
 3926 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3927 %{
 3928   constraint(ALLOC_IN_RC(ptr_reg));
 3929   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3930   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3931 
 3932   op_cost(10);
 3933   format %{"[$reg + pos $idx << $scale]" %}
 3934   interface(MEMORY_INTER) %{
 3935     base($reg);
 3936     index($idx);
 3937     scale($scale);
 3938     disp(0x0);
 3939   %}
 3940 %}
 3941 
 3942 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3943 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3944 %{
 3945   constraint(ALLOC_IN_RC(ptr_reg));
 3946   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3947 
 3948   op_cost(10);
 3949   format %{"[$reg + $off + $lreg << $scale]" %}
 3950   interface(MEMORY_INTER) %{
 3951     base($reg);
 3952     index($lreg);
 3953     scale($scale);
 3954     disp($off);
 3955   %}
 3956 %}
 3957 
 3958 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3959 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3960 %{
 3961   constraint(ALLOC_IN_RC(ptr_reg));
 3962   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3963   match(AddP (AddP reg (ConvI2L idx)) off);
 3964 
 3965   op_cost(10);
 3966   format %{"[$reg + $off + $idx]" %}
 3967   interface(MEMORY_INTER) %{
 3968     base($reg);
 3969     index($idx);
 3970     scale(0x0);
 3971     disp($off);
 3972   %}
 3973 %}
 3974 
 3975 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3976 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3977 %{
 3978   constraint(ALLOC_IN_RC(ptr_reg));
 3979   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3980   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3981 
 3982   op_cost(10);
 3983   format %{"[$reg + $off + $idx << $scale]" %}
 3984   interface(MEMORY_INTER) %{
 3985     base($reg);
 3986     index($idx);
 3987     scale($scale);
 3988     disp($off);
 3989   %}
 3990 %}
 3991 
 3992 // Indirect Narrow Oop Plus Offset Operand
 3993 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3994 // we can't free r12 even with CompressedOops::base() == NULL.
 3995 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3996   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3997   constraint(ALLOC_IN_RC(ptr_reg));
 3998   match(AddP (DecodeN reg) off);
 3999 
 4000   op_cost(10);
 4001   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 4002   interface(MEMORY_INTER) %{
 4003     base(0xc); // R12
 4004     index($reg);
 4005     scale(0x3);
 4006     disp($off);
 4007   %}
 4008 %}
 4009 
 4010 // Indirect Memory Operand
 4011 operand indirectNarrow(rRegN reg)
 4012 %{
 4013   predicate(CompressedOops::shift() == 0);
 4014   constraint(ALLOC_IN_RC(ptr_reg));
 4015   match(DecodeN reg);
 4016 
 4017   format %{ "[$reg]" %}
 4018   interface(MEMORY_INTER) %{
 4019     base($reg);
 4020     index(0x4);
 4021     scale(0x0);
 4022     disp(0x0);
 4023   %}
 4024 %}
 4025 
 4026 // Indirect Memory Plus Short Offset Operand
 4027 operand indOffset8Narrow(rRegN reg, immL8 off)
 4028 %{
 4029   predicate(CompressedOops::shift() == 0);
 4030   constraint(ALLOC_IN_RC(ptr_reg));
 4031   match(AddP (DecodeN reg) off);
 4032 
 4033   format %{ "[$reg + $off (8-bit)]" %}
 4034   interface(MEMORY_INTER) %{
 4035     base($reg);
 4036     index(0x4);
 4037     scale(0x0);
 4038     disp($off);
 4039   %}
 4040 %}
 4041 
 4042 // Indirect Memory Plus Long Offset Operand
 4043 operand indOffset32Narrow(rRegN reg, immL32 off)
 4044 %{
 4045   predicate(CompressedOops::shift() == 0);
 4046   constraint(ALLOC_IN_RC(ptr_reg));
 4047   match(AddP (DecodeN reg) off);
 4048 
 4049   format %{ "[$reg + $off (32-bit)]" %}
 4050   interface(MEMORY_INTER) %{
 4051     base($reg);
 4052     index(0x4);
 4053     scale(0x0);
 4054     disp($off);
 4055   %}
 4056 %}
 4057 
 4058 // Indirect Memory Plus Index Register Plus Offset Operand
 4059 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4060 %{
 4061   predicate(CompressedOops::shift() == 0);
 4062   constraint(ALLOC_IN_RC(ptr_reg));
 4063   match(AddP (AddP (DecodeN reg) lreg) off);
 4064 
 4065   op_cost(10);
 4066   format %{"[$reg + $off + $lreg]" %}
 4067   interface(MEMORY_INTER) %{
 4068     base($reg);
 4069     index($lreg);
 4070     scale(0x0);
 4071     disp($off);
 4072   %}
 4073 %}
 4074 
 4075 // Indirect Memory Plus Index Register Plus Offset Operand
 4076 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4077 %{
 4078   predicate(CompressedOops::shift() == 0);
 4079   constraint(ALLOC_IN_RC(ptr_reg));
 4080   match(AddP (DecodeN reg) lreg);
 4081 
 4082   op_cost(10);
 4083   format %{"[$reg + $lreg]" %}
 4084   interface(MEMORY_INTER) %{
 4085     base($reg);
 4086     index($lreg);
 4087     scale(0x0);
 4088     disp(0x0);
 4089   %}
 4090 %}
 4091 
 4092 // Indirect Memory Times Scale Plus Index Register
 4093 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4094 %{
 4095   predicate(CompressedOops::shift() == 0);
 4096   constraint(ALLOC_IN_RC(ptr_reg));
 4097   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4098 
 4099   op_cost(10);
 4100   format %{"[$reg + $lreg << $scale]" %}
 4101   interface(MEMORY_INTER) %{
 4102     base($reg);
 4103     index($lreg);
 4104     scale($scale);
 4105     disp(0x0);
 4106   %}
 4107 %}
 4108 
 4109 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4110 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4111 %{
 4112   predicate(CompressedOops::shift() == 0);
 4113   constraint(ALLOC_IN_RC(ptr_reg));
 4114   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4115 
 4116   op_cost(10);
 4117   format %{"[$reg + $off + $lreg << $scale]" %}
 4118   interface(MEMORY_INTER) %{
 4119     base($reg);
 4120     index($lreg);
 4121     scale($scale);
 4122     disp($off);
 4123   %}
 4124 %}
 4125 
 4126 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4127 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4128 %{
 4129   constraint(ALLOC_IN_RC(ptr_reg));
 4130   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4131   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4132 
 4133   op_cost(10);
 4134   format %{"[$reg + $off + $idx]" %}
 4135   interface(MEMORY_INTER) %{
 4136     base($reg);
 4137     index($idx);
 4138     scale(0x0);
 4139     disp($off);
 4140   %}
 4141 %}
 4142 
 4143 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4144 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4145 %{
 4146   constraint(ALLOC_IN_RC(ptr_reg));
 4147   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4148   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4149 
 4150   op_cost(10);
 4151   format %{"[$reg + $off + $idx << $scale]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base($reg);
 4154     index($idx);
 4155     scale($scale);
 4156     disp($off);
 4157   %}
 4158 %}
 4159 
 4160 //----------Special Memory Operands--------------------------------------------
 4161 // Stack Slot Operand - This operand is used for loading and storing temporary
 4162 //                      values on the stack where a match requires a value to
 4163 //                      flow through memory.
 4164 operand stackSlotP(sRegP reg)
 4165 %{
 4166   constraint(ALLOC_IN_RC(stack_slots));
 4167   // No match rule because this operand is only generated in matching
 4168 
 4169   format %{ "[$reg]" %}
 4170   interface(MEMORY_INTER) %{
 4171     base(0x4);   // RSP
 4172     index(0x4);  // No Index
 4173     scale(0x0);  // No Scale
 4174     disp($reg);  // Stack Offset
 4175   %}
 4176 %}
 4177 
 4178 operand stackSlotI(sRegI reg)
 4179 %{
 4180   constraint(ALLOC_IN_RC(stack_slots));
 4181   // No match rule because this operand is only generated in matching
 4182 
 4183   format %{ "[$reg]" %}
 4184   interface(MEMORY_INTER) %{
 4185     base(0x4);   // RSP
 4186     index(0x4);  // No Index
 4187     scale(0x0);  // No Scale
 4188     disp($reg);  // Stack Offset
 4189   %}
 4190 %}
 4191 
 4192 operand stackSlotF(sRegF reg)
 4193 %{
 4194   constraint(ALLOC_IN_RC(stack_slots));
 4195   // No match rule because this operand is only generated in matching
 4196 
 4197   format %{ "[$reg]" %}
 4198   interface(MEMORY_INTER) %{
 4199     base(0x4);   // RSP
 4200     index(0x4);  // No Index
 4201     scale(0x0);  // No Scale
 4202     disp($reg);  // Stack Offset
 4203   %}
 4204 %}
 4205 
 4206 operand stackSlotD(sRegD reg)
 4207 %{
 4208   constraint(ALLOC_IN_RC(stack_slots));
 4209   // No match rule because this operand is only generated in matching
 4210 
 4211   format %{ "[$reg]" %}
 4212   interface(MEMORY_INTER) %{
 4213     base(0x4);   // RSP
 4214     index(0x4);  // No Index
 4215     scale(0x0);  // No Scale
 4216     disp($reg);  // Stack Offset
 4217   %}
 4218 %}
 4219 operand stackSlotL(sRegL reg)
 4220 %{
 4221   constraint(ALLOC_IN_RC(stack_slots));
 4222   // No match rule because this operand is only generated in matching
 4223 
 4224   format %{ "[$reg]" %}
 4225   interface(MEMORY_INTER) %{
 4226     base(0x4);   // RSP
 4227     index(0x4);  // No Index
 4228     scale(0x0);  // No Scale
 4229     disp($reg);  // Stack Offset
 4230   %}
 4231 %}
 4232 
 4233 //----------Conditional Branch Operands----------------------------------------
 4234 // Comparison Op  - This is the operation of the comparison, and is limited to
 4235 //                  the following set of codes:
 4236 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4237 //
 4238 // Other attributes of the comparison, such as unsignedness, are specified
 4239 // by the comparison instruction that sets a condition code flags register.
 4240 // That result is represented by a flags operand whose subtype is appropriate
 4241 // to the unsignedness (etc.) of the comparison.
 4242 //
 4243 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4244 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4245 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4246 
 4247 // Comparison Code
 4248 operand cmpOp()
 4249 %{
 4250   match(Bool);
 4251 
 4252   format %{ "" %}
 4253   interface(COND_INTER) %{
 4254     equal(0x4, "e");
 4255     not_equal(0x5, "ne");
 4256     less(0xC, "l");
 4257     greater_equal(0xD, "ge");
 4258     less_equal(0xE, "le");
 4259     greater(0xF, "g");
 4260     overflow(0x0, "o");
 4261     no_overflow(0x1, "no");
 4262   %}
 4263 %}
 4264 
 4265 // Comparison Code, unsigned compare.  Used by FP also, with
 4266 // C2 (unordered) turned into GT or LT already.  The other bits
 4267 // C0 and C3 are turned into Carry & Zero flags.
 4268 operand cmpOpU()
 4269 %{
 4270   match(Bool);
 4271 
 4272   format %{ "" %}
 4273   interface(COND_INTER) %{
 4274     equal(0x4, "e");
 4275     not_equal(0x5, "ne");
 4276     less(0x2, "b");
 4277     greater_equal(0x3, "ae");
 4278     less_equal(0x6, "be");
 4279     greater(0x7, "a");
 4280     overflow(0x0, "o");
 4281     no_overflow(0x1, "no");
 4282   %}
 4283 %}
 4284 
 4285 
 4286 // Floating comparisons that don't require any fixup for the unordered case,
 4287 // If both inputs of the comparison are the same, ZF is always set so we
 4288 // don't need to use cmpOpUCF2 for eq/ne
 4289 operand cmpOpUCF() %{
 4290   match(Bool);
 4291   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4292             n->as_Bool()->_test._test == BoolTest::ge ||
 4293             n->as_Bool()->_test._test == BoolTest::le ||
 4294             n->as_Bool()->_test._test == BoolTest::gt ||
 4295             n->in(1)->in(1) == n->in(1)->in(2));
 4296   format %{ "" %}
 4297   interface(COND_INTER) %{
 4298     equal(0xb, "np");
 4299     not_equal(0xa, "p");
 4300     less(0x2, "b");
 4301     greater_equal(0x3, "ae");
 4302     less_equal(0x6, "be");
 4303     greater(0x7, "a");
 4304     overflow(0x0, "o");
 4305     no_overflow(0x1, "no");
 4306   %}
 4307 %}
 4308 
 4309 
 4310 // Floating comparisons that can be fixed up with extra conditional jumps
 4311 operand cmpOpUCF2() %{
 4312   match(Bool);
 4313   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4314              n->as_Bool()->_test._test == BoolTest::eq) &&
 4315             n->in(1)->in(1) != n->in(1)->in(2));
 4316   format %{ "" %}
 4317   interface(COND_INTER) %{
 4318     equal(0x4, "e");
 4319     not_equal(0x5, "ne");
 4320     less(0x2, "b");
 4321     greater_equal(0x3, "ae");
 4322     less_equal(0x6, "be");
 4323     greater(0x7, "a");
 4324     overflow(0x0, "o");
 4325     no_overflow(0x1, "no");
 4326   %}
 4327 %}
 4328 
 4329 //----------OPERAND CLASSES----------------------------------------------------
 4330 // Operand Classes are groups of operands that are used as to simplify
 4331 // instruction definitions by not requiring the AD writer to specify separate
 4332 // instructions for every form of operand when the instruction accepts
 4333 // multiple operand types with the same basic encoding and format.  The classic
 4334 // case of this is memory operands.
 4335 
 4336 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4337                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4338                indCompressedOopOffset,
 4339                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4340                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4341                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4342 
 4343 //----------PIPELINE-----------------------------------------------------------
 4344 // Rules which define the behavior of the target architectures pipeline.
 4345 pipeline %{
 4346 
 4347 //----------ATTRIBUTES---------------------------------------------------------
 4348 attributes %{
 4349   variable_size_instructions;        // Fixed size instructions
 4350   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4351   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4352   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4353   instruction_fetch_units = 1;       // of 16 bytes
 4354 
 4355   // List of nop instructions
 4356   nops( MachNop );
 4357 %}
 4358 
 4359 //----------RESOURCES----------------------------------------------------------
 4360 // Resources are the functional units available to the machine
 4361 
 4362 // Generic P2/P3 pipeline
 4363 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4364 // 3 instructions decoded per cycle.
 4365 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4366 // 3 ALU op, only ALU0 handles mul instructions.
 4367 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4368            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4369            BR, FPU,
 4370            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4371 
 4372 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4373 // Pipeline Description specifies the stages in the machine's pipeline
 4374 
 4375 // Generic P2/P3 pipeline
 4376 pipe_desc(S0, S1, S2, S3, S4, S5);
 4377 
 4378 //----------PIPELINE CLASSES---------------------------------------------------
 4379 // Pipeline Classes describe the stages in which input and output are
 4380 // referenced by the hardware pipeline.
 4381 
 4382 // Naming convention: ialu or fpu
 4383 // Then: _reg
 4384 // Then: _reg if there is a 2nd register
 4385 // Then: _long if it's a pair of instructions implementing a long
 4386 // Then: _fat if it requires the big decoder
 4387 //   Or: _mem if it requires the big decoder and a memory unit.
 4388 
 4389 // Integer ALU reg operation
 4390 pipe_class ialu_reg(rRegI dst)
 4391 %{
 4392     single_instruction;
 4393     dst    : S4(write);
 4394     dst    : S3(read);
 4395     DECODE : S0;        // any decoder
 4396     ALU    : S3;        // any alu
 4397 %}
 4398 
 4399 // Long ALU reg operation
 4400 pipe_class ialu_reg_long(rRegL dst)
 4401 %{
 4402     instruction_count(2);
 4403     dst    : S4(write);
 4404     dst    : S3(read);
 4405     DECODE : S0(2);     // any 2 decoders
 4406     ALU    : S3(2);     // both alus
 4407 %}
 4408 
 4409 // Integer ALU reg operation using big decoder
 4410 pipe_class ialu_reg_fat(rRegI dst)
 4411 %{
 4412     single_instruction;
 4413     dst    : S4(write);
 4414     dst    : S3(read);
 4415     D0     : S0;        // big decoder only
 4416     ALU    : S3;        // any alu
 4417 %}
 4418 
 4419 // Integer ALU reg-reg operation
 4420 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4421 %{
 4422     single_instruction;
 4423     dst    : S4(write);
 4424     src    : S3(read);
 4425     DECODE : S0;        // any decoder
 4426     ALU    : S3;        // any alu
 4427 %}
 4428 
 4429 // Integer ALU reg-reg operation
 4430 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4431 %{
 4432     single_instruction;
 4433     dst    : S4(write);
 4434     src    : S3(read);
 4435     D0     : S0;        // big decoder only
 4436     ALU    : S3;        // any alu
 4437 %}
 4438 
 4439 // Integer ALU reg-mem operation
 4440 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4441 %{
 4442     single_instruction;
 4443     dst    : S5(write);
 4444     mem    : S3(read);
 4445     D0     : S0;        // big decoder only
 4446     ALU    : S4;        // any alu
 4447     MEM    : S3;        // any mem
 4448 %}
 4449 
 4450 // Integer mem operation (prefetch)
 4451 pipe_class ialu_mem(memory mem)
 4452 %{
 4453     single_instruction;
 4454     mem    : S3(read);
 4455     D0     : S0;        // big decoder only
 4456     MEM    : S3;        // any mem
 4457 %}
 4458 
 4459 // Integer Store to Memory
 4460 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4461 %{
 4462     single_instruction;
 4463     mem    : S3(read);
 4464     src    : S5(read);
 4465     D0     : S0;        // big decoder only
 4466     ALU    : S4;        // any alu
 4467     MEM    : S3;
 4468 %}
 4469 
 4470 // // Long Store to Memory
 4471 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4472 // %{
 4473 //     instruction_count(2);
 4474 //     mem    : S3(read);
 4475 //     src    : S5(read);
 4476 //     D0     : S0(2);          // big decoder only; twice
 4477 //     ALU    : S4(2);     // any 2 alus
 4478 //     MEM    : S3(2);  // Both mems
 4479 // %}
 4480 
 4481 // Integer Store to Memory
 4482 pipe_class ialu_mem_imm(memory mem)
 4483 %{
 4484     single_instruction;
 4485     mem    : S3(read);
 4486     D0     : S0;        // big decoder only
 4487     ALU    : S4;        // any alu
 4488     MEM    : S3;
 4489 %}
 4490 
 4491 // Integer ALU0 reg-reg operation
 4492 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4493 %{
 4494     single_instruction;
 4495     dst    : S4(write);
 4496     src    : S3(read);
 4497     D0     : S0;        // Big decoder only
 4498     ALU0   : S3;        // only alu0
 4499 %}
 4500 
 4501 // Integer ALU0 reg-mem operation
 4502 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4503 %{
 4504     single_instruction;
 4505     dst    : S5(write);
 4506     mem    : S3(read);
 4507     D0     : S0;        // big decoder only
 4508     ALU0   : S4;        // ALU0 only
 4509     MEM    : S3;        // any mem
 4510 %}
 4511 
 4512 // Integer ALU reg-reg operation
 4513 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     src2   : S3(read);
 4519     DECODE : S0;        // any decoder
 4520     ALU    : S3;        // any alu
 4521 %}
 4522 
 4523 // Integer ALU reg-imm operation
 4524 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4525 %{
 4526     single_instruction;
 4527     cr     : S4(write);
 4528     src1   : S3(read);
 4529     DECODE : S0;        // any decoder
 4530     ALU    : S3;        // any alu
 4531 %}
 4532 
 4533 // Integer ALU reg-mem operation
 4534 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4535 %{
 4536     single_instruction;
 4537     cr     : S4(write);
 4538     src1   : S3(read);
 4539     src2   : S3(read);
 4540     D0     : S0;        // big decoder only
 4541     ALU    : S4;        // any alu
 4542     MEM    : S3;
 4543 %}
 4544 
 4545 // Conditional move reg-reg
 4546 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4547 %{
 4548     instruction_count(4);
 4549     y      : S4(read);
 4550     q      : S3(read);
 4551     p      : S3(read);
 4552     DECODE : S0(4);     // any decoder
 4553 %}
 4554 
 4555 // Conditional move reg-reg
 4556 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4557 %{
 4558     single_instruction;
 4559     dst    : S4(write);
 4560     src    : S3(read);
 4561     cr     : S3(read);
 4562     DECODE : S0;        // any decoder
 4563 %}
 4564 
 4565 // Conditional move reg-mem
 4566 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0;        // any decoder
 4573     MEM    : S3;
 4574 %}
 4575 
 4576 // Conditional move reg-reg long
 4577 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4578 %{
 4579     single_instruction;
 4580     dst    : S4(write);
 4581     src    : S3(read);
 4582     cr     : S3(read);
 4583     DECODE : S0(2);     // any 2 decoders
 4584 %}
 4585 
 4586 // XXX
 4587 // // Conditional move double reg-reg
 4588 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4589 // %{
 4590 //     single_instruction;
 4591 //     dst    : S4(write);
 4592 //     src    : S3(read);
 4593 //     cr     : S3(read);
 4594 //     DECODE : S0;     // any decoder
 4595 // %}
 4596 
 4597 // Float reg-reg operation
 4598 pipe_class fpu_reg(regD dst)
 4599 %{
 4600     instruction_count(2);
 4601     dst    : S3(read);
 4602     DECODE : S0(2);     // any 2 decoders
 4603     FPU    : S3;
 4604 %}
 4605 
 4606 // Float reg-reg operation
 4607 pipe_class fpu_reg_reg(regD dst, regD src)
 4608 %{
 4609     instruction_count(2);
 4610     dst    : S4(write);
 4611     src    : S3(read);
 4612     DECODE : S0(2);     // any 2 decoders
 4613     FPU    : S3;
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4618 %{
 4619     instruction_count(3);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     DECODE : S0(3);     // any 3 decoders
 4624     FPU    : S3(2);
 4625 %}
 4626 
 4627 // Float reg-reg operation
 4628 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4629 %{
 4630     instruction_count(4);
 4631     dst    : S4(write);
 4632     src1   : S3(read);
 4633     src2   : S3(read);
 4634     src3   : S3(read);
 4635     DECODE : S0(4);     // any 3 decoders
 4636     FPU    : S3(2);
 4637 %}
 4638 
 4639 // Float reg-reg operation
 4640 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4641 %{
 4642     instruction_count(4);
 4643     dst    : S4(write);
 4644     src1   : S3(read);
 4645     src2   : S3(read);
 4646     src3   : S3(read);
 4647     DECODE : S1(3);     // any 3 decoders
 4648     D0     : S0;        // Big decoder only
 4649     FPU    : S3(2);
 4650     MEM    : S3;
 4651 %}
 4652 
 4653 // Float reg-mem operation
 4654 pipe_class fpu_reg_mem(regD dst, memory mem)
 4655 %{
 4656     instruction_count(2);
 4657     dst    : S5(write);
 4658     mem    : S3(read);
 4659     D0     : S0;        // big decoder only
 4660     DECODE : S1;        // any decoder for FPU POP
 4661     FPU    : S4;
 4662     MEM    : S3;        // any mem
 4663 %}
 4664 
 4665 // Float reg-mem operation
 4666 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4667 %{
 4668     instruction_count(3);
 4669     dst    : S5(write);
 4670     src1   : S3(read);
 4671     mem    : S3(read);
 4672     D0     : S0;        // big decoder only
 4673     DECODE : S1(2);     // any decoder for FPU POP
 4674     FPU    : S4;
 4675     MEM    : S3;        // any mem
 4676 %}
 4677 
 4678 // Float mem-reg operation
 4679 pipe_class fpu_mem_reg(memory mem, regD src)
 4680 %{
 4681     instruction_count(2);
 4682     src    : S5(read);
 4683     mem    : S3(read);
 4684     DECODE : S0;        // any decoder for FPU PUSH
 4685     D0     : S1;        // big decoder only
 4686     FPU    : S4;
 4687     MEM    : S3;        // any mem
 4688 %}
 4689 
 4690 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4691 %{
 4692     instruction_count(3);
 4693     src1   : S3(read);
 4694     src2   : S3(read);
 4695     mem    : S3(read);
 4696     DECODE : S0(2);     // any decoder for FPU PUSH
 4697     D0     : S1;        // big decoder only
 4698     FPU    : S4;
 4699     MEM    : S3;        // any mem
 4700 %}
 4701 
 4702 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4703 %{
 4704     instruction_count(3);
 4705     src1   : S3(read);
 4706     src2   : S3(read);
 4707     mem    : S4(read);
 4708     DECODE : S0;        // any decoder for FPU PUSH
 4709     D0     : S0(2);     // big decoder only
 4710     FPU    : S4;
 4711     MEM    : S3(2);     // any mem
 4712 %}
 4713 
 4714 pipe_class fpu_mem_mem(memory dst, memory src1)
 4715 %{
 4716     instruction_count(2);
 4717     src1   : S3(read);
 4718     dst    : S4(read);
 4719     D0     : S0(2);     // big decoder only
 4720     MEM    : S3(2);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S3(read);
 4727     src2   : S3(read);
 4728     dst    : S4(read);
 4729     D0     : S0(3);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(3);     // any mem
 4732 %}
 4733 
 4734 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4735 %{
 4736     instruction_count(3);
 4737     src1   : S4(read);
 4738     mem    : S4(read);
 4739     DECODE : S0;        // any decoder for FPU PUSH
 4740     D0     : S0(2);     // big decoder only
 4741     FPU    : S4;
 4742     MEM    : S3(2);     // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_con(regD dst)
 4747 %{
 4748     instruction_count(2);
 4749     dst    : S5(write);
 4750     D0     : S0;        // big decoder only for the load
 4751     DECODE : S1;        // any decoder for FPU POP
 4752     FPU    : S4;
 4753     MEM    : S3;        // any mem
 4754 %}
 4755 
 4756 // Float load constant
 4757 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4758 %{
 4759     instruction_count(3);
 4760     dst    : S5(write);
 4761     src    : S3(read);
 4762     D0     : S0;        // big decoder only for the load
 4763     DECODE : S1(2);     // any decoder for FPU POP
 4764     FPU    : S4;
 4765     MEM    : S3;        // any mem
 4766 %}
 4767 
 4768 // UnConditional branch
 4769 pipe_class pipe_jmp(label labl)
 4770 %{
 4771     single_instruction;
 4772     BR   : S3;
 4773 %}
 4774 
 4775 // Conditional branch
 4776 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4777 %{
 4778     single_instruction;
 4779     cr    : S1(read);
 4780     BR    : S3;
 4781 %}
 4782 
 4783 // Allocation idiom
 4784 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4785 %{
 4786     instruction_count(1); force_serialization;
 4787     fixed_latency(6);
 4788     heap_ptr : S3(read);
 4789     DECODE   : S0(3);
 4790     D0       : S2;
 4791     MEM      : S3;
 4792     ALU      : S3(2);
 4793     dst      : S5(write);
 4794     BR       : S5;
 4795 %}
 4796 
 4797 // Generic big/slow expanded idiom
 4798 pipe_class pipe_slow()
 4799 %{
 4800     instruction_count(10); multiple_bundles; force_serialization;
 4801     fixed_latency(100);
 4802     D0  : S0(2);
 4803     MEM : S3(2);
 4804 %}
 4805 
 4806 // The real do-nothing guy
 4807 pipe_class empty()
 4808 %{
 4809     instruction_count(0);
 4810 %}
 4811 
 4812 // Define the class for the Nop node
 4813 define
 4814 %{
 4815    MachNop = empty;
 4816 %}
 4817 
 4818 %}
 4819 
 4820 //----------INSTRUCTIONS-------------------------------------------------------
 4821 //
 4822 // match      -- States which machine-independent subtree may be replaced
 4823 //               by this instruction.
 4824 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4825 //               selection to identify a minimum cost tree of machine
 4826 //               instructions that matches a tree of machine-independent
 4827 //               instructions.
 4828 // format     -- A string providing the disassembly for this instruction.
 4829 //               The value of an instruction's operand may be inserted
 4830 //               by referring to it with a '$' prefix.
 4831 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4832 //               to within an encode class as $primary, $secondary, and $tertiary
 4833 //               rrspectively.  The primary opcode is commonly used to
 4834 //               indicate the type of machine instruction, while secondary
 4835 //               and tertiary are often used for prefix options or addressing
 4836 //               modes.
 4837 // ins_encode -- A list of encode classes with parameters. The encode class
 4838 //               name must have been defined in an 'enc_class' specification
 4839 //               in the encode section of the architecture description.
 4840 
 4841 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4842 // Load Float
 4843 instruct MoveF2VL(vlRegF dst, regF src) %{
 4844   match(Set dst src);
 4845   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4846   ins_encode %{
 4847     ShouldNotReachHere();
 4848   %}
 4849   ins_pipe( fpu_reg_reg );
 4850 %}
 4851 
 4852 // Load Float
 4853 instruct MoveF2LEG(legRegF dst, regF src) %{
 4854   match(Set dst src);
 4855   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4856   ins_encode %{
 4857     ShouldNotReachHere();
 4858   %}
 4859   ins_pipe( fpu_reg_reg );
 4860 %}
 4861 
 4862 // Load Float
 4863 instruct MoveVL2F(regF dst, vlRegF src) %{
 4864   match(Set dst src);
 4865   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4866   ins_encode %{
 4867     ShouldNotReachHere();
 4868   %}
 4869   ins_pipe( fpu_reg_reg );
 4870 %}
 4871 
 4872 // Load Float
 4873 instruct MoveLEG2F(regF dst, legRegF src) %{
 4874   match(Set dst src);
 4875   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4876   ins_encode %{
 4877     ShouldNotReachHere();
 4878   %}
 4879   ins_pipe( fpu_reg_reg );
 4880 %}
 4881 
 4882 // Load Double
 4883 instruct MoveD2VL(vlRegD dst, regD src) %{
 4884   match(Set dst src);
 4885   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4886   ins_encode %{
 4887     ShouldNotReachHere();
 4888   %}
 4889   ins_pipe( fpu_reg_reg );
 4890 %}
 4891 
 4892 // Load Double
 4893 instruct MoveD2LEG(legRegD dst, regD src) %{
 4894   match(Set dst src);
 4895   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4896   ins_encode %{
 4897     ShouldNotReachHere();
 4898   %}
 4899   ins_pipe( fpu_reg_reg );
 4900 %}
 4901 
 4902 // Load Double
 4903 instruct MoveVL2D(regD dst, vlRegD src) %{
 4904   match(Set dst src);
 4905   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4906   ins_encode %{
 4907     ShouldNotReachHere();
 4908   %}
 4909   ins_pipe( fpu_reg_reg );
 4910 %}
 4911 
 4912 // Load Double
 4913 instruct MoveLEG2D(regD dst, legRegD src) %{
 4914   match(Set dst src);
 4915   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4916   ins_encode %{
 4917     ShouldNotReachHere();
 4918   %}
 4919   ins_pipe( fpu_reg_reg );
 4920 %}
 4921 
 4922 //----------Load/Store/Move Instructions---------------------------------------
 4923 //----------Load Instructions--------------------------------------------------
 4924 
 4925 // Load Byte (8 bit signed)
 4926 instruct loadB(rRegI dst, memory mem)
 4927 %{
 4928   match(Set dst (LoadB mem));
 4929 
 4930   ins_cost(125);
 4931   format %{ "movsbl  $dst, $mem\t# byte" %}
 4932 
 4933   ins_encode %{
 4934     __ movsbl($dst$$Register, $mem$$Address);
 4935   %}
 4936 
 4937   ins_pipe(ialu_reg_mem);
 4938 %}
 4939 
 4940 // Load Byte (8 bit signed) into Long Register
 4941 instruct loadB2L(rRegL dst, memory mem)
 4942 %{
 4943   match(Set dst (ConvI2L (LoadB mem)));
 4944 
 4945   ins_cost(125);
 4946   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4947 
 4948   ins_encode %{
 4949     __ movsbq($dst$$Register, $mem$$Address);
 4950   %}
 4951 
 4952   ins_pipe(ialu_reg_mem);
 4953 %}
 4954 
 4955 // Load Unsigned Byte (8 bit UNsigned)
 4956 instruct loadUB(rRegI dst, memory mem)
 4957 %{
 4958   match(Set dst (LoadUB mem));
 4959 
 4960   ins_cost(125);
 4961   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4962 
 4963   ins_encode %{
 4964     __ movzbl($dst$$Register, $mem$$Address);
 4965   %}
 4966 
 4967   ins_pipe(ialu_reg_mem);
 4968 %}
 4969 
 4970 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4971 instruct loadUB2L(rRegL dst, memory mem)
 4972 %{
 4973   match(Set dst (ConvI2L (LoadUB mem)));
 4974 
 4975   ins_cost(125);
 4976   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4977 
 4978   ins_encode %{
 4979     __ movzbq($dst$$Register, $mem$$Address);
 4980   %}
 4981 
 4982   ins_pipe(ialu_reg_mem);
 4983 %}
 4984 
 4985 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4986 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4987   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4988   effect(KILL cr);
 4989 
 4990   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4991             "andl    $dst, right_n_bits($mask, 8)" %}
 4992   ins_encode %{
 4993     Register Rdst = $dst$$Register;
 4994     __ movzbq(Rdst, $mem$$Address);
 4995     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4996   %}
 4997   ins_pipe(ialu_reg_mem);
 4998 %}
 4999 
 5000 // Load Short (16 bit signed)
 5001 instruct loadS(rRegI dst, memory mem)
 5002 %{
 5003   match(Set dst (LoadS mem));
 5004 
 5005   ins_cost(125);
 5006   format %{ "movswl $dst, $mem\t# short" %}
 5007 
 5008   ins_encode %{
 5009     __ movswl($dst$$Register, $mem$$Address);
 5010   %}
 5011 
 5012   ins_pipe(ialu_reg_mem);
 5013 %}
 5014 
 5015 // Load Short (16 bit signed) to Byte (8 bit signed)
 5016 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5017   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5018 
 5019   ins_cost(125);
 5020   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5021   ins_encode %{
 5022     __ movsbl($dst$$Register, $mem$$Address);
 5023   %}
 5024   ins_pipe(ialu_reg_mem);
 5025 %}
 5026 
 5027 // Load Short (16 bit signed) into Long Register
 5028 instruct loadS2L(rRegL dst, memory mem)
 5029 %{
 5030   match(Set dst (ConvI2L (LoadS mem)));
 5031 
 5032   ins_cost(125);
 5033   format %{ "movswq $dst, $mem\t# short -> long" %}
 5034 
 5035   ins_encode %{
 5036     __ movswq($dst$$Register, $mem$$Address);
 5037   %}
 5038 
 5039   ins_pipe(ialu_reg_mem);
 5040 %}
 5041 
 5042 // Load Unsigned Short/Char (16 bit UNsigned)
 5043 instruct loadUS(rRegI dst, memory mem)
 5044 %{
 5045   match(Set dst (LoadUS mem));
 5046 
 5047   ins_cost(125);
 5048   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5049 
 5050   ins_encode %{
 5051     __ movzwl($dst$$Register, $mem$$Address);
 5052   %}
 5053 
 5054   ins_pipe(ialu_reg_mem);
 5055 %}
 5056 
 5057 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5058 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5059   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5060 
 5061   ins_cost(125);
 5062   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5063   ins_encode %{
 5064     __ movsbl($dst$$Register, $mem$$Address);
 5065   %}
 5066   ins_pipe(ialu_reg_mem);
 5067 %}
 5068 
 5069 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5070 instruct loadUS2L(rRegL dst, memory mem)
 5071 %{
 5072   match(Set dst (ConvI2L (LoadUS mem)));
 5073 
 5074   ins_cost(125);
 5075   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5076 
 5077   ins_encode %{
 5078     __ movzwq($dst$$Register, $mem$$Address);
 5079   %}
 5080 
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5085 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087 
 5088   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5089   ins_encode %{
 5090     __ movzbq($dst$$Register, $mem$$Address);
 5091   %}
 5092   ins_pipe(ialu_reg_mem);
 5093 %}
 5094 
 5095 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5096 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5097   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5098   effect(KILL cr);
 5099 
 5100   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5101             "andl    $dst, right_n_bits($mask, 16)" %}
 5102   ins_encode %{
 5103     Register Rdst = $dst$$Register;
 5104     __ movzwq(Rdst, $mem$$Address);
 5105     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5106   %}
 5107   ins_pipe(ialu_reg_mem);
 5108 %}
 5109 
 5110 // Load Integer
 5111 instruct loadI(rRegI dst, memory mem)
 5112 %{
 5113   match(Set dst (LoadI mem));
 5114 
 5115   ins_cost(125);
 5116   format %{ "movl    $dst, $mem\t# int" %}
 5117 
 5118   ins_encode %{
 5119     __ movl($dst$$Register, $mem$$Address);
 5120   %}
 5121 
 5122   ins_pipe(ialu_reg_mem);
 5123 %}
 5124 
 5125 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5126 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5127   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5128 
 5129   ins_cost(125);
 5130   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5131   ins_encode %{
 5132     __ movsbl($dst$$Register, $mem$$Address);
 5133   %}
 5134   ins_pipe(ialu_reg_mem);
 5135 %}
 5136 
 5137 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5138 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5139   match(Set dst (AndI (LoadI mem) mask));
 5140 
 5141   ins_cost(125);
 5142   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5143   ins_encode %{
 5144     __ movzbl($dst$$Register, $mem$$Address);
 5145   %}
 5146   ins_pipe(ialu_reg_mem);
 5147 %}
 5148 
 5149 // Load Integer (32 bit signed) to Short (16 bit signed)
 5150 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5151   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5152 
 5153   ins_cost(125);
 5154   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5155   ins_encode %{
 5156     __ movswl($dst$$Register, $mem$$Address);
 5157   %}
 5158   ins_pipe(ialu_reg_mem);
 5159 %}
 5160 
 5161 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5162 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5163   match(Set dst (AndI (LoadI mem) mask));
 5164 
 5165   ins_cost(125);
 5166   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5167   ins_encode %{
 5168     __ movzwl($dst$$Register, $mem$$Address);
 5169   %}
 5170   ins_pipe(ialu_reg_mem);
 5171 %}
 5172 
 5173 // Load Integer into Long Register
 5174 instruct loadI2L(rRegL dst, memory mem)
 5175 %{
 5176   match(Set dst (ConvI2L (LoadI mem)));
 5177 
 5178   ins_cost(125);
 5179   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5180 
 5181   ins_encode %{
 5182     __ movslq($dst$$Register, $mem$$Address);
 5183   %}
 5184 
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFF into Long Register
 5189 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5193   ins_encode %{
 5194     __ movzbq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with mask 0xFFFF into Long Register
 5200 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202 
 5203   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5204   ins_encode %{
 5205     __ movzwq($dst$$Register, $mem$$Address);
 5206   %}
 5207   ins_pipe(ialu_reg_mem);
 5208 %}
 5209 
 5210 // Load Integer with a 31-bit mask into Long Register
 5211 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5212   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5213   effect(KILL cr);
 5214 
 5215   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5216             "andl    $dst, $mask" %}
 5217   ins_encode %{
 5218     Register Rdst = $dst$$Register;
 5219     __ movl(Rdst, $mem$$Address);
 5220     __ andl(Rdst, $mask$$constant);
 5221   %}
 5222   ins_pipe(ialu_reg_mem);
 5223 %}
 5224 
 5225 // Load Unsigned Integer into Long Register
 5226 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5227 %{
 5228   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5229 
 5230   ins_cost(125);
 5231   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5232 
 5233   ins_encode %{
 5234     __ movl($dst$$Register, $mem$$Address);
 5235   %}
 5236 
 5237   ins_pipe(ialu_reg_mem);
 5238 %}
 5239 
 5240 // Load Long
 5241 instruct loadL(rRegL dst, memory mem)
 5242 %{
 5243   match(Set dst (LoadL mem));
 5244 
 5245   ins_cost(125);
 5246   format %{ "movq    $dst, $mem\t# long" %}
 5247 
 5248   ins_encode %{
 5249     __ movq($dst$$Register, $mem$$Address);
 5250   %}
 5251 
 5252   ins_pipe(ialu_reg_mem); // XXX
 5253 %}
 5254 
 5255 // Load Range
 5256 instruct loadRange(rRegI dst, memory mem)
 5257 %{
 5258   match(Set dst (LoadRange mem));
 5259 
 5260   ins_cost(125); // XXX
 5261   format %{ "movl    $dst, $mem\t# range" %}
 5262   ins_encode %{
 5263     __ movl($dst$$Register, $mem$$Address);
 5264   %}
 5265   ins_pipe(ialu_reg_mem);
 5266 %}
 5267 
 5268 // Load Pointer
 5269 instruct loadP(rRegP dst, memory mem)
 5270 %{
 5271   match(Set dst (LoadP mem));
 5272   predicate(n->as_Load()->barrier_data() == 0);
 5273 
 5274   ins_cost(125); // XXX
 5275   format %{ "movq    $dst, $mem\t# ptr" %}
 5276   ins_encode %{
 5277     __ movq($dst$$Register, $mem$$Address);
 5278   %}
 5279   ins_pipe(ialu_reg_mem); // XXX
 5280 %}
 5281 
 5282 // Load Compressed Pointer
 5283 instruct loadN(rRegN dst, memory mem)
 5284 %{
 5285    match(Set dst (LoadN mem));
 5286 
 5287    ins_cost(125); // XXX
 5288    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5289    ins_encode %{
 5290      __ movl($dst$$Register, $mem$$Address);
 5291    %}
 5292    ins_pipe(ialu_reg_mem); // XXX
 5293 %}
 5294 
 5295 
 5296 // Load Klass Pointer
 5297 instruct loadKlass(rRegP dst, memory mem)
 5298 %{
 5299   match(Set dst (LoadKlass mem));
 5300 
 5301   ins_cost(125); // XXX
 5302   format %{ "movq    $dst, $mem\t# class" %}
 5303   ins_encode %{
 5304     __ movq($dst$$Register, $mem$$Address);
 5305   %}
 5306   ins_pipe(ialu_reg_mem); // XXX
 5307 %}
 5308 
 5309 // Load narrow Klass Pointer
 5310 instruct loadNKlass(rRegN dst, memory mem)
 5311 %{
 5312   match(Set dst (LoadNKlass mem));
 5313 
 5314   ins_cost(125); // XXX
 5315   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5316   ins_encode %{
 5317     __ movl($dst$$Register, $mem$$Address);


 5318   %}
 5319   ins_pipe(ialu_reg_mem); // XXX
 5320 %}
 5321 
 5322 // Load Float
 5323 instruct loadF(regF dst, memory mem)
 5324 %{
 5325   match(Set dst (LoadF mem));
 5326 
 5327   ins_cost(145); // XXX
 5328   format %{ "movss   $dst, $mem\t# float" %}
 5329   ins_encode %{
 5330     __ movflt($dst$$XMMRegister, $mem$$Address);
 5331   %}
 5332   ins_pipe(pipe_slow); // XXX
 5333 %}
 5334 
 5335 // Load Double
 5336 instruct loadD_partial(regD dst, memory mem)
 5337 %{
 5338   predicate(!UseXmmLoadAndClearUpper);
 5339   match(Set dst (LoadD mem));
 5340 
 5341   ins_cost(145); // XXX
 5342   format %{ "movlpd  $dst, $mem\t# double" %}
 5343   ins_encode %{
 5344     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5345   %}
 5346   ins_pipe(pipe_slow); // XXX
 5347 %}
 5348 
 5349 instruct loadD(regD dst, memory mem)
 5350 %{
 5351   predicate(UseXmmLoadAndClearUpper);
 5352   match(Set dst (LoadD mem));
 5353 
 5354   ins_cost(145); // XXX
 5355   format %{ "movsd   $dst, $mem\t# double" %}
 5356   ins_encode %{
 5357     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5358   %}
 5359   ins_pipe(pipe_slow); // XXX
 5360 %}
 5361 
 5362 
 5363 // Following pseudo code describes the algorithm for max[FD]:
 5364 // Min algorithm is on similar lines
 5365 //  btmp = (b < +0.0) ? a : b
 5366 //  atmp = (b < +0.0) ? b : a
 5367 //  Tmp  = Max_Float(atmp , btmp)
 5368 //  Res  = (atmp == NaN) ? atmp : Tmp
 5369 
 5370 // max = java.lang.Math.max(float a, float b)
 5371 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5372   predicate(UseAVX > 0 && !n->is_reduction());
 5373   match(Set dst (MaxF a b));
 5374   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5375   format %{
 5376      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5377      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5378      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5379      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5380      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5381   %}
 5382   ins_encode %{
 5383     int vector_len = Assembler::AVX_128bit;
 5384     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5385     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5386     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5387     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5388     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5389  %}
 5390   ins_pipe( pipe_slow );
 5391 %}
 5392 
 5393 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5394   predicate(UseAVX > 0 && n->is_reduction());
 5395   match(Set dst (MaxF a b));
 5396   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5397 
 5398   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5399   ins_encode %{
 5400     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5401                     false /*min*/, true /*single*/);
 5402   %}
 5403   ins_pipe( pipe_slow );
 5404 %}
 5405 
 5406 // max = java.lang.Math.max(double a, double b)
 5407 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5408   predicate(UseAVX > 0 && !n->is_reduction());
 5409   match(Set dst (MaxD a b));
 5410   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5411   format %{
 5412      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5413      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5414      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5415      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5416      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5417   %}
 5418   ins_encode %{
 5419     int vector_len = Assembler::AVX_128bit;
 5420     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5421     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5422     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5423     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5424     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5425   %}
 5426   ins_pipe( pipe_slow );
 5427 %}
 5428 
 5429 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5430   predicate(UseAVX > 0 && n->is_reduction());
 5431   match(Set dst (MaxD a b));
 5432   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5433 
 5434   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5435   ins_encode %{
 5436     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5437                     false /*min*/, false /*single*/);
 5438   %}
 5439   ins_pipe( pipe_slow );
 5440 %}
 5441 
 5442 // min = java.lang.Math.min(float a, float b)
 5443 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5444   predicate(UseAVX > 0 && !n->is_reduction());
 5445   match(Set dst (MinF a b));
 5446   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5447   format %{
 5448      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5449      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5450      "vminss           $tmp,$atmp,$btmp           \n\t"
 5451      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5452      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5453   %}
 5454   ins_encode %{
 5455     int vector_len = Assembler::AVX_128bit;
 5456     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5457     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5458     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5459     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5460     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5461   %}
 5462   ins_pipe( pipe_slow );
 5463 %}
 5464 
 5465 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5466   predicate(UseAVX > 0 && n->is_reduction());
 5467   match(Set dst (MinF a b));
 5468   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5469 
 5470   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5471   ins_encode %{
 5472     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5473                     true /*min*/, true /*single*/);
 5474   %}
 5475   ins_pipe( pipe_slow );
 5476 %}
 5477 
 5478 // min = java.lang.Math.min(double a, double b)
 5479 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5480   predicate(UseAVX > 0 && !n->is_reduction());
 5481   match(Set dst (MinD a b));
 5482   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5483   format %{
 5484      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5485      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5486      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5487      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5488      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5489   %}
 5490   ins_encode %{
 5491     int vector_len = Assembler::AVX_128bit;
 5492     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5493     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5494     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5495     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5496     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5497   %}
 5498   ins_pipe( pipe_slow );
 5499 %}
 5500 
 5501 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5502   predicate(UseAVX > 0 && n->is_reduction());
 5503   match(Set dst (MinD a b));
 5504   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5505 
 5506   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5507   ins_encode %{
 5508     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5509                     true /*min*/, false /*single*/);
 5510   %}
 5511   ins_pipe( pipe_slow );
 5512 %}
 5513 
 5514 // Load Effective Address
 5515 instruct leaP8(rRegP dst, indOffset8 mem)
 5516 %{
 5517   match(Set dst mem);
 5518 
 5519   ins_cost(110); // XXX
 5520   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5521   ins_encode %{
 5522     __ leaq($dst$$Register, $mem$$Address);
 5523   %}
 5524   ins_pipe(ialu_reg_reg_fat);
 5525 %}
 5526 
 5527 instruct leaP32(rRegP dst, indOffset32 mem)
 5528 %{
 5529   match(Set dst mem);
 5530 
 5531   ins_cost(110);
 5532   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5533   ins_encode %{
 5534     __ leaq($dst$$Register, $mem$$Address);
 5535   %}
 5536   ins_pipe(ialu_reg_reg_fat);
 5537 %}
 5538 
 5539 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5540 %{
 5541   match(Set dst mem);
 5542 
 5543   ins_cost(110);
 5544   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5545   ins_encode %{
 5546     __ leaq($dst$$Register, $mem$$Address);
 5547   %}
 5548   ins_pipe(ialu_reg_reg_fat);
 5549 %}
 5550 
 5551 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5552 %{
 5553   match(Set dst mem);
 5554 
 5555   ins_cost(110);
 5556   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5557   ins_encode %{
 5558     __ leaq($dst$$Register, $mem$$Address);
 5559   %}
 5560   ins_pipe(ialu_reg_reg_fat);
 5561 %}
 5562 
 5563 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5564 %{
 5565   match(Set dst mem);
 5566 
 5567   ins_cost(110);
 5568   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5569   ins_encode %{
 5570     __ leaq($dst$$Register, $mem$$Address);
 5571   %}
 5572   ins_pipe(ialu_reg_reg_fat);
 5573 %}
 5574 
 5575 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5576 %{
 5577   match(Set dst mem);
 5578 
 5579   ins_cost(110);
 5580   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5581   ins_encode %{
 5582     __ leaq($dst$$Register, $mem$$Address);
 5583   %}
 5584   ins_pipe(ialu_reg_reg_fat);
 5585 %}
 5586 
 5587 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5588 %{
 5589   match(Set dst mem);
 5590 
 5591   ins_cost(110);
 5592   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5593   ins_encode %{
 5594     __ leaq($dst$$Register, $mem$$Address);
 5595   %}
 5596   ins_pipe(ialu_reg_reg_fat);
 5597 %}
 5598 
 5599 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5600 %{
 5601   match(Set dst mem);
 5602 
 5603   ins_cost(110);
 5604   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5605   ins_encode %{
 5606     __ leaq($dst$$Register, $mem$$Address);
 5607   %}
 5608   ins_pipe(ialu_reg_reg_fat);
 5609 %}
 5610 
 5611 // Load Effective Address which uses Narrow (32-bits) oop
 5612 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5613 %{
 5614   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5615   match(Set dst mem);
 5616 
 5617   ins_cost(110);
 5618   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5619   ins_encode %{
 5620     __ leaq($dst$$Register, $mem$$Address);
 5621   %}
 5622   ins_pipe(ialu_reg_reg_fat);
 5623 %}
 5624 
 5625 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5626 %{
 5627   predicate(CompressedOops::shift() == 0);
 5628   match(Set dst mem);
 5629 
 5630   ins_cost(110); // XXX
 5631   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5632   ins_encode %{
 5633     __ leaq($dst$$Register, $mem$$Address);
 5634   %}
 5635   ins_pipe(ialu_reg_reg_fat);
 5636 %}
 5637 
 5638 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5639 %{
 5640   predicate(CompressedOops::shift() == 0);
 5641   match(Set dst mem);
 5642 
 5643   ins_cost(110);
 5644   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5645   ins_encode %{
 5646     __ leaq($dst$$Register, $mem$$Address);
 5647   %}
 5648   ins_pipe(ialu_reg_reg_fat);
 5649 %}
 5650 
 5651 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5652 %{
 5653   predicate(CompressedOops::shift() == 0);
 5654   match(Set dst mem);
 5655 
 5656   ins_cost(110);
 5657   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5658   ins_encode %{
 5659     __ leaq($dst$$Register, $mem$$Address);
 5660   %}
 5661   ins_pipe(ialu_reg_reg_fat);
 5662 %}
 5663 
 5664 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5665 %{
 5666   predicate(CompressedOops::shift() == 0);
 5667   match(Set dst mem);
 5668 
 5669   ins_cost(110);
 5670   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5671   ins_encode %{
 5672     __ leaq($dst$$Register, $mem$$Address);
 5673   %}
 5674   ins_pipe(ialu_reg_reg_fat);
 5675 %}
 5676 
 5677 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5678 %{
 5679   predicate(CompressedOops::shift() == 0);
 5680   match(Set dst mem);
 5681 
 5682   ins_cost(110);
 5683   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5684   ins_encode %{
 5685     __ leaq($dst$$Register, $mem$$Address);
 5686   %}
 5687   ins_pipe(ialu_reg_reg_fat);
 5688 %}
 5689 
 5690 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5691 %{
 5692   predicate(CompressedOops::shift() == 0);
 5693   match(Set dst mem);
 5694 
 5695   ins_cost(110);
 5696   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5697   ins_encode %{
 5698     __ leaq($dst$$Register, $mem$$Address);
 5699   %}
 5700   ins_pipe(ialu_reg_reg_fat);
 5701 %}
 5702 
 5703 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5704 %{
 5705   predicate(CompressedOops::shift() == 0);
 5706   match(Set dst mem);
 5707 
 5708   ins_cost(110);
 5709   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5710   ins_encode %{
 5711     __ leaq($dst$$Register, $mem$$Address);
 5712   %}
 5713   ins_pipe(ialu_reg_reg_fat);
 5714 %}
 5715 
 5716 instruct loadConI(rRegI dst, immI src)
 5717 %{
 5718   match(Set dst src);
 5719 
 5720   format %{ "movl    $dst, $src\t# int" %}
 5721   ins_encode %{
 5722     __ movl($dst$$Register, $src$$constant);
 5723   %}
 5724   ins_pipe(ialu_reg_fat); // XXX
 5725 %}
 5726 
 5727 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5728 %{
 5729   match(Set dst src);
 5730   effect(KILL cr);
 5731 
 5732   ins_cost(50);
 5733   format %{ "xorl    $dst, $dst\t# int" %}
 5734   ins_encode %{
 5735     __ xorl($dst$$Register, $dst$$Register);
 5736   %}
 5737   ins_pipe(ialu_reg);
 5738 %}
 5739 
 5740 instruct loadConL(rRegL dst, immL src)
 5741 %{
 5742   match(Set dst src);
 5743 
 5744   ins_cost(150);
 5745   format %{ "movq    $dst, $src\t# long" %}
 5746   ins_encode %{
 5747     __ mov64($dst$$Register, $src$$constant);
 5748   %}
 5749   ins_pipe(ialu_reg);
 5750 %}
 5751 
 5752 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5753 %{
 5754   match(Set dst src);
 5755   effect(KILL cr);
 5756 
 5757   ins_cost(50);
 5758   format %{ "xorl    $dst, $dst\t# long" %}
 5759   ins_encode %{
 5760     __ xorl($dst$$Register, $dst$$Register);
 5761   %}
 5762   ins_pipe(ialu_reg); // XXX
 5763 %}
 5764 
 5765 instruct loadConUL32(rRegL dst, immUL32 src)
 5766 %{
 5767   match(Set dst src);
 5768 
 5769   ins_cost(60);
 5770   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5771   ins_encode %{
 5772     __ movl($dst$$Register, $src$$constant);
 5773   %}
 5774   ins_pipe(ialu_reg);
 5775 %}
 5776 
 5777 instruct loadConL32(rRegL dst, immL32 src)
 5778 %{
 5779   match(Set dst src);
 5780 
 5781   ins_cost(70);
 5782   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5783   ins_encode %{
 5784     __ movq($dst$$Register, $src$$constant);
 5785   %}
 5786   ins_pipe(ialu_reg);
 5787 %}
 5788 
 5789 instruct loadConP(rRegP dst, immP con) %{
 5790   match(Set dst con);
 5791 
 5792   format %{ "movq    $dst, $con\t# ptr" %}
 5793   ins_encode %{
 5794     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5795   %}
 5796   ins_pipe(ialu_reg_fat); // XXX
 5797 %}
 5798 
 5799 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5800 %{
 5801   match(Set dst src);
 5802   effect(KILL cr);
 5803 
 5804   ins_cost(50);
 5805   format %{ "xorl    $dst, $dst\t# ptr" %}
 5806   ins_encode %{
 5807     __ xorl($dst$$Register, $dst$$Register);
 5808   %}
 5809   ins_pipe(ialu_reg);
 5810 %}
 5811 
 5812 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5813 %{
 5814   match(Set dst src);
 5815   effect(KILL cr);
 5816 
 5817   ins_cost(60);
 5818   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5819   ins_encode %{
 5820     __ movl($dst$$Register, $src$$constant);
 5821   %}
 5822   ins_pipe(ialu_reg);
 5823 %}
 5824 
 5825 instruct loadConF(regF dst, immF con) %{
 5826   match(Set dst con);
 5827   ins_cost(125);
 5828   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5829   ins_encode %{
 5830     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5831   %}
 5832   ins_pipe(pipe_slow);
 5833 %}
 5834 
 5835 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5836   match(Set dst src);
 5837   effect(KILL cr);
 5838   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5839   ins_encode %{
 5840     __ xorq($dst$$Register, $dst$$Register);
 5841   %}
 5842   ins_pipe(ialu_reg);
 5843 %}
 5844 
 5845 instruct loadConN(rRegN dst, immN src) %{
 5846   match(Set dst src);
 5847 
 5848   ins_cost(125);
 5849   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5850   ins_encode %{
 5851     address con = (address)$src$$constant;
 5852     if (con == NULL) {
 5853       ShouldNotReachHere();
 5854     } else {
 5855       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5856     }
 5857   %}
 5858   ins_pipe(ialu_reg_fat); // XXX
 5859 %}
 5860 
 5861 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5862   match(Set dst src);
 5863 
 5864   ins_cost(125);
 5865   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5866   ins_encode %{
 5867     address con = (address)$src$$constant;
 5868     if (con == NULL) {
 5869       ShouldNotReachHere();
 5870     } else {
 5871       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5872     }
 5873   %}
 5874   ins_pipe(ialu_reg_fat); // XXX
 5875 %}
 5876 
 5877 instruct loadConF0(regF dst, immF0 src)
 5878 %{
 5879   match(Set dst src);
 5880   ins_cost(100);
 5881 
 5882   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5883   ins_encode %{
 5884     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5885   %}
 5886   ins_pipe(pipe_slow);
 5887 %}
 5888 
 5889 // Use the same format since predicate() can not be used here.
 5890 instruct loadConD(regD dst, immD con) %{
 5891   match(Set dst con);
 5892   ins_cost(125);
 5893   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5894   ins_encode %{
 5895     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5896   %}
 5897   ins_pipe(pipe_slow);
 5898 %}
 5899 
 5900 instruct loadConD0(regD dst, immD0 src)
 5901 %{
 5902   match(Set dst src);
 5903   ins_cost(100);
 5904 
 5905   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5906   ins_encode %{
 5907     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5908   %}
 5909   ins_pipe(pipe_slow);
 5910 %}
 5911 
 5912 instruct loadSSI(rRegI dst, stackSlotI src)
 5913 %{
 5914   match(Set dst src);
 5915 
 5916   ins_cost(125);
 5917   format %{ "movl    $dst, $src\t# int stk" %}
 5918   opcode(0x8B);
 5919   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5920   ins_pipe(ialu_reg_mem);
 5921 %}
 5922 
 5923 instruct loadSSL(rRegL dst, stackSlotL src)
 5924 %{
 5925   match(Set dst src);
 5926 
 5927   ins_cost(125);
 5928   format %{ "movq    $dst, $src\t# long stk" %}
 5929   opcode(0x8B);
 5930   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5931   ins_pipe(ialu_reg_mem);
 5932 %}
 5933 
 5934 instruct loadSSP(rRegP dst, stackSlotP src)
 5935 %{
 5936   match(Set dst src);
 5937 
 5938   ins_cost(125);
 5939   format %{ "movq    $dst, $src\t# ptr stk" %}
 5940   opcode(0x8B);
 5941   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5942   ins_pipe(ialu_reg_mem);
 5943 %}
 5944 
 5945 instruct loadSSF(regF dst, stackSlotF src)
 5946 %{
 5947   match(Set dst src);
 5948 
 5949   ins_cost(125);
 5950   format %{ "movss   $dst, $src\t# float stk" %}
 5951   ins_encode %{
 5952     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5953   %}
 5954   ins_pipe(pipe_slow); // XXX
 5955 %}
 5956 
 5957 // Use the same format since predicate() can not be used here.
 5958 instruct loadSSD(regD dst, stackSlotD src)
 5959 %{
 5960   match(Set dst src);
 5961 
 5962   ins_cost(125);
 5963   format %{ "movsd   $dst, $src\t# double stk" %}
 5964   ins_encode  %{
 5965     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5966   %}
 5967   ins_pipe(pipe_slow); // XXX
 5968 %}
 5969 
 5970 // Prefetch instructions for allocation.
 5971 // Must be safe to execute with invalid address (cannot fault).
 5972 
 5973 instruct prefetchAlloc( memory mem ) %{
 5974   predicate(AllocatePrefetchInstr==3);
 5975   match(PrefetchAllocation mem);
 5976   ins_cost(125);
 5977 
 5978   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5979   ins_encode %{
 5980     __ prefetchw($mem$$Address);
 5981   %}
 5982   ins_pipe(ialu_mem);
 5983 %}
 5984 
 5985 instruct prefetchAllocNTA( memory mem ) %{
 5986   predicate(AllocatePrefetchInstr==0);
 5987   match(PrefetchAllocation mem);
 5988   ins_cost(125);
 5989 
 5990   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5991   ins_encode %{
 5992     __ prefetchnta($mem$$Address);
 5993   %}
 5994   ins_pipe(ialu_mem);
 5995 %}
 5996 
 5997 instruct prefetchAllocT0( memory mem ) %{
 5998   predicate(AllocatePrefetchInstr==1);
 5999   match(PrefetchAllocation mem);
 6000   ins_cost(125);
 6001 
 6002   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6003   ins_encode %{
 6004     __ prefetcht0($mem$$Address);
 6005   %}
 6006   ins_pipe(ialu_mem);
 6007 %}
 6008 
 6009 instruct prefetchAllocT2( memory mem ) %{
 6010   predicate(AllocatePrefetchInstr==2);
 6011   match(PrefetchAllocation mem);
 6012   ins_cost(125);
 6013 
 6014   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6015   ins_encode %{
 6016     __ prefetcht2($mem$$Address);
 6017   %}
 6018   ins_pipe(ialu_mem);
 6019 %}
 6020 
 6021 //----------Store Instructions-------------------------------------------------
 6022 
 6023 // Store Byte
 6024 instruct storeB(memory mem, rRegI src)
 6025 %{
 6026   match(Set mem (StoreB mem src));
 6027 
 6028   ins_cost(125); // XXX
 6029   format %{ "movb    $mem, $src\t# byte" %}
 6030   ins_encode %{
 6031     __ movb($mem$$Address, $src$$Register);
 6032   %}
 6033   ins_pipe(ialu_mem_reg);
 6034 %}
 6035 
 6036 // Store Char/Short
 6037 instruct storeC(memory mem, rRegI src)
 6038 %{
 6039   match(Set mem (StoreC mem src));
 6040 
 6041   ins_cost(125); // XXX
 6042   format %{ "movw    $mem, $src\t# char/short" %}
 6043   ins_encode %{
 6044     __ movw($mem$$Address, $src$$Register);
 6045   %}
 6046   ins_pipe(ialu_mem_reg);
 6047 %}
 6048 
 6049 // Store Integer
 6050 instruct storeI(memory mem, rRegI src)
 6051 %{
 6052   match(Set mem (StoreI mem src));
 6053 
 6054   ins_cost(125); // XXX
 6055   format %{ "movl    $mem, $src\t# int" %}
 6056   ins_encode %{
 6057     __ movl($mem$$Address, $src$$Register);
 6058   %}
 6059   ins_pipe(ialu_mem_reg);
 6060 %}
 6061 
 6062 // Store Long
 6063 instruct storeL(memory mem, rRegL src)
 6064 %{
 6065   match(Set mem (StoreL mem src));
 6066 
 6067   ins_cost(125); // XXX
 6068   format %{ "movq    $mem, $src\t# long" %}
 6069   ins_encode %{
 6070     __ movq($mem$$Address, $src$$Register);
 6071   %}
 6072   ins_pipe(ialu_mem_reg); // XXX
 6073 %}
 6074 
 6075 // Store Pointer
 6076 instruct storeP(memory mem, any_RegP src)
 6077 %{
 6078   match(Set mem (StoreP mem src));
 6079 
 6080   ins_cost(125); // XXX
 6081   format %{ "movq    $mem, $src\t# ptr" %}
 6082   ins_encode %{
 6083     __ movq($mem$$Address, $src$$Register);
 6084   %}
 6085   ins_pipe(ialu_mem_reg);
 6086 %}
 6087 
 6088 instruct storeImmP0(memory mem, immP0 zero)
 6089 %{
 6090   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6091   match(Set mem (StoreP mem zero));
 6092 
 6093   ins_cost(125); // XXX
 6094   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6095   ins_encode %{
 6096     __ movq($mem$$Address, r12);
 6097   %}
 6098   ins_pipe(ialu_mem_reg);
 6099 %}
 6100 
 6101 // Store NULL Pointer, mark word, or other simple pointer constant.
 6102 instruct storeImmP(memory mem, immP31 src)
 6103 %{
 6104   match(Set mem (StoreP mem src));
 6105 
 6106   ins_cost(150); // XXX
 6107   format %{ "movq    $mem, $src\t# ptr" %}
 6108   ins_encode %{
 6109     __ movq($mem$$Address, $src$$constant);
 6110   %}
 6111   ins_pipe(ialu_mem_imm);
 6112 %}
 6113 
 6114 // Store Compressed Pointer
 6115 instruct storeN(memory mem, rRegN src)
 6116 %{
 6117   match(Set mem (StoreN mem src));
 6118 
 6119   ins_cost(125); // XXX
 6120   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6121   ins_encode %{
 6122     __ movl($mem$$Address, $src$$Register);
 6123   %}
 6124   ins_pipe(ialu_mem_reg);
 6125 %}
 6126 
 6127 instruct storeNKlass(memory mem, rRegN src)
 6128 %{
 6129   match(Set mem (StoreNKlass mem src));
 6130 
 6131   ins_cost(125); // XXX
 6132   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6133   ins_encode %{
 6134     __ movl($mem$$Address, $src$$Register);
 6135   %}
 6136   ins_pipe(ialu_mem_reg);
 6137 %}
 6138 
 6139 instruct storeImmN0(memory mem, immN0 zero)
 6140 %{
 6141   predicate(CompressedOops::base() == NULL);
 6142   match(Set mem (StoreN mem zero));
 6143 
 6144   ins_cost(125); // XXX
 6145   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6146   ins_encode %{
 6147     __ movl($mem$$Address, r12);
 6148   %}
 6149   ins_pipe(ialu_mem_reg);
 6150 %}
 6151 
 6152 instruct storeImmN(memory mem, immN src)
 6153 %{
 6154   match(Set mem (StoreN mem src));
 6155 
 6156   ins_cost(150); // XXX
 6157   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6158   ins_encode %{
 6159     address con = (address)$src$$constant;
 6160     if (con == NULL) {
 6161       __ movl($mem$$Address, (int32_t)0);
 6162     } else {
 6163       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6164     }
 6165   %}
 6166   ins_pipe(ialu_mem_imm);
 6167 %}
 6168 
 6169 instruct storeImmNKlass(memory mem, immNKlass src)
 6170 %{
 6171   match(Set mem (StoreNKlass mem src));
 6172 
 6173   ins_cost(150); // XXX
 6174   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6175   ins_encode %{
 6176     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6177   %}
 6178   ins_pipe(ialu_mem_imm);
 6179 %}
 6180 
 6181 // Store Integer Immediate
 6182 instruct storeImmI0(memory mem, immI_0 zero)
 6183 %{
 6184   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6185   match(Set mem (StoreI mem zero));
 6186 
 6187   ins_cost(125); // XXX
 6188   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6189   ins_encode %{
 6190     __ movl($mem$$Address, r12);
 6191   %}
 6192   ins_pipe(ialu_mem_reg);
 6193 %}
 6194 
 6195 instruct storeImmI(memory mem, immI src)
 6196 %{
 6197   match(Set mem (StoreI mem src));
 6198 
 6199   ins_cost(150);
 6200   format %{ "movl    $mem, $src\t# int" %}
 6201   ins_encode %{
 6202     __ movl($mem$$Address, $src$$constant);
 6203   %}
 6204   ins_pipe(ialu_mem_imm);
 6205 %}
 6206 
 6207 // Store Long Immediate
 6208 instruct storeImmL0(memory mem, immL0 zero)
 6209 %{
 6210   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6211   match(Set mem (StoreL mem zero));
 6212 
 6213   ins_cost(125); // XXX
 6214   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6215   ins_encode %{
 6216     __ movq($mem$$Address, r12);
 6217   %}
 6218   ins_pipe(ialu_mem_reg);
 6219 %}
 6220 
 6221 instruct storeImmL(memory mem, immL32 src)
 6222 %{
 6223   match(Set mem (StoreL mem src));
 6224 
 6225   ins_cost(150);
 6226   format %{ "movq    $mem, $src\t# long" %}
 6227   ins_encode %{
 6228     __ movq($mem$$Address, $src$$constant);
 6229   %}
 6230   ins_pipe(ialu_mem_imm);
 6231 %}
 6232 
 6233 // Store Short/Char Immediate
 6234 instruct storeImmC0(memory mem, immI_0 zero)
 6235 %{
 6236   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6237   match(Set mem (StoreC mem zero));
 6238 
 6239   ins_cost(125); // XXX
 6240   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6241   ins_encode %{
 6242     __ movw($mem$$Address, r12);
 6243   %}
 6244   ins_pipe(ialu_mem_reg);
 6245 %}
 6246 
 6247 instruct storeImmI16(memory mem, immI16 src)
 6248 %{
 6249   predicate(UseStoreImmI16);
 6250   match(Set mem (StoreC mem src));
 6251 
 6252   ins_cost(150);
 6253   format %{ "movw    $mem, $src\t# short/char" %}
 6254   ins_encode %{
 6255     __ movw($mem$$Address, $src$$constant);
 6256   %}
 6257   ins_pipe(ialu_mem_imm);
 6258 %}
 6259 
 6260 // Store Byte Immediate
 6261 instruct storeImmB0(memory mem, immI_0 zero)
 6262 %{
 6263   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6264   match(Set mem (StoreB mem zero));
 6265 
 6266   ins_cost(125); // XXX
 6267   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6268   ins_encode %{
 6269     __ movb($mem$$Address, r12);
 6270   %}
 6271   ins_pipe(ialu_mem_reg);
 6272 %}
 6273 
 6274 instruct storeImmB(memory mem, immI8 src)
 6275 %{
 6276   match(Set mem (StoreB mem src));
 6277 
 6278   ins_cost(150); // XXX
 6279   format %{ "movb    $mem, $src\t# byte" %}
 6280   ins_encode %{
 6281     __ movb($mem$$Address, $src$$constant);
 6282   %}
 6283   ins_pipe(ialu_mem_imm);
 6284 %}
 6285 
 6286 // Store CMS card-mark Immediate
 6287 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6288 %{
 6289   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6290   match(Set mem (StoreCM mem zero));
 6291 
 6292   ins_cost(125); // XXX
 6293   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6294   ins_encode %{
 6295     __ movb($mem$$Address, r12);
 6296   %}
 6297   ins_pipe(ialu_mem_reg);
 6298 %}
 6299 
 6300 instruct storeImmCM0(memory mem, immI_0 src)
 6301 %{
 6302   match(Set mem (StoreCM mem src));
 6303 
 6304   ins_cost(150); // XXX
 6305   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6306   ins_encode %{
 6307     __ movb($mem$$Address, $src$$constant);
 6308   %}
 6309   ins_pipe(ialu_mem_imm);
 6310 %}
 6311 
 6312 // Store Float
 6313 instruct storeF(memory mem, regF src)
 6314 %{
 6315   match(Set mem (StoreF mem src));
 6316 
 6317   ins_cost(95); // XXX
 6318   format %{ "movss   $mem, $src\t# float" %}
 6319   ins_encode %{
 6320     __ movflt($mem$$Address, $src$$XMMRegister);
 6321   %}
 6322   ins_pipe(pipe_slow); // XXX
 6323 %}
 6324 
 6325 // Store immediate Float value (it is faster than store from XMM register)
 6326 instruct storeF0(memory mem, immF0 zero)
 6327 %{
 6328   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6329   match(Set mem (StoreF mem zero));
 6330 
 6331   ins_cost(25); // XXX
 6332   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6333   ins_encode %{
 6334     __ movl($mem$$Address, r12);
 6335   %}
 6336   ins_pipe(ialu_mem_reg);
 6337 %}
 6338 
 6339 instruct storeF_imm(memory mem, immF src)
 6340 %{
 6341   match(Set mem (StoreF mem src));
 6342 
 6343   ins_cost(50);
 6344   format %{ "movl    $mem, $src\t# float" %}
 6345   ins_encode %{
 6346     __ movl($mem$$Address, jint_cast($src$$constant));
 6347   %}
 6348   ins_pipe(ialu_mem_imm);
 6349 %}
 6350 
 6351 // Store Double
 6352 instruct storeD(memory mem, regD src)
 6353 %{
 6354   match(Set mem (StoreD mem src));
 6355 
 6356   ins_cost(95); // XXX
 6357   format %{ "movsd   $mem, $src\t# double" %}
 6358   ins_encode %{
 6359     __ movdbl($mem$$Address, $src$$XMMRegister);
 6360   %}
 6361   ins_pipe(pipe_slow); // XXX
 6362 %}
 6363 
 6364 // Store immediate double 0.0 (it is faster than store from XMM register)
 6365 instruct storeD0_imm(memory mem, immD0 src)
 6366 %{
 6367   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6368   match(Set mem (StoreD mem src));
 6369 
 6370   ins_cost(50);
 6371   format %{ "movq    $mem, $src\t# double 0." %}
 6372   ins_encode %{
 6373     __ movq($mem$$Address, $src$$constant);
 6374   %}
 6375   ins_pipe(ialu_mem_imm);
 6376 %}
 6377 
 6378 instruct storeD0(memory mem, immD0 zero)
 6379 %{
 6380   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6381   match(Set mem (StoreD mem zero));
 6382 
 6383   ins_cost(25); // XXX
 6384   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6385   ins_encode %{
 6386     __ movq($mem$$Address, r12);
 6387   %}
 6388   ins_pipe(ialu_mem_reg);
 6389 %}
 6390 
 6391 instruct storeSSI(stackSlotI dst, rRegI src)
 6392 %{
 6393   match(Set dst src);
 6394 
 6395   ins_cost(100);
 6396   format %{ "movl    $dst, $src\t# int stk" %}
 6397   opcode(0x89);
 6398   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6399   ins_pipe( ialu_mem_reg );
 6400 %}
 6401 
 6402 instruct storeSSL(stackSlotL dst, rRegL src)
 6403 %{
 6404   match(Set dst src);
 6405 
 6406   ins_cost(100);
 6407   format %{ "movq    $dst, $src\t# long stk" %}
 6408   opcode(0x89);
 6409   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6410   ins_pipe(ialu_mem_reg);
 6411 %}
 6412 
 6413 instruct storeSSP(stackSlotP dst, rRegP src)
 6414 %{
 6415   match(Set dst src);
 6416 
 6417   ins_cost(100);
 6418   format %{ "movq    $dst, $src\t# ptr stk" %}
 6419   opcode(0x89);
 6420   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6421   ins_pipe(ialu_mem_reg);
 6422 %}
 6423 
 6424 instruct storeSSF(stackSlotF dst, regF src)
 6425 %{
 6426   match(Set dst src);
 6427 
 6428   ins_cost(95); // XXX
 6429   format %{ "movss   $dst, $src\t# float stk" %}
 6430   ins_encode %{
 6431     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6432   %}
 6433   ins_pipe(pipe_slow); // XXX
 6434 %}
 6435 
 6436 instruct storeSSD(stackSlotD dst, regD src)
 6437 %{
 6438   match(Set dst src);
 6439 
 6440   ins_cost(95); // XXX
 6441   format %{ "movsd   $dst, $src\t# double stk" %}
 6442   ins_encode %{
 6443     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6444   %}
 6445   ins_pipe(pipe_slow); // XXX
 6446 %}
 6447 
 6448 instruct cacheWB(indirect addr)
 6449 %{
 6450   predicate(VM_Version::supports_data_cache_line_flush());
 6451   match(CacheWB addr);
 6452 
 6453   ins_cost(100);
 6454   format %{"cache wb $addr" %}
 6455   ins_encode %{
 6456     assert($addr->index_position() < 0, "should be");
 6457     assert($addr$$disp == 0, "should be");
 6458     __ cache_wb(Address($addr$$base$$Register, 0));
 6459   %}
 6460   ins_pipe(pipe_slow); // XXX
 6461 %}
 6462 
 6463 instruct cacheWBPreSync()
 6464 %{
 6465   predicate(VM_Version::supports_data_cache_line_flush());
 6466   match(CacheWBPreSync);
 6467 
 6468   ins_cost(100);
 6469   format %{"cache wb presync" %}
 6470   ins_encode %{
 6471     __ cache_wbsync(true);
 6472   %}
 6473   ins_pipe(pipe_slow); // XXX
 6474 %}
 6475 
 6476 instruct cacheWBPostSync()
 6477 %{
 6478   predicate(VM_Version::supports_data_cache_line_flush());
 6479   match(CacheWBPostSync);
 6480 
 6481   ins_cost(100);
 6482   format %{"cache wb postsync" %}
 6483   ins_encode %{
 6484     __ cache_wbsync(false);
 6485   %}
 6486   ins_pipe(pipe_slow); // XXX
 6487 %}
 6488 
 6489 //----------BSWAP Instructions-------------------------------------------------
 6490 instruct bytes_reverse_int(rRegI dst) %{
 6491   match(Set dst (ReverseBytesI dst));
 6492 
 6493   format %{ "bswapl  $dst" %}
 6494   ins_encode %{
 6495     __ bswapl($dst$$Register);
 6496   %}
 6497   ins_pipe( ialu_reg );
 6498 %}
 6499 
 6500 instruct bytes_reverse_long(rRegL dst) %{
 6501   match(Set dst (ReverseBytesL dst));
 6502 
 6503   format %{ "bswapq  $dst" %}
 6504   ins_encode %{
 6505     __ bswapq($dst$$Register);
 6506   %}
 6507   ins_pipe( ialu_reg);
 6508 %}
 6509 
 6510 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6511   match(Set dst (ReverseBytesUS dst));
 6512   effect(KILL cr);
 6513 
 6514   format %{ "bswapl  $dst\n\t"
 6515             "shrl    $dst,16\n\t" %}
 6516   ins_encode %{
 6517     __ bswapl($dst$$Register);
 6518     __ shrl($dst$$Register, 16);
 6519   %}
 6520   ins_pipe( ialu_reg );
 6521 %}
 6522 
 6523 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6524   match(Set dst (ReverseBytesS dst));
 6525   effect(KILL cr);
 6526 
 6527   format %{ "bswapl  $dst\n\t"
 6528             "sar     $dst,16\n\t" %}
 6529   ins_encode %{
 6530     __ bswapl($dst$$Register);
 6531     __ sarl($dst$$Register, 16);
 6532   %}
 6533   ins_pipe( ialu_reg );
 6534 %}
 6535 
 6536 //---------- Zeros Count Instructions ------------------------------------------
 6537 
 6538 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6539   predicate(UseCountLeadingZerosInstruction);
 6540   match(Set dst (CountLeadingZerosI src));
 6541   effect(KILL cr);
 6542 
 6543   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6544   ins_encode %{
 6545     __ lzcntl($dst$$Register, $src$$Register);
 6546   %}
 6547   ins_pipe(ialu_reg);
 6548 %}
 6549 
 6550 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6551   predicate(UseCountLeadingZerosInstruction);
 6552   match(Set dst (CountLeadingZerosI (LoadI src)));
 6553   effect(KILL cr);
 6554   ins_cost(175);
 6555   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6556   ins_encode %{
 6557     __ lzcntl($dst$$Register, $src$$Address);
 6558   %}
 6559   ins_pipe(ialu_reg_mem);
 6560 %}
 6561 
 6562 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6563   predicate(!UseCountLeadingZerosInstruction);
 6564   match(Set dst (CountLeadingZerosI src));
 6565   effect(KILL cr);
 6566 
 6567   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6568             "jnz     skip\n\t"
 6569             "movl    $dst, -1\n"
 6570       "skip:\n\t"
 6571             "negl    $dst\n\t"
 6572             "addl    $dst, 31" %}
 6573   ins_encode %{
 6574     Register Rdst = $dst$$Register;
 6575     Register Rsrc = $src$$Register;
 6576     Label skip;
 6577     __ bsrl(Rdst, Rsrc);
 6578     __ jccb(Assembler::notZero, skip);
 6579     __ movl(Rdst, -1);
 6580     __ bind(skip);
 6581     __ negl(Rdst);
 6582     __ addl(Rdst, BitsPerInt - 1);
 6583   %}
 6584   ins_pipe(ialu_reg);
 6585 %}
 6586 
 6587 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6588   predicate(UseCountLeadingZerosInstruction);
 6589   match(Set dst (CountLeadingZerosL src));
 6590   effect(KILL cr);
 6591 
 6592   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6593   ins_encode %{
 6594     __ lzcntq($dst$$Register, $src$$Register);
 6595   %}
 6596   ins_pipe(ialu_reg);
 6597 %}
 6598 
 6599 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6600   predicate(UseCountLeadingZerosInstruction);
 6601   match(Set dst (CountLeadingZerosL (LoadL src)));
 6602   effect(KILL cr);
 6603   ins_cost(175);
 6604   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6605   ins_encode %{
 6606     __ lzcntq($dst$$Register, $src$$Address);
 6607   %}
 6608   ins_pipe(ialu_reg_mem);
 6609 %}
 6610 
 6611 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6612   predicate(!UseCountLeadingZerosInstruction);
 6613   match(Set dst (CountLeadingZerosL src));
 6614   effect(KILL cr);
 6615 
 6616   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6617             "jnz     skip\n\t"
 6618             "movl    $dst, -1\n"
 6619       "skip:\n\t"
 6620             "negl    $dst\n\t"
 6621             "addl    $dst, 63" %}
 6622   ins_encode %{
 6623     Register Rdst = $dst$$Register;
 6624     Register Rsrc = $src$$Register;
 6625     Label skip;
 6626     __ bsrq(Rdst, Rsrc);
 6627     __ jccb(Assembler::notZero, skip);
 6628     __ movl(Rdst, -1);
 6629     __ bind(skip);
 6630     __ negl(Rdst);
 6631     __ addl(Rdst, BitsPerLong - 1);
 6632   %}
 6633   ins_pipe(ialu_reg);
 6634 %}
 6635 
 6636 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6637   predicate(UseCountTrailingZerosInstruction);
 6638   match(Set dst (CountTrailingZerosI src));
 6639   effect(KILL cr);
 6640 
 6641   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6642   ins_encode %{
 6643     __ tzcntl($dst$$Register, $src$$Register);
 6644   %}
 6645   ins_pipe(ialu_reg);
 6646 %}
 6647 
 6648 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6649   predicate(UseCountTrailingZerosInstruction);
 6650   match(Set dst (CountTrailingZerosI (LoadI src)));
 6651   effect(KILL cr);
 6652   ins_cost(175);
 6653   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6654   ins_encode %{
 6655     __ tzcntl($dst$$Register, $src$$Address);
 6656   %}
 6657   ins_pipe(ialu_reg_mem);
 6658 %}
 6659 
 6660 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6661   predicate(!UseCountTrailingZerosInstruction);
 6662   match(Set dst (CountTrailingZerosI src));
 6663   effect(KILL cr);
 6664 
 6665   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6666             "jnz     done\n\t"
 6667             "movl    $dst, 32\n"
 6668       "done:" %}
 6669   ins_encode %{
 6670     Register Rdst = $dst$$Register;
 6671     Label done;
 6672     __ bsfl(Rdst, $src$$Register);
 6673     __ jccb(Assembler::notZero, done);
 6674     __ movl(Rdst, BitsPerInt);
 6675     __ bind(done);
 6676   %}
 6677   ins_pipe(ialu_reg);
 6678 %}
 6679 
 6680 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6681   predicate(UseCountTrailingZerosInstruction);
 6682   match(Set dst (CountTrailingZerosL src));
 6683   effect(KILL cr);
 6684 
 6685   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6686   ins_encode %{
 6687     __ tzcntq($dst$$Register, $src$$Register);
 6688   %}
 6689   ins_pipe(ialu_reg);
 6690 %}
 6691 
 6692 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6693   predicate(UseCountTrailingZerosInstruction);
 6694   match(Set dst (CountTrailingZerosL (LoadL src)));
 6695   effect(KILL cr);
 6696   ins_cost(175);
 6697   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6698   ins_encode %{
 6699     __ tzcntq($dst$$Register, $src$$Address);
 6700   %}
 6701   ins_pipe(ialu_reg_mem);
 6702 %}
 6703 
 6704 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6705   predicate(!UseCountTrailingZerosInstruction);
 6706   match(Set dst (CountTrailingZerosL src));
 6707   effect(KILL cr);
 6708 
 6709   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6710             "jnz     done\n\t"
 6711             "movl    $dst, 64\n"
 6712       "done:" %}
 6713   ins_encode %{
 6714     Register Rdst = $dst$$Register;
 6715     Label done;
 6716     __ bsfq(Rdst, $src$$Register);
 6717     __ jccb(Assembler::notZero, done);
 6718     __ movl(Rdst, BitsPerLong);
 6719     __ bind(done);
 6720   %}
 6721   ins_pipe(ialu_reg);
 6722 %}
 6723 
 6724 
 6725 //---------- Population Count Instructions -------------------------------------
 6726 
 6727 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6728   predicate(UsePopCountInstruction);
 6729   match(Set dst (PopCountI src));
 6730   effect(KILL cr);
 6731 
 6732   format %{ "popcnt  $dst, $src" %}
 6733   ins_encode %{
 6734     __ popcntl($dst$$Register, $src$$Register);
 6735   %}
 6736   ins_pipe(ialu_reg);
 6737 %}
 6738 
 6739 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6740   predicate(UsePopCountInstruction);
 6741   match(Set dst (PopCountI (LoadI mem)));
 6742   effect(KILL cr);
 6743 
 6744   format %{ "popcnt  $dst, $mem" %}
 6745   ins_encode %{
 6746     __ popcntl($dst$$Register, $mem$$Address);
 6747   %}
 6748   ins_pipe(ialu_reg);
 6749 %}
 6750 
 6751 // Note: Long.bitCount(long) returns an int.
 6752 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6753   predicate(UsePopCountInstruction);
 6754   match(Set dst (PopCountL src));
 6755   effect(KILL cr);
 6756 
 6757   format %{ "popcnt  $dst, $src" %}
 6758   ins_encode %{
 6759     __ popcntq($dst$$Register, $src$$Register);
 6760   %}
 6761   ins_pipe(ialu_reg);
 6762 %}
 6763 
 6764 // Note: Long.bitCount(long) returns an int.
 6765 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6766   predicate(UsePopCountInstruction);
 6767   match(Set dst (PopCountL (LoadL mem)));
 6768   effect(KILL cr);
 6769 
 6770   format %{ "popcnt  $dst, $mem" %}
 6771   ins_encode %{
 6772     __ popcntq($dst$$Register, $mem$$Address);
 6773   %}
 6774   ins_pipe(ialu_reg);
 6775 %}
 6776 
 6777 
 6778 //----------MemBar Instructions-----------------------------------------------
 6779 // Memory barrier flavors
 6780 
 6781 instruct membar_acquire()
 6782 %{
 6783   match(MemBarAcquire);
 6784   match(LoadFence);
 6785   ins_cost(0);
 6786 
 6787   size(0);
 6788   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6789   ins_encode();
 6790   ins_pipe(empty);
 6791 %}
 6792 
 6793 instruct membar_acquire_lock()
 6794 %{
 6795   match(MemBarAcquireLock);
 6796   ins_cost(0);
 6797 
 6798   size(0);
 6799   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6800   ins_encode();
 6801   ins_pipe(empty);
 6802 %}
 6803 
 6804 instruct membar_release()
 6805 %{
 6806   match(MemBarRelease);
 6807   match(StoreFence);
 6808   ins_cost(0);
 6809 
 6810   size(0);
 6811   format %{ "MEMBAR-release ! (empty encoding)" %}
 6812   ins_encode();
 6813   ins_pipe(empty);
 6814 %}
 6815 
 6816 instruct membar_release_lock()
 6817 %{
 6818   match(MemBarReleaseLock);
 6819   ins_cost(0);
 6820 
 6821   size(0);
 6822   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6823   ins_encode();
 6824   ins_pipe(empty);
 6825 %}
 6826 
 6827 instruct membar_volatile(rFlagsReg cr) %{
 6828   match(MemBarVolatile);
 6829   effect(KILL cr);
 6830   ins_cost(400);
 6831 
 6832   format %{
 6833     $$template
 6834     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6835   %}
 6836   ins_encode %{
 6837     __ membar(Assembler::StoreLoad);
 6838   %}
 6839   ins_pipe(pipe_slow);
 6840 %}
 6841 
 6842 instruct unnecessary_membar_volatile()
 6843 %{
 6844   match(MemBarVolatile);
 6845   predicate(Matcher::post_store_load_barrier(n));
 6846   ins_cost(0);
 6847 
 6848   size(0);
 6849   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6850   ins_encode();
 6851   ins_pipe(empty);
 6852 %}
 6853 
 6854 instruct membar_storestore() %{
 6855   match(MemBarStoreStore);
 6856   match(StoreStoreFence);
 6857   ins_cost(0);
 6858 
 6859   size(0);
 6860   format %{ "MEMBAR-storestore (empty encoding)" %}
 6861   ins_encode( );
 6862   ins_pipe(empty);
 6863 %}
 6864 
 6865 //----------Move Instructions--------------------------------------------------
 6866 
 6867 instruct castX2P(rRegP dst, rRegL src)
 6868 %{
 6869   match(Set dst (CastX2P src));
 6870 
 6871   format %{ "movq    $dst, $src\t# long->ptr" %}
 6872   ins_encode %{
 6873     if ($dst$$reg != $src$$reg) {
 6874       __ movptr($dst$$Register, $src$$Register);
 6875     }
 6876   %}
 6877   ins_pipe(ialu_reg_reg); // XXX
 6878 %}
 6879 
 6880 instruct castP2X(rRegL dst, rRegP src)
 6881 %{
 6882   match(Set dst (CastP2X src));
 6883 
 6884   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6885   ins_encode %{
 6886     if ($dst$$reg != $src$$reg) {
 6887       __ movptr($dst$$Register, $src$$Register);
 6888     }
 6889   %}
 6890   ins_pipe(ialu_reg_reg); // XXX
 6891 %}
 6892 
 6893 // Convert oop into int for vectors alignment masking
 6894 instruct convP2I(rRegI dst, rRegP src)
 6895 %{
 6896   match(Set dst (ConvL2I (CastP2X src)));
 6897 
 6898   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6899   ins_encode %{
 6900     __ movl($dst$$Register, $src$$Register);
 6901   %}
 6902   ins_pipe(ialu_reg_reg); // XXX
 6903 %}
 6904 
 6905 // Convert compressed oop into int for vectors alignment masking
 6906 // in case of 32bit oops (heap < 4Gb).
 6907 instruct convN2I(rRegI dst, rRegN src)
 6908 %{
 6909   predicate(CompressedOops::shift() == 0);
 6910   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6911 
 6912   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6913   ins_encode %{
 6914     __ movl($dst$$Register, $src$$Register);
 6915   %}
 6916   ins_pipe(ialu_reg_reg); // XXX
 6917 %}
 6918 
 6919 // Convert oop pointer into compressed form
 6920 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6921   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6922   match(Set dst (EncodeP src));
 6923   effect(KILL cr);
 6924   format %{ "encode_heap_oop $dst,$src" %}
 6925   ins_encode %{
 6926     Register s = $src$$Register;
 6927     Register d = $dst$$Register;
 6928     if (s != d) {
 6929       __ movq(d, s);
 6930     }
 6931     __ encode_heap_oop(d);
 6932   %}
 6933   ins_pipe(ialu_reg_long);
 6934 %}
 6935 
 6936 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6937   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6938   match(Set dst (EncodeP src));
 6939   effect(KILL cr);
 6940   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6941   ins_encode %{
 6942     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6943   %}
 6944   ins_pipe(ialu_reg_long);
 6945 %}
 6946 
 6947 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6948   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6949             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6950   match(Set dst (DecodeN src));
 6951   effect(KILL cr);
 6952   format %{ "decode_heap_oop $dst,$src" %}
 6953   ins_encode %{
 6954     Register s = $src$$Register;
 6955     Register d = $dst$$Register;
 6956     if (s != d) {
 6957       __ movq(d, s);
 6958     }
 6959     __ decode_heap_oop(d);
 6960   %}
 6961   ins_pipe(ialu_reg_long);
 6962 %}
 6963 
 6964 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6965   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6966             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6967   match(Set dst (DecodeN src));
 6968   effect(KILL cr);
 6969   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6970   ins_encode %{
 6971     Register s = $src$$Register;
 6972     Register d = $dst$$Register;
 6973     if (s != d) {
 6974       __ decode_heap_oop_not_null(d, s);
 6975     } else {
 6976       __ decode_heap_oop_not_null(d);
 6977     }
 6978   %}
 6979   ins_pipe(ialu_reg_long);
 6980 %}
 6981 
 6982 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6983   match(Set dst (EncodePKlass src));
 6984   effect(TEMP dst, KILL cr);
 6985   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6986   ins_encode %{
 6987     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6988   %}
 6989   ins_pipe(ialu_reg_long);
 6990 %}
 6991 
 6992 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6993   match(Set dst (DecodeNKlass src));
 6994   effect(TEMP dst, KILL cr);
 6995   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6996   ins_encode %{
 6997     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6998   %}
 6999   ins_pipe(ialu_reg_long);
 7000 %}
 7001 
 7002 //----------Conditional Move---------------------------------------------------
 7003 // Jump
 7004 // dummy instruction for generating temp registers
 7005 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7006   match(Jump (LShiftL switch_val shift));
 7007   ins_cost(350);
 7008   predicate(false);
 7009   effect(TEMP dest);
 7010 
 7011   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7012             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7013   ins_encode %{
 7014     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7015     // to do that and the compiler is using that register as one it can allocate.
 7016     // So we build it all by hand.
 7017     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7018     // ArrayAddress dispatch(table, index);
 7019     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7020     __ lea($dest$$Register, $constantaddress);
 7021     __ jmp(dispatch);
 7022   %}
 7023   ins_pipe(pipe_jmp);
 7024 %}
 7025 
 7026 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7027   match(Jump (AddL (LShiftL switch_val shift) offset));
 7028   ins_cost(350);
 7029   effect(TEMP dest);
 7030 
 7031   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7032             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7033   ins_encode %{
 7034     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7035     // to do that and the compiler is using that register as one it can allocate.
 7036     // So we build it all by hand.
 7037     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7038     // ArrayAddress dispatch(table, index);
 7039     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7040     __ lea($dest$$Register, $constantaddress);
 7041     __ jmp(dispatch);
 7042   %}
 7043   ins_pipe(pipe_jmp);
 7044 %}
 7045 
 7046 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7047   match(Jump switch_val);
 7048   ins_cost(350);
 7049   effect(TEMP dest);
 7050 
 7051   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7052             "jmp     [$dest + $switch_val]\n\t" %}
 7053   ins_encode %{
 7054     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7055     // to do that and the compiler is using that register as one it can allocate.
 7056     // So we build it all by hand.
 7057     // Address index(noreg, switch_reg, Address::times_1);
 7058     // ArrayAddress dispatch(table, index);
 7059     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7060     __ lea($dest$$Register, $constantaddress);
 7061     __ jmp(dispatch);
 7062   %}
 7063   ins_pipe(pipe_jmp);
 7064 %}
 7065 
 7066 // Conditional move
 7067 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7068 %{
 7069   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7070   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7071 
 7072   ins_cost(100); // XXX
 7073   format %{ "setbn$cop $dst\t# signed, int" %}
 7074   ins_encode %{
 7075     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7076     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7077   %}
 7078   ins_pipe(ialu_reg);
 7079 %}
 7080 
 7081 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7082 %{
 7083   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7084 
 7085   ins_cost(200); // XXX
 7086   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7087   ins_encode %{
 7088     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7089   %}
 7090   ins_pipe(pipe_cmov_reg);
 7091 %}
 7092 
 7093 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7094 %{
 7095   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7096   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7097 
 7098   ins_cost(100); // XXX
 7099   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7100   ins_encode %{
 7101     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7102     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7103   %}
 7104   ins_pipe(ialu_reg);
 7105 %}
 7106 
 7107 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7108   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7109 
 7110   ins_cost(200); // XXX
 7111   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7112   ins_encode %{
 7113     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7114   %}
 7115   ins_pipe(pipe_cmov_reg);
 7116 %}
 7117 
 7118 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7119 %{
 7120   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7121   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7122 
 7123   ins_cost(100); // XXX
 7124   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7125   ins_encode %{
 7126     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7127     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7128   %}
 7129   ins_pipe(ialu_reg);
 7130 %}
 7131 
 7132 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7133   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7134   ins_cost(200);
 7135   expand %{
 7136     cmovI_regU(cop, cr, dst, src);
 7137   %}
 7138 %}
 7139 
 7140 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7141   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7142   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7143 
 7144   ins_cost(200); // XXX
 7145   format %{ "cmovpl  $dst, $src\n\t"
 7146             "cmovnel $dst, $src" %}
 7147   ins_encode %{
 7148     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7149     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7150   %}
 7151   ins_pipe(pipe_cmov_reg);
 7152 %}
 7153 
 7154 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7155 // inputs of the CMove
 7156 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7157   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7158   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7159 
 7160   ins_cost(200); // XXX
 7161   format %{ "cmovpl  $dst, $src\n\t"
 7162             "cmovnel $dst, $src" %}
 7163   ins_encode %{
 7164     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7165     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7166   %}
 7167   ins_pipe(pipe_cmov_reg);
 7168 %}
 7169 
 7170 // Conditional move
 7171 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7172   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7173 
 7174   ins_cost(250); // XXX
 7175   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7176   ins_encode %{
 7177     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7178   %}
 7179   ins_pipe(pipe_cmov_mem);
 7180 %}
 7181 
 7182 // Conditional move
 7183 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7184 %{
 7185   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7186 
 7187   ins_cost(250); // XXX
 7188   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7189   ins_encode %{
 7190     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7191   %}
 7192   ins_pipe(pipe_cmov_mem);
 7193 %}
 7194 
 7195 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7196   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7197   ins_cost(250);
 7198   expand %{
 7199     cmovI_memU(cop, cr, dst, src);
 7200   %}
 7201 %}
 7202 
 7203 // Conditional move
 7204 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7205 %{
 7206   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7207 
 7208   ins_cost(200); // XXX
 7209   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7210   ins_encode %{
 7211     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7212   %}
 7213   ins_pipe(pipe_cmov_reg);
 7214 %}
 7215 
 7216 // Conditional move
 7217 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7218 %{
 7219   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7220 
 7221   ins_cost(200); // XXX
 7222   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7223   ins_encode %{
 7224     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7225   %}
 7226   ins_pipe(pipe_cmov_reg);
 7227 %}
 7228 
 7229 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7230   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7231   ins_cost(200);
 7232   expand %{
 7233     cmovN_regU(cop, cr, dst, src);
 7234   %}
 7235 %}
 7236 
 7237 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7238   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7239   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7240 
 7241   ins_cost(200); // XXX
 7242   format %{ "cmovpl  $dst, $src\n\t"
 7243             "cmovnel $dst, $src" %}
 7244   ins_encode %{
 7245     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7246     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7247   %}
 7248   ins_pipe(pipe_cmov_reg);
 7249 %}
 7250 
 7251 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7252 // inputs of the CMove
 7253 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7254   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7255   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7256 
 7257   ins_cost(200); // XXX
 7258   format %{ "cmovpl  $dst, $src\n\t"
 7259             "cmovnel $dst, $src" %}
 7260   ins_encode %{
 7261     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7262     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7263   %}
 7264   ins_pipe(pipe_cmov_reg);
 7265 %}
 7266 
 7267 // Conditional move
 7268 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7269 %{
 7270   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7271 
 7272   ins_cost(200); // XXX
 7273   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7274   ins_encode %{
 7275     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7276   %}
 7277   ins_pipe(pipe_cmov_reg);  // XXX
 7278 %}
 7279 
 7280 // Conditional move
 7281 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7282 %{
 7283   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7284 
 7285   ins_cost(200); // XXX
 7286   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7287   ins_encode %{
 7288     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7289   %}
 7290   ins_pipe(pipe_cmov_reg); // XXX
 7291 %}
 7292 
 7293 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7294   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7295   ins_cost(200);
 7296   expand %{
 7297     cmovP_regU(cop, cr, dst, src);
 7298   %}
 7299 %}
 7300 
 7301 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7302   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7303   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7304 
 7305   ins_cost(200); // XXX
 7306   format %{ "cmovpq  $dst, $src\n\t"
 7307             "cmovneq $dst, $src" %}
 7308   ins_encode %{
 7309     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7310     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7311   %}
 7312   ins_pipe(pipe_cmov_reg);
 7313 %}
 7314 
 7315 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7316 // inputs of the CMove
 7317 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7318   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7319   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7320 
 7321   ins_cost(200); // XXX
 7322   format %{ "cmovpq  $dst, $src\n\t"
 7323             "cmovneq $dst, $src" %}
 7324   ins_encode %{
 7325     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7326     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7327   %}
 7328   ins_pipe(pipe_cmov_reg);
 7329 %}
 7330 
 7331 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7332 // correctly meets the two pointer arguments; one is an incoming
 7333 // register but the other is a memory operand.  ALSO appears to
 7334 // be buggy with implicit null checks.
 7335 //
 7336 //// Conditional move
 7337 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7338 //%{
 7339 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7340 //  ins_cost(250);
 7341 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7342 //  opcode(0x0F,0x40);
 7343 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7344 //  ins_pipe( pipe_cmov_mem );
 7345 //%}
 7346 //
 7347 //// Conditional move
 7348 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7349 //%{
 7350 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7351 //  ins_cost(250);
 7352 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7353 //  opcode(0x0F,0x40);
 7354 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7355 //  ins_pipe( pipe_cmov_mem );
 7356 //%}
 7357 
 7358 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7359 %{
 7360   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7361   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7362 
 7363   ins_cost(100); // XXX
 7364   format %{ "setbn$cop $dst\t# signed, long" %}
 7365   ins_encode %{
 7366     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7367     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7368   %}
 7369   ins_pipe(ialu_reg);
 7370 %}
 7371 
 7372 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7373 %{
 7374   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7375 
 7376   ins_cost(200); // XXX
 7377   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7378   ins_encode %{
 7379     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7380   %}
 7381   ins_pipe(pipe_cmov_reg);  // XXX
 7382 %}
 7383 
 7384 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7385 %{
 7386   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7387 
 7388   ins_cost(200); // XXX
 7389   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7390   ins_encode %{
 7391     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7392   %}
 7393   ins_pipe(pipe_cmov_mem);  // XXX
 7394 %}
 7395 
 7396 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7397 %{
 7398   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7399   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7400 
 7401   ins_cost(100); // XXX
 7402   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7403   ins_encode %{
 7404     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7405     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7406   %}
 7407   ins_pipe(ialu_reg);
 7408 %}
 7409 
 7410 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7411 %{
 7412   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7413 
 7414   ins_cost(200); // XXX
 7415   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7416   ins_encode %{
 7417     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7418   %}
 7419   ins_pipe(pipe_cmov_reg); // XXX
 7420 %}
 7421 
 7422 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7423 %{
 7424   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7425   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7426 
 7427   ins_cost(100); // XXX
 7428   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7429   ins_encode %{
 7430     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7431     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7432   %}
 7433   ins_pipe(ialu_reg);
 7434 %}
 7435 
 7436 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7437   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7438   ins_cost(200);
 7439   expand %{
 7440     cmovL_regU(cop, cr, dst, src);
 7441   %}
 7442 %}
 7443 
 7444 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7445   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7446   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7447 
 7448   ins_cost(200); // XXX
 7449   format %{ "cmovpq  $dst, $src\n\t"
 7450             "cmovneq $dst, $src" %}
 7451   ins_encode %{
 7452     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7453     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7454   %}
 7455   ins_pipe(pipe_cmov_reg);
 7456 %}
 7457 
 7458 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7459 // inputs of the CMove
 7460 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7461   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7462   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7463 
 7464   ins_cost(200); // XXX
 7465   format %{ "cmovpq  $dst, $src\n\t"
 7466             "cmovneq $dst, $src" %}
 7467   ins_encode %{
 7468     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7469     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7470   %}
 7471   ins_pipe(pipe_cmov_reg);
 7472 %}
 7473 
 7474 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7475 %{
 7476   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7477 
 7478   ins_cost(200); // XXX
 7479   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7480   ins_encode %{
 7481     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7482   %}
 7483   ins_pipe(pipe_cmov_mem); // XXX
 7484 %}
 7485 
 7486 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7487   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7488   ins_cost(200);
 7489   expand %{
 7490     cmovL_memU(cop, cr, dst, src);
 7491   %}
 7492 %}
 7493 
 7494 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7495 %{
 7496   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7497 
 7498   ins_cost(200); // XXX
 7499   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7500             "movss     $dst, $src\n"
 7501     "skip:" %}
 7502   ins_encode %{
 7503     Label Lskip;
 7504     // Invert sense of branch from sense of CMOV
 7505     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7506     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7507     __ bind(Lskip);
 7508   %}
 7509   ins_pipe(pipe_slow);
 7510 %}
 7511 
 7512 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7513 // %{
 7514 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7515 
 7516 //   ins_cost(200); // XXX
 7517 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7518 //             "movss     $dst, $src\n"
 7519 //     "skip:" %}
 7520 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7521 //   ins_pipe(pipe_slow);
 7522 // %}
 7523 
 7524 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7525 %{
 7526   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7527 
 7528   ins_cost(200); // XXX
 7529   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7530             "movss     $dst, $src\n"
 7531     "skip:" %}
 7532   ins_encode %{
 7533     Label Lskip;
 7534     // Invert sense of branch from sense of CMOV
 7535     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7536     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7537     __ bind(Lskip);
 7538   %}
 7539   ins_pipe(pipe_slow);
 7540 %}
 7541 
 7542 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7543   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7544   ins_cost(200);
 7545   expand %{
 7546     cmovF_regU(cop, cr, dst, src);
 7547   %}
 7548 %}
 7549 
 7550 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7551 %{
 7552   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7553 
 7554   ins_cost(200); // XXX
 7555   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7556             "movsd     $dst, $src\n"
 7557     "skip:" %}
 7558   ins_encode %{
 7559     Label Lskip;
 7560     // Invert sense of branch from sense of CMOV
 7561     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7562     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7563     __ bind(Lskip);
 7564   %}
 7565   ins_pipe(pipe_slow);
 7566 %}
 7567 
 7568 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7569 %{
 7570   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7571 
 7572   ins_cost(200); // XXX
 7573   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7574             "movsd     $dst, $src\n"
 7575     "skip:" %}
 7576   ins_encode %{
 7577     Label Lskip;
 7578     // Invert sense of branch from sense of CMOV
 7579     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7580     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7581     __ bind(Lskip);
 7582   %}
 7583   ins_pipe(pipe_slow);
 7584 %}
 7585 
 7586 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7587   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7588   ins_cost(200);
 7589   expand %{
 7590     cmovD_regU(cop, cr, dst, src);
 7591   %}
 7592 %}
 7593 
 7594 //----------Arithmetic Instructions--------------------------------------------
 7595 //----------Addition Instructions----------------------------------------------
 7596 
 7597 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7598 %{
 7599   match(Set dst (AddI dst src));
 7600   effect(KILL cr);
 7601 
 7602   format %{ "addl    $dst, $src\t# int" %}
 7603   ins_encode %{
 7604     __ addl($dst$$Register, $src$$Register);
 7605   %}
 7606   ins_pipe(ialu_reg_reg);
 7607 %}
 7608 
 7609 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7610 %{
 7611   match(Set dst (AddI dst src));
 7612   effect(KILL cr);
 7613 
 7614   format %{ "addl    $dst, $src\t# int" %}
 7615   ins_encode %{
 7616     __ addl($dst$$Register, $src$$constant);
 7617   %}
 7618   ins_pipe( ialu_reg );
 7619 %}
 7620 
 7621 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7622 %{
 7623   match(Set dst (AddI dst (LoadI src)));
 7624   effect(KILL cr);
 7625 
 7626   ins_cost(125); // XXX
 7627   format %{ "addl    $dst, $src\t# int" %}
 7628   ins_encode %{
 7629     __ addl($dst$$Register, $src$$Address);
 7630   %}
 7631   ins_pipe(ialu_reg_mem);
 7632 %}
 7633 
 7634 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7635 %{
 7636   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7637   effect(KILL cr);
 7638 
 7639   ins_cost(150); // XXX
 7640   format %{ "addl    $dst, $src\t# int" %}
 7641   ins_encode %{
 7642     __ addl($dst$$Address, $src$$Register);
 7643   %}
 7644   ins_pipe(ialu_mem_reg);
 7645 %}
 7646 
 7647 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7648 %{
 7649   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7650   effect(KILL cr);
 7651 
 7652   ins_cost(125); // XXX
 7653   format %{ "addl    $dst, $src\t# int" %}
 7654   ins_encode %{
 7655     __ addl($dst$$Address, $src$$constant);
 7656   %}
 7657   ins_pipe(ialu_mem_imm);
 7658 %}
 7659 
 7660 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7661 %{
 7662   predicate(UseIncDec);
 7663   match(Set dst (AddI dst src));
 7664   effect(KILL cr);
 7665 
 7666   format %{ "incl    $dst\t# int" %}
 7667   ins_encode %{
 7668     __ incrementl($dst$$Register);
 7669   %}
 7670   ins_pipe(ialu_reg);
 7671 %}
 7672 
 7673 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7674 %{
 7675   predicate(UseIncDec);
 7676   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7677   effect(KILL cr);
 7678 
 7679   ins_cost(125); // XXX
 7680   format %{ "incl    $dst\t# int" %}
 7681   ins_encode %{
 7682     __ incrementl($dst$$Address);
 7683   %}
 7684   ins_pipe(ialu_mem_imm);
 7685 %}
 7686 
 7687 // XXX why does that use AddI
 7688 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7689 %{
 7690   predicate(UseIncDec);
 7691   match(Set dst (AddI dst src));
 7692   effect(KILL cr);
 7693 
 7694   format %{ "decl    $dst\t# int" %}
 7695   ins_encode %{
 7696     __ decrementl($dst$$Register);
 7697   %}
 7698   ins_pipe(ialu_reg);
 7699 %}
 7700 
 7701 // XXX why does that use AddI
 7702 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7703 %{
 7704   predicate(UseIncDec);
 7705   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7706   effect(KILL cr);
 7707 
 7708   ins_cost(125); // XXX
 7709   format %{ "decl    $dst\t# int" %}
 7710   ins_encode %{
 7711     __ decrementl($dst$$Address);
 7712   %}
 7713   ins_pipe(ialu_mem_imm);
 7714 %}
 7715 
 7716 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7717 %{
 7718   predicate(VM_Version::supports_fast_2op_lea());
 7719   match(Set dst (AddI (LShiftI index scale) disp));
 7720 
 7721   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7722   ins_encode %{
 7723     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7724     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7725   %}
 7726   ins_pipe(ialu_reg_reg);
 7727 %}
 7728 
 7729 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7730 %{
 7731   predicate(VM_Version::supports_fast_3op_lea());
 7732   match(Set dst (AddI (AddI base index) disp));
 7733 
 7734   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7735   ins_encode %{
 7736     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7737   %}
 7738   ins_pipe(ialu_reg_reg);
 7739 %}
 7740 
 7741 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7742 %{
 7743   predicate(VM_Version::supports_fast_2op_lea());
 7744   match(Set dst (AddI base (LShiftI index scale)));
 7745 
 7746   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7747   ins_encode %{
 7748     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7749     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7750   %}
 7751   ins_pipe(ialu_reg_reg);
 7752 %}
 7753 
 7754 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7755 %{
 7756   predicate(VM_Version::supports_fast_3op_lea());
 7757   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7758 
 7759   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7760   ins_encode %{
 7761     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7762     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7763   %}
 7764   ins_pipe(ialu_reg_reg);
 7765 %}
 7766 
 7767 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7768 %{
 7769   match(Set dst (AddL dst src));
 7770   effect(KILL cr);
 7771 
 7772   format %{ "addq    $dst, $src\t# long" %}
 7773   ins_encode %{
 7774     __ addq($dst$$Register, $src$$Register);
 7775   %}
 7776   ins_pipe(ialu_reg_reg);
 7777 %}
 7778 
 7779 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7780 %{
 7781   match(Set dst (AddL dst src));
 7782   effect(KILL cr);
 7783 
 7784   format %{ "addq    $dst, $src\t# long" %}
 7785   ins_encode %{
 7786     __ addq($dst$$Register, $src$$constant);
 7787   %}
 7788   ins_pipe( ialu_reg );
 7789 %}
 7790 
 7791 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7792 %{
 7793   match(Set dst (AddL dst (LoadL src)));
 7794   effect(KILL cr);
 7795 
 7796   ins_cost(125); // XXX
 7797   format %{ "addq    $dst, $src\t# long" %}
 7798   ins_encode %{
 7799     __ addq($dst$$Register, $src$$Address);
 7800   %}
 7801   ins_pipe(ialu_reg_mem);
 7802 %}
 7803 
 7804 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7805 %{
 7806   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7807   effect(KILL cr);
 7808 
 7809   ins_cost(150); // XXX
 7810   format %{ "addq    $dst, $src\t# long" %}
 7811   ins_encode %{
 7812     __ addq($dst$$Address, $src$$Register);
 7813   %}
 7814   ins_pipe(ialu_mem_reg);
 7815 %}
 7816 
 7817 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7818 %{
 7819   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7820   effect(KILL cr);
 7821 
 7822   ins_cost(125); // XXX
 7823   format %{ "addq    $dst, $src\t# long" %}
 7824   ins_encode %{
 7825     __ addq($dst$$Address, $src$$constant);
 7826   %}
 7827   ins_pipe(ialu_mem_imm);
 7828 %}
 7829 
 7830 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7831 %{
 7832   predicate(UseIncDec);
 7833   match(Set dst (AddL dst src));
 7834   effect(KILL cr);
 7835 
 7836   format %{ "incq    $dst\t# long" %}
 7837   ins_encode %{
 7838     __ incrementq($dst$$Register);
 7839   %}
 7840   ins_pipe(ialu_reg);
 7841 %}
 7842 
 7843 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7844 %{
 7845   predicate(UseIncDec);
 7846   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7847   effect(KILL cr);
 7848 
 7849   ins_cost(125); // XXX
 7850   format %{ "incq    $dst\t# long" %}
 7851   ins_encode %{
 7852     __ incrementq($dst$$Address);
 7853   %}
 7854   ins_pipe(ialu_mem_imm);
 7855 %}
 7856 
 7857 // XXX why does that use AddL
 7858 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7859 %{
 7860   predicate(UseIncDec);
 7861   match(Set dst (AddL dst src));
 7862   effect(KILL cr);
 7863 
 7864   format %{ "decq    $dst\t# long" %}
 7865   ins_encode %{
 7866     __ decrementq($dst$$Register);
 7867   %}
 7868   ins_pipe(ialu_reg);
 7869 %}
 7870 
 7871 // XXX why does that use AddL
 7872 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7873 %{
 7874   predicate(UseIncDec);
 7875   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7876   effect(KILL cr);
 7877 
 7878   ins_cost(125); // XXX
 7879   format %{ "decq    $dst\t# long" %}
 7880   ins_encode %{
 7881     __ decrementq($dst$$Address);
 7882   %}
 7883   ins_pipe(ialu_mem_imm);
 7884 %}
 7885 
 7886 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7887 %{
 7888   predicate(VM_Version::supports_fast_2op_lea());
 7889   match(Set dst (AddL (LShiftL index scale) disp));
 7890 
 7891   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7892   ins_encode %{
 7893     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7894     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7895   %}
 7896   ins_pipe(ialu_reg_reg);
 7897 %}
 7898 
 7899 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7900 %{
 7901   predicate(VM_Version::supports_fast_3op_lea());
 7902   match(Set dst (AddL (AddL base index) disp));
 7903 
 7904   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7905   ins_encode %{
 7906     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7907   %}
 7908   ins_pipe(ialu_reg_reg);
 7909 %}
 7910 
 7911 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7912 %{
 7913   predicate(VM_Version::supports_fast_2op_lea());
 7914   match(Set dst (AddL base (LShiftL index scale)));
 7915 
 7916   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7917   ins_encode %{
 7918     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7919     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7920   %}
 7921   ins_pipe(ialu_reg_reg);
 7922 %}
 7923 
 7924 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7925 %{
 7926   predicate(VM_Version::supports_fast_3op_lea());
 7927   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7928 
 7929   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7930   ins_encode %{
 7931     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7932     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7933   %}
 7934   ins_pipe(ialu_reg_reg);
 7935 %}
 7936 
 7937 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7938 %{
 7939   match(Set dst (AddP dst src));
 7940   effect(KILL cr);
 7941 
 7942   format %{ "addq    $dst, $src\t# ptr" %}
 7943   ins_encode %{
 7944     __ addq($dst$$Register, $src$$Register);
 7945   %}
 7946   ins_pipe(ialu_reg_reg);
 7947 %}
 7948 
 7949 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7950 %{
 7951   match(Set dst (AddP dst src));
 7952   effect(KILL cr);
 7953 
 7954   format %{ "addq    $dst, $src\t# ptr" %}
 7955   ins_encode %{
 7956     __ addq($dst$$Register, $src$$constant);
 7957   %}
 7958   ins_pipe( ialu_reg );
 7959 %}
 7960 
 7961 // XXX addP mem ops ????
 7962 
 7963 instruct checkCastPP(rRegP dst)
 7964 %{
 7965   match(Set dst (CheckCastPP dst));
 7966 
 7967   size(0);
 7968   format %{ "# checkcastPP of $dst" %}
 7969   ins_encode(/* empty encoding */);
 7970   ins_pipe(empty);
 7971 %}
 7972 
 7973 instruct castPP(rRegP dst)
 7974 %{
 7975   match(Set dst (CastPP dst));
 7976 
 7977   size(0);
 7978   format %{ "# castPP of $dst" %}
 7979   ins_encode(/* empty encoding */);
 7980   ins_pipe(empty);
 7981 %}
 7982 
 7983 instruct castII(rRegI dst)
 7984 %{
 7985   match(Set dst (CastII dst));
 7986 
 7987   size(0);
 7988   format %{ "# castII of $dst" %}
 7989   ins_encode(/* empty encoding */);
 7990   ins_cost(0);
 7991   ins_pipe(empty);
 7992 %}
 7993 
 7994 instruct castLL(rRegL dst)
 7995 %{
 7996   match(Set dst (CastLL dst));
 7997 
 7998   size(0);
 7999   format %{ "# castLL of $dst" %}
 8000   ins_encode(/* empty encoding */);
 8001   ins_cost(0);
 8002   ins_pipe(empty);
 8003 %}
 8004 
 8005 instruct castFF(regF dst)
 8006 %{
 8007   match(Set dst (CastFF dst));
 8008 
 8009   size(0);
 8010   format %{ "# castFF of $dst" %}
 8011   ins_encode(/* empty encoding */);
 8012   ins_cost(0);
 8013   ins_pipe(empty);
 8014 %}
 8015 
 8016 instruct castDD(regD dst)
 8017 %{
 8018   match(Set dst (CastDD dst));
 8019 
 8020   size(0);
 8021   format %{ "# castDD of $dst" %}
 8022   ins_encode(/* empty encoding */);
 8023   ins_cost(0);
 8024   ins_pipe(empty);
 8025 %}
 8026 
 8027 // LoadP-locked same as a regular LoadP when used with compare-swap
 8028 instruct loadPLocked(rRegP dst, memory mem)
 8029 %{
 8030   match(Set dst (LoadPLocked mem));
 8031 
 8032   ins_cost(125); // XXX
 8033   format %{ "movq    $dst, $mem\t# ptr locked" %}
 8034   ins_encode %{
 8035     __ movq($dst$$Register, $mem$$Address);
 8036   %}
 8037   ins_pipe(ialu_reg_mem); // XXX
 8038 %}
 8039 
 8040 // Conditional-store of the updated heap-top.
 8041 // Used during allocation of the shared heap.
 8042 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 8043 
 8044 instruct storePConditional(memory heap_top_ptr,
 8045                            rax_RegP oldval, rRegP newval,
 8046                            rFlagsReg cr)
 8047 %{
 8048   predicate(n->as_LoadStore()->barrier_data() == 0);
 8049   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8050 
 8051   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 8052             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 8053   ins_encode %{
 8054     __ lock();
 8055     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 8056   %}
 8057   ins_pipe(pipe_cmpxchg);
 8058 %}
 8059 
 8060 // Conditional-store of an int value.
 8061 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 8062 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 8063 %{
 8064   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 8065   effect(KILL oldval);
 8066 
 8067   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 8068   opcode(0x0F, 0xB1);
 8069   ins_encode(lock_prefix,
 8070              REX_reg_mem(newval, mem),
 8071              OpcP, OpcS,
 8072              reg_mem(newval, mem));
 8073   ins_pipe(pipe_cmpxchg);
 8074 %}
 8075 
 8076 // Conditional-store of a long value.
 8077 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 8078 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 8079 %{
 8080   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8081   effect(KILL oldval);
 8082 
 8083   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 8084   ins_encode %{
 8085     __ lock();
 8086     __ cmpxchgq($newval$$Register, $mem$$Address);
 8087   %}
 8088   ins_pipe(pipe_cmpxchg);
 8089 %}
 8090 
 8091 
 8092 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8093 instruct compareAndSwapP(rRegI res,
 8094                          memory mem_ptr,
 8095                          rax_RegP oldval, rRegP newval,
 8096                          rFlagsReg cr)
 8097 %{
 8098   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8099   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8100   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8101   effect(KILL cr, KILL oldval);
 8102 
 8103   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8104             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8105             "sete    $res\n\t"
 8106             "movzbl  $res, $res" %}
 8107   ins_encode %{
 8108     __ lock();
 8109     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8110     __ sete($res$$Register);
 8111     __ movzbl($res$$Register, $res$$Register);
 8112   %}
 8113   ins_pipe( pipe_cmpxchg );
 8114 %}
 8115 
 8116 instruct compareAndSwapL(rRegI res,
 8117                          memory mem_ptr,
 8118                          rax_RegL oldval, rRegL newval,
 8119                          rFlagsReg cr)
 8120 %{
 8121   predicate(VM_Version::supports_cx8());
 8122   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8123   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8124   effect(KILL cr, KILL oldval);
 8125 
 8126   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8127             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8128             "sete    $res\n\t"
 8129             "movzbl  $res, $res" %}
 8130   ins_encode %{
 8131     __ lock();
 8132     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8133     __ sete($res$$Register);
 8134     __ movzbl($res$$Register, $res$$Register);
 8135   %}
 8136   ins_pipe( pipe_cmpxchg );
 8137 %}
 8138 
 8139 instruct compareAndSwapI(rRegI res,
 8140                          memory mem_ptr,
 8141                          rax_RegI oldval, rRegI newval,
 8142                          rFlagsReg cr)
 8143 %{
 8144   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8145   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8146   effect(KILL cr, KILL oldval);
 8147 
 8148   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8149             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8150             "sete    $res\n\t"
 8151             "movzbl  $res, $res" %}
 8152   ins_encode %{
 8153     __ lock();
 8154     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8155     __ sete($res$$Register);
 8156     __ movzbl($res$$Register, $res$$Register);
 8157   %}
 8158   ins_pipe( pipe_cmpxchg );
 8159 %}
 8160 
 8161 instruct compareAndSwapB(rRegI res,
 8162                          memory mem_ptr,
 8163                          rax_RegI oldval, rRegI newval,
 8164                          rFlagsReg cr)
 8165 %{
 8166   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8167   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8168   effect(KILL cr, KILL oldval);
 8169 
 8170   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8171             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8172             "sete    $res\n\t"
 8173             "movzbl  $res, $res" %}
 8174   ins_encode %{
 8175     __ lock();
 8176     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8177     __ sete($res$$Register);
 8178     __ movzbl($res$$Register, $res$$Register);
 8179   %}
 8180   ins_pipe( pipe_cmpxchg );
 8181 %}
 8182 
 8183 instruct compareAndSwapS(rRegI res,
 8184                          memory mem_ptr,
 8185                          rax_RegI oldval, rRegI newval,
 8186                          rFlagsReg cr)
 8187 %{
 8188   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8189   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8190   effect(KILL cr, KILL oldval);
 8191 
 8192   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8193             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8194             "sete    $res\n\t"
 8195             "movzbl  $res, $res" %}
 8196   ins_encode %{
 8197     __ lock();
 8198     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8199     __ sete($res$$Register);
 8200     __ movzbl($res$$Register, $res$$Register);
 8201   %}
 8202   ins_pipe( pipe_cmpxchg );
 8203 %}
 8204 
 8205 instruct compareAndSwapN(rRegI res,
 8206                           memory mem_ptr,
 8207                           rax_RegN oldval, rRegN newval,
 8208                           rFlagsReg cr) %{
 8209   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8210   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8211   effect(KILL cr, KILL oldval);
 8212 
 8213   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8214             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8215             "sete    $res\n\t"
 8216             "movzbl  $res, $res" %}
 8217   ins_encode %{
 8218     __ lock();
 8219     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8220     __ sete($res$$Register);
 8221     __ movzbl($res$$Register, $res$$Register);
 8222   %}
 8223   ins_pipe( pipe_cmpxchg );
 8224 %}
 8225 
 8226 instruct compareAndExchangeB(
 8227                          memory mem_ptr,
 8228                          rax_RegI oldval, rRegI newval,
 8229                          rFlagsReg cr)
 8230 %{
 8231   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8232   effect(KILL cr);
 8233 
 8234   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8235             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8236   ins_encode %{
 8237     __ lock();
 8238     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8239   %}
 8240   ins_pipe( pipe_cmpxchg );
 8241 %}
 8242 
 8243 instruct compareAndExchangeS(
 8244                          memory mem_ptr,
 8245                          rax_RegI oldval, rRegI newval,
 8246                          rFlagsReg cr)
 8247 %{
 8248   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8249   effect(KILL cr);
 8250 
 8251   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8252             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8253   ins_encode %{
 8254     __ lock();
 8255     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8256   %}
 8257   ins_pipe( pipe_cmpxchg );
 8258 %}
 8259 
 8260 instruct compareAndExchangeI(
 8261                          memory mem_ptr,
 8262                          rax_RegI oldval, rRegI newval,
 8263                          rFlagsReg cr)
 8264 %{
 8265   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8266   effect(KILL cr);
 8267 
 8268   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8269             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8270   ins_encode %{
 8271     __ lock();
 8272     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8273   %}
 8274   ins_pipe( pipe_cmpxchg );
 8275 %}
 8276 
 8277 instruct compareAndExchangeL(
 8278                          memory mem_ptr,
 8279                          rax_RegL oldval, rRegL newval,
 8280                          rFlagsReg cr)
 8281 %{
 8282   predicate(VM_Version::supports_cx8());
 8283   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8284   effect(KILL cr);
 8285 
 8286   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8287             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8288   ins_encode %{
 8289     __ lock();
 8290     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8291   %}
 8292   ins_pipe( pipe_cmpxchg );
 8293 %}
 8294 
 8295 instruct compareAndExchangeN(
 8296                           memory mem_ptr,
 8297                           rax_RegN oldval, rRegN newval,
 8298                           rFlagsReg cr) %{
 8299   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8300   effect(KILL cr);
 8301 
 8302   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8303             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8304   ins_encode %{
 8305     __ lock();
 8306     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8307   %}
 8308   ins_pipe( pipe_cmpxchg );
 8309 %}
 8310 
 8311 instruct compareAndExchangeP(
 8312                          memory mem_ptr,
 8313                          rax_RegP oldval, rRegP newval,
 8314                          rFlagsReg cr)
 8315 %{
 8316   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8317   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8318   effect(KILL cr);
 8319 
 8320   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8321             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8322   ins_encode %{
 8323     __ lock();
 8324     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8325   %}
 8326   ins_pipe( pipe_cmpxchg );
 8327 %}
 8328 
 8329 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8330   predicate(n->as_LoadStore()->result_not_used());
 8331   match(Set dummy (GetAndAddB mem add));
 8332   effect(KILL cr);
 8333   format %{ "ADDB  [$mem],$add" %}
 8334   ins_encode %{
 8335     __ lock();
 8336     __ addb($mem$$Address, $add$$constant);
 8337   %}
 8338   ins_pipe( pipe_cmpxchg );
 8339 %}
 8340 
 8341 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8342   match(Set newval (GetAndAddB mem newval));
 8343   effect(KILL cr);
 8344   format %{ "XADDB  [$mem],$newval" %}
 8345   ins_encode %{
 8346     __ lock();
 8347     __ xaddb($mem$$Address, $newval$$Register);
 8348   %}
 8349   ins_pipe( pipe_cmpxchg );
 8350 %}
 8351 
 8352 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8353   predicate(n->as_LoadStore()->result_not_used());
 8354   match(Set dummy (GetAndAddS mem add));
 8355   effect(KILL cr);
 8356   format %{ "ADDW  [$mem],$add" %}
 8357   ins_encode %{
 8358     __ lock();
 8359     __ addw($mem$$Address, $add$$constant);
 8360   %}
 8361   ins_pipe( pipe_cmpxchg );
 8362 %}
 8363 
 8364 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8365   match(Set newval (GetAndAddS mem newval));
 8366   effect(KILL cr);
 8367   format %{ "XADDW  [$mem],$newval" %}
 8368   ins_encode %{
 8369     __ lock();
 8370     __ xaddw($mem$$Address, $newval$$Register);
 8371   %}
 8372   ins_pipe( pipe_cmpxchg );
 8373 %}
 8374 
 8375 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8376   predicate(n->as_LoadStore()->result_not_used());
 8377   match(Set dummy (GetAndAddI mem add));
 8378   effect(KILL cr);
 8379   format %{ "ADDL  [$mem],$add" %}
 8380   ins_encode %{
 8381     __ lock();
 8382     __ addl($mem$$Address, $add$$constant);
 8383   %}
 8384   ins_pipe( pipe_cmpxchg );
 8385 %}
 8386 
 8387 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8388   match(Set newval (GetAndAddI mem newval));
 8389   effect(KILL cr);
 8390   format %{ "XADDL  [$mem],$newval" %}
 8391   ins_encode %{
 8392     __ lock();
 8393     __ xaddl($mem$$Address, $newval$$Register);
 8394   %}
 8395   ins_pipe( pipe_cmpxchg );
 8396 %}
 8397 
 8398 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8399   predicate(n->as_LoadStore()->result_not_used());
 8400   match(Set dummy (GetAndAddL mem add));
 8401   effect(KILL cr);
 8402   format %{ "ADDQ  [$mem],$add" %}
 8403   ins_encode %{
 8404     __ lock();
 8405     __ addq($mem$$Address, $add$$constant);
 8406   %}
 8407   ins_pipe( pipe_cmpxchg );
 8408 %}
 8409 
 8410 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8411   match(Set newval (GetAndAddL mem newval));
 8412   effect(KILL cr);
 8413   format %{ "XADDQ  [$mem],$newval" %}
 8414   ins_encode %{
 8415     __ lock();
 8416     __ xaddq($mem$$Address, $newval$$Register);
 8417   %}
 8418   ins_pipe( pipe_cmpxchg );
 8419 %}
 8420 
 8421 instruct xchgB( memory mem, rRegI newval) %{
 8422   match(Set newval (GetAndSetB mem newval));
 8423   format %{ "XCHGB  $newval,[$mem]" %}
 8424   ins_encode %{
 8425     __ xchgb($newval$$Register, $mem$$Address);
 8426   %}
 8427   ins_pipe( pipe_cmpxchg );
 8428 %}
 8429 
 8430 instruct xchgS( memory mem, rRegI newval) %{
 8431   match(Set newval (GetAndSetS mem newval));
 8432   format %{ "XCHGW  $newval,[$mem]" %}
 8433   ins_encode %{
 8434     __ xchgw($newval$$Register, $mem$$Address);
 8435   %}
 8436   ins_pipe( pipe_cmpxchg );
 8437 %}
 8438 
 8439 instruct xchgI( memory mem, rRegI newval) %{
 8440   match(Set newval (GetAndSetI mem newval));
 8441   format %{ "XCHGL  $newval,[$mem]" %}
 8442   ins_encode %{
 8443     __ xchgl($newval$$Register, $mem$$Address);
 8444   %}
 8445   ins_pipe( pipe_cmpxchg );
 8446 %}
 8447 
 8448 instruct xchgL( memory mem, rRegL newval) %{
 8449   match(Set newval (GetAndSetL mem newval));
 8450   format %{ "XCHGL  $newval,[$mem]" %}
 8451   ins_encode %{
 8452     __ xchgq($newval$$Register, $mem$$Address);
 8453   %}
 8454   ins_pipe( pipe_cmpxchg );
 8455 %}
 8456 
 8457 instruct xchgP( memory mem, rRegP newval) %{
 8458   match(Set newval (GetAndSetP mem newval));
 8459   predicate(n->as_LoadStore()->barrier_data() == 0);
 8460   format %{ "XCHGQ  $newval,[$mem]" %}
 8461   ins_encode %{
 8462     __ xchgq($newval$$Register, $mem$$Address);
 8463   %}
 8464   ins_pipe( pipe_cmpxchg );
 8465 %}
 8466 
 8467 instruct xchgN( memory mem, rRegN newval) %{
 8468   match(Set newval (GetAndSetN mem newval));
 8469   format %{ "XCHGL  $newval,$mem]" %}
 8470   ins_encode %{
 8471     __ xchgl($newval$$Register, $mem$$Address);
 8472   %}
 8473   ins_pipe( pipe_cmpxchg );
 8474 %}
 8475 
 8476 //----------Abs Instructions-------------------------------------------
 8477 
 8478 // Integer Absolute Instructions
 8479 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8480 %{
 8481   match(Set dst (AbsI src));
 8482   effect(TEMP dst, TEMP tmp, KILL cr);
 8483   format %{ "movl $tmp, $src\n\t"
 8484             "sarl $tmp, 31\n\t"
 8485             "movl $dst, $src\n\t"
 8486             "xorl $dst, $tmp\n\t"
 8487             "subl $dst, $tmp\n"
 8488           %}
 8489   ins_encode %{
 8490     __ movl($tmp$$Register, $src$$Register);
 8491     __ sarl($tmp$$Register, 31);
 8492     __ movl($dst$$Register, $src$$Register);
 8493     __ xorl($dst$$Register, $tmp$$Register);
 8494     __ subl($dst$$Register, $tmp$$Register);
 8495   %}
 8496 
 8497   ins_pipe(ialu_reg_reg);
 8498 %}
 8499 
 8500 // Long Absolute Instructions
 8501 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8502 %{
 8503   match(Set dst (AbsL src));
 8504   effect(TEMP dst, TEMP tmp, KILL cr);
 8505   format %{ "movq $tmp, $src\n\t"
 8506             "sarq $tmp, 63\n\t"
 8507             "movq $dst, $src\n\t"
 8508             "xorq $dst, $tmp\n\t"
 8509             "subq $dst, $tmp\n"
 8510           %}
 8511   ins_encode %{
 8512     __ movq($tmp$$Register, $src$$Register);
 8513     __ sarq($tmp$$Register, 63);
 8514     __ movq($dst$$Register, $src$$Register);
 8515     __ xorq($dst$$Register, $tmp$$Register);
 8516     __ subq($dst$$Register, $tmp$$Register);
 8517   %}
 8518 
 8519   ins_pipe(ialu_reg_reg);
 8520 %}
 8521 
 8522 //----------Subtraction Instructions-------------------------------------------
 8523 
 8524 // Integer Subtraction Instructions
 8525 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8526 %{
 8527   match(Set dst (SubI dst src));
 8528   effect(KILL cr);
 8529 
 8530   format %{ "subl    $dst, $src\t# int" %}
 8531   ins_encode %{
 8532     __ subl($dst$$Register, $src$$Register);
 8533   %}
 8534   ins_pipe(ialu_reg_reg);
 8535 %}
 8536 
 8537 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8538 %{
 8539   match(Set dst (SubI dst src));
 8540   effect(KILL cr);
 8541 
 8542   format %{ "subl    $dst, $src\t# int" %}
 8543   ins_encode %{
 8544     __ subl($dst$$Register, $src$$constant);
 8545   %}
 8546   ins_pipe(ialu_reg);
 8547 %}
 8548 
 8549 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8550 %{
 8551   match(Set dst (SubI dst (LoadI src)));
 8552   effect(KILL cr);
 8553 
 8554   ins_cost(125);
 8555   format %{ "subl    $dst, $src\t# int" %}
 8556   ins_encode %{
 8557     __ subl($dst$$Register, $src$$Address);
 8558   %}
 8559   ins_pipe(ialu_reg_mem);
 8560 %}
 8561 
 8562 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8563 %{
 8564   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8565   effect(KILL cr);
 8566 
 8567   ins_cost(150);
 8568   format %{ "subl    $dst, $src\t# int" %}
 8569   ins_encode %{
 8570     __ subl($dst$$Address, $src$$Register);
 8571   %}
 8572   ins_pipe(ialu_mem_reg);
 8573 %}
 8574 
 8575 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8576 %{
 8577   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8578   effect(KILL cr);
 8579 
 8580   ins_cost(125); // XXX
 8581   format %{ "subl    $dst, $src\t# int" %}
 8582   ins_encode %{
 8583     __ subl($dst$$Address, $src$$constant);
 8584   %}
 8585   ins_pipe(ialu_mem_imm);
 8586 %}
 8587 
 8588 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8589 %{
 8590   match(Set dst (SubL dst src));
 8591   effect(KILL cr);
 8592 
 8593   format %{ "subq    $dst, $src\t# long" %}
 8594   ins_encode %{
 8595     __ subq($dst$$Register, $src$$Register);
 8596   %}
 8597   ins_pipe(ialu_reg_reg);
 8598 %}
 8599 
 8600 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8601 %{
 8602   match(Set dst (SubL dst src));
 8603   effect(KILL cr);
 8604 
 8605   format %{ "subq    $dst, $src\t# long" %}
 8606   ins_encode %{
 8607     __ subq($dst$$Register, $src$$constant);
 8608   %}
 8609   ins_pipe(ialu_reg);
 8610 %}
 8611 
 8612 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8613 %{
 8614   match(Set dst (SubL dst (LoadL src)));
 8615   effect(KILL cr);
 8616 
 8617   ins_cost(125);
 8618   format %{ "subq    $dst, $src\t# long" %}
 8619   ins_encode %{
 8620     __ subq($dst$$Register, $src$$Address);
 8621   %}
 8622   ins_pipe(ialu_reg_mem);
 8623 %}
 8624 
 8625 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8626 %{
 8627   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8628   effect(KILL cr);
 8629 
 8630   ins_cost(150);
 8631   format %{ "subq    $dst, $src\t# long" %}
 8632   ins_encode %{
 8633     __ subq($dst$$Address, $src$$Register);
 8634   %}
 8635   ins_pipe(ialu_mem_reg);
 8636 %}
 8637 
 8638 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8639 %{
 8640   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8641   effect(KILL cr);
 8642 
 8643   ins_cost(125); // XXX
 8644   format %{ "subq    $dst, $src\t# long" %}
 8645   ins_encode %{
 8646     __ subq($dst$$Address, $src$$constant);
 8647   %}
 8648   ins_pipe(ialu_mem_imm);
 8649 %}
 8650 
 8651 // Subtract from a pointer
 8652 // XXX hmpf???
 8653 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8654 %{
 8655   match(Set dst (AddP dst (SubI zero src)));
 8656   effect(KILL cr);
 8657 
 8658   format %{ "subq    $dst, $src\t# ptr - int" %}
 8659   opcode(0x2B);
 8660   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8661   ins_pipe(ialu_reg_reg);
 8662 %}
 8663 
 8664 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8665 %{
 8666   match(Set dst (SubI zero dst));
 8667   effect(KILL cr);
 8668 
 8669   format %{ "negl    $dst\t# int" %}
 8670   ins_encode %{
 8671     __ negl($dst$$Register);
 8672   %}
 8673   ins_pipe(ialu_reg);
 8674 %}
 8675 
 8676 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8677 %{
 8678   match(Set dst (NegI dst));
 8679   effect(KILL cr);
 8680 
 8681   format %{ "negl    $dst\t# int" %}
 8682   ins_encode %{
 8683     __ negl($dst$$Register);
 8684   %}
 8685   ins_pipe(ialu_reg);
 8686 %}
 8687 
 8688 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8689 %{
 8690   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8691   effect(KILL cr);
 8692 
 8693   format %{ "negl    $dst\t# int" %}
 8694   ins_encode %{
 8695     __ negl($dst$$Address);
 8696   %}
 8697   ins_pipe(ialu_reg);
 8698 %}
 8699 
 8700 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8701 %{
 8702   match(Set dst (SubL zero dst));
 8703   effect(KILL cr);
 8704 
 8705   format %{ "negq    $dst\t# long" %}
 8706   ins_encode %{
 8707     __ negq($dst$$Register);
 8708   %}
 8709   ins_pipe(ialu_reg);
 8710 %}
 8711 
 8712 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8713 %{
 8714   match(Set dst (NegL dst));
 8715   effect(KILL cr);
 8716 
 8717   format %{ "negq    $dst\t# int" %}
 8718   ins_encode %{
 8719     __ negq($dst$$Register);
 8720   %}
 8721   ins_pipe(ialu_reg);
 8722 %}
 8723 
 8724 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8725 %{
 8726   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8727   effect(KILL cr);
 8728 
 8729   format %{ "negq    $dst\t# long" %}
 8730   ins_encode %{
 8731     __ negq($dst$$Address);
 8732   %}
 8733   ins_pipe(ialu_reg);
 8734 %}
 8735 
 8736 //----------Multiplication/Division Instructions-------------------------------
 8737 // Integer Multiplication Instructions
 8738 // Multiply Register
 8739 
 8740 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8741 %{
 8742   match(Set dst (MulI dst src));
 8743   effect(KILL cr);
 8744 
 8745   ins_cost(300);
 8746   format %{ "imull   $dst, $src\t# int" %}
 8747   ins_encode %{
 8748     __ imull($dst$$Register, $src$$Register);
 8749   %}
 8750   ins_pipe(ialu_reg_reg_alu0);
 8751 %}
 8752 
 8753 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8754 %{
 8755   match(Set dst (MulI src imm));
 8756   effect(KILL cr);
 8757 
 8758   ins_cost(300);
 8759   format %{ "imull   $dst, $src, $imm\t# int" %}
 8760   ins_encode %{
 8761     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8762   %}
 8763   ins_pipe(ialu_reg_reg_alu0);
 8764 %}
 8765 
 8766 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8767 %{
 8768   match(Set dst (MulI dst (LoadI src)));
 8769   effect(KILL cr);
 8770 
 8771   ins_cost(350);
 8772   format %{ "imull   $dst, $src\t# int" %}
 8773   ins_encode %{
 8774     __ imull($dst$$Register, $src$$Address);
 8775   %}
 8776   ins_pipe(ialu_reg_mem_alu0);
 8777 %}
 8778 
 8779 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8780 %{
 8781   match(Set dst (MulI (LoadI src) imm));
 8782   effect(KILL cr);
 8783 
 8784   ins_cost(300);
 8785   format %{ "imull   $dst, $src, $imm\t# int" %}
 8786   ins_encode %{
 8787     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8788   %}
 8789   ins_pipe(ialu_reg_mem_alu0);
 8790 %}
 8791 
 8792 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8793 %{
 8794   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8795   effect(KILL cr, KILL src2);
 8796 
 8797   expand %{ mulI_rReg(dst, src1, cr);
 8798            mulI_rReg(src2, src3, cr);
 8799            addI_rReg(dst, src2, cr); %}
 8800 %}
 8801 
 8802 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8803 %{
 8804   match(Set dst (MulL dst src));
 8805   effect(KILL cr);
 8806 
 8807   ins_cost(300);
 8808   format %{ "imulq   $dst, $src\t# long" %}
 8809   ins_encode %{
 8810     __ imulq($dst$$Register, $src$$Register);
 8811   %}
 8812   ins_pipe(ialu_reg_reg_alu0);
 8813 %}
 8814 
 8815 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8816 %{
 8817   match(Set dst (MulL src imm));
 8818   effect(KILL cr);
 8819 
 8820   ins_cost(300);
 8821   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8822   ins_encode %{
 8823     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8824   %}
 8825   ins_pipe(ialu_reg_reg_alu0);
 8826 %}
 8827 
 8828 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8829 %{
 8830   match(Set dst (MulL dst (LoadL src)));
 8831   effect(KILL cr);
 8832 
 8833   ins_cost(350);
 8834   format %{ "imulq   $dst, $src\t# long" %}
 8835   ins_encode %{
 8836     __ imulq($dst$$Register, $src$$Address);
 8837   %}
 8838   ins_pipe(ialu_reg_mem_alu0);
 8839 %}
 8840 
 8841 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8842 %{
 8843   match(Set dst (MulL (LoadL src) imm));
 8844   effect(KILL cr);
 8845 
 8846   ins_cost(300);
 8847   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8848   ins_encode %{
 8849     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8850   %}
 8851   ins_pipe(ialu_reg_mem_alu0);
 8852 %}
 8853 
 8854 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8855 %{
 8856   match(Set dst (MulHiL src rax));
 8857   effect(USE_KILL rax, KILL cr);
 8858 
 8859   ins_cost(300);
 8860   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8861   ins_encode %{
 8862     __ imulq($src$$Register);
 8863   %}
 8864   ins_pipe(ialu_reg_reg_alu0);
 8865 %}
 8866 
 8867 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8868 %{
 8869   match(Set dst (UMulHiL src rax));
 8870   effect(USE_KILL rax, KILL cr);
 8871 
 8872   ins_cost(300);
 8873   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8874   ins_encode %{
 8875     __ mulq($src$$Register);
 8876   %}
 8877   ins_pipe(ialu_reg_reg_alu0);
 8878 %}
 8879 
 8880 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8881                    rFlagsReg cr)
 8882 %{
 8883   match(Set rax (DivI rax div));
 8884   effect(KILL rdx, KILL cr);
 8885 
 8886   ins_cost(30*100+10*100); // XXX
 8887   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8888             "jne,s   normal\n\t"
 8889             "xorl    rdx, rdx\n\t"
 8890             "cmpl    $div, -1\n\t"
 8891             "je,s    done\n"
 8892     "normal: cdql\n\t"
 8893             "idivl   $div\n"
 8894     "done:"        %}
 8895   ins_encode(cdql_enc(div));
 8896   ins_pipe(ialu_reg_reg_alu0);
 8897 %}
 8898 
 8899 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8900                    rFlagsReg cr)
 8901 %{
 8902   match(Set rax (DivL rax div));
 8903   effect(KILL rdx, KILL cr);
 8904 
 8905   ins_cost(30*100+10*100); // XXX
 8906   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8907             "cmpq    rax, rdx\n\t"
 8908             "jne,s   normal\n\t"
 8909             "xorl    rdx, rdx\n\t"
 8910             "cmpq    $div, -1\n\t"
 8911             "je,s    done\n"
 8912     "normal: cdqq\n\t"
 8913             "idivq   $div\n"
 8914     "done:"        %}
 8915   ins_encode(cdqq_enc(div));
 8916   ins_pipe(ialu_reg_reg_alu0);
 8917 %}
 8918 
 8919 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8920 %{
 8921   match(Set rax (UDivI rax div));
 8922   effect(KILL rdx, KILL cr);
 8923 
 8924   ins_cost(300);
 8925   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8926   ins_encode %{
 8927     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8928   %}
 8929   ins_pipe(ialu_reg_reg_alu0);
 8930 %}
 8931 
 8932 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8933 %{
 8934   match(Set rax (UDivL rax div));
 8935   effect(KILL rdx, KILL cr);
 8936 
 8937   ins_cost(300);
 8938   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8939   ins_encode %{
 8940      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8941   %}
 8942   ins_pipe(ialu_reg_reg_alu0);
 8943 %}
 8944 
 8945 // Integer DIVMOD with Register, both quotient and mod results
 8946 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8947                              rFlagsReg cr)
 8948 %{
 8949   match(DivModI rax div);
 8950   effect(KILL cr);
 8951 
 8952   ins_cost(30*100+10*100); // XXX
 8953   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8954             "jne,s   normal\n\t"
 8955             "xorl    rdx, rdx\n\t"
 8956             "cmpl    $div, -1\n\t"
 8957             "je,s    done\n"
 8958     "normal: cdql\n\t"
 8959             "idivl   $div\n"
 8960     "done:"        %}
 8961   ins_encode(cdql_enc(div));
 8962   ins_pipe(pipe_slow);
 8963 %}
 8964 
 8965 // Long DIVMOD with Register, both quotient and mod results
 8966 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8967                              rFlagsReg cr)
 8968 %{
 8969   match(DivModL rax div);
 8970   effect(KILL cr);
 8971 
 8972   ins_cost(30*100+10*100); // XXX
 8973   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8974             "cmpq    rax, rdx\n\t"
 8975             "jne,s   normal\n\t"
 8976             "xorl    rdx, rdx\n\t"
 8977             "cmpq    $div, -1\n\t"
 8978             "je,s    done\n"
 8979     "normal: cdqq\n\t"
 8980             "idivq   $div\n"
 8981     "done:"        %}
 8982   ins_encode(cdqq_enc(div));
 8983   ins_pipe(pipe_slow);
 8984 %}
 8985 
 8986 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8987 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8988                               no_rax_rdx_RegI div, rFlagsReg cr)
 8989 %{
 8990   match(UDivModI rax div);
 8991   effect(TEMP tmp, KILL cr);
 8992 
 8993   ins_cost(300);
 8994   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8995             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8996           %}
 8997   ins_encode %{
 8998     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8999   %}
 9000   ins_pipe(pipe_slow);
 9001 %}
 9002 
 9003 // Unsigned long DIVMOD with Register, both quotient and mod results
 9004 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 9005                               no_rax_rdx_RegL div, rFlagsReg cr)
 9006 %{
 9007   match(UDivModL rax div);
 9008   effect(TEMP tmp, KILL cr);
 9009 
 9010   ins_cost(300);
 9011   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 9012             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 9013           %}
 9014   ins_encode %{
 9015     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 9016   %}
 9017   ins_pipe(pipe_slow);
 9018 %}
 9019 
 9020 
 9021 //----------- DivL-By-Constant-Expansions--------------------------------------
 9022 // DivI cases are handled by the compiler
 9023 
 9024 // Magic constant, reciprocal of 10
 9025 instruct loadConL_0x6666666666666667(rRegL dst)
 9026 %{
 9027   effect(DEF dst);
 9028 
 9029   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 9030   ins_encode(load_immL(dst, 0x6666666666666667));
 9031   ins_pipe(ialu_reg);
 9032 %}
 9033 
 9034 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 9035 %{
 9036   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 9037 
 9038   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 9039   ins_encode %{
 9040     __ imulq($src$$Register);
 9041   %}
 9042   ins_pipe(ialu_reg_reg_alu0);
 9043 %}
 9044 
 9045 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 9046 %{
 9047   effect(USE_DEF dst, KILL cr);
 9048 
 9049   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 9050   ins_encode %{
 9051     __ sarq($dst$$Register, 63);
 9052   %}
 9053   ins_pipe(ialu_reg);
 9054 %}
 9055 
 9056 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 9057 %{
 9058   effect(USE_DEF dst, KILL cr);
 9059 
 9060   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 9061   ins_encode %{
 9062     __ sarq($dst$$Register, 2);
 9063   %}
 9064   ins_pipe(ialu_reg);
 9065 %}
 9066 
 9067 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 9068 %{
 9069   match(Set dst (DivL src div));
 9070 
 9071   ins_cost((5+8)*100);
 9072   expand %{
 9073     rax_RegL rax;                     // Killed temp
 9074     rFlagsReg cr;                     // Killed
 9075     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 9076     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 9077     sarL_rReg_63(src, cr);            // sarq  src, 63
 9078     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 9079     subL_rReg(dst, src, cr);          // subl  rdx, src
 9080   %}
 9081 %}
 9082 
 9083 //-----------------------------------------------------------------------------
 9084 
 9085 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 9086                    rFlagsReg cr)
 9087 %{
 9088   match(Set rdx (ModI rax div));
 9089   effect(KILL rax, KILL cr);
 9090 
 9091   ins_cost(300); // XXX
 9092   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 9093             "jne,s   normal\n\t"
 9094             "xorl    rdx, rdx\n\t"
 9095             "cmpl    $div, -1\n\t"
 9096             "je,s    done\n"
 9097     "normal: cdql\n\t"
 9098             "idivl   $div\n"
 9099     "done:"        %}
 9100   ins_encode(cdql_enc(div));
 9101   ins_pipe(ialu_reg_reg_alu0);
 9102 %}
 9103 
 9104 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 9105                    rFlagsReg cr)
 9106 %{
 9107   match(Set rdx (ModL rax div));
 9108   effect(KILL rax, KILL cr);
 9109 
 9110   ins_cost(300); // XXX
 9111   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 9112             "cmpq    rax, rdx\n\t"
 9113             "jne,s   normal\n\t"
 9114             "xorl    rdx, rdx\n\t"
 9115             "cmpq    $div, -1\n\t"
 9116             "je,s    done\n"
 9117     "normal: cdqq\n\t"
 9118             "idivq   $div\n"
 9119     "done:"        %}
 9120   ins_encode(cdqq_enc(div));
 9121   ins_pipe(ialu_reg_reg_alu0);
 9122 %}
 9123 
 9124 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9125 %{
 9126   match(Set rdx (UModI rax div));
 9127   effect(KILL rax, KILL cr);
 9128 
 9129   ins_cost(300);
 9130   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9131   ins_encode %{
 9132     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9133   %}
 9134   ins_pipe(ialu_reg_reg_alu0);
 9135 %}
 9136 
 9137 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9138 %{
 9139   match(Set rdx (UModL rax div));
 9140   effect(KILL rax, KILL cr);
 9141 
 9142   ins_cost(300);
 9143   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9144   ins_encode %{
 9145     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9146   %}
 9147   ins_pipe(ialu_reg_reg_alu0);
 9148 %}
 9149 
 9150 // Integer Shift Instructions
 9151 // Shift Left by one
 9152 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9153 %{
 9154   match(Set dst (LShiftI dst shift));
 9155   effect(KILL cr);
 9156 
 9157   format %{ "sall    $dst, $shift" %}
 9158   ins_encode %{
 9159     __ sall($dst$$Register, $shift$$constant);
 9160   %}
 9161   ins_pipe(ialu_reg);
 9162 %}
 9163 
 9164 // Shift Left by one
 9165 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9166 %{
 9167   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9168   effect(KILL cr);
 9169 
 9170   format %{ "sall    $dst, $shift\t" %}
 9171   ins_encode %{
 9172     __ sall($dst$$Address, $shift$$constant);
 9173   %}
 9174   ins_pipe(ialu_mem_imm);
 9175 %}
 9176 
 9177 // Shift Left by 8-bit immediate
 9178 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9179 %{
 9180   match(Set dst (LShiftI dst shift));
 9181   effect(KILL cr);
 9182 
 9183   format %{ "sall    $dst, $shift" %}
 9184   ins_encode %{
 9185     __ sall($dst$$Register, $shift$$constant);
 9186   %}
 9187   ins_pipe(ialu_reg);
 9188 %}
 9189 
 9190 // Shift Left by 8-bit immediate
 9191 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9192 %{
 9193   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9194   effect(KILL cr);
 9195 
 9196   format %{ "sall    $dst, $shift" %}
 9197   ins_encode %{
 9198     __ sall($dst$$Address, $shift$$constant);
 9199   %}
 9200   ins_pipe(ialu_mem_imm);
 9201 %}
 9202 
 9203 // Shift Left by variable
 9204 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9205 %{
 9206   predicate(!VM_Version::supports_bmi2());
 9207   match(Set dst (LShiftI dst shift));
 9208   effect(KILL cr);
 9209 
 9210   format %{ "sall    $dst, $shift" %}
 9211   ins_encode %{
 9212     __ sall($dst$$Register);
 9213   %}
 9214   ins_pipe(ialu_reg_reg);
 9215 %}
 9216 
 9217 // Shift Left by variable
 9218 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9219 %{
 9220   predicate(!VM_Version::supports_bmi2());
 9221   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9222   effect(KILL cr);
 9223 
 9224   format %{ "sall    $dst, $shift" %}
 9225   ins_encode %{
 9226     __ sall($dst$$Address);
 9227   %}
 9228   ins_pipe(ialu_mem_reg);
 9229 %}
 9230 
 9231 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9232 %{
 9233   predicate(VM_Version::supports_bmi2());
 9234   match(Set dst (LShiftI src shift));
 9235 
 9236   format %{ "shlxl   $dst, $src, $shift" %}
 9237   ins_encode %{
 9238     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9239   %}
 9240   ins_pipe(ialu_reg_reg);
 9241 %}
 9242 
 9243 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9244 %{
 9245   predicate(VM_Version::supports_bmi2());
 9246   match(Set dst (LShiftI (LoadI src) shift));
 9247   ins_cost(175);
 9248   format %{ "shlxl   $dst, $src, $shift" %}
 9249   ins_encode %{
 9250     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9251   %}
 9252   ins_pipe(ialu_reg_mem);
 9253 %}
 9254 
 9255 // Arithmetic shift right by one
 9256 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9257 %{
 9258   match(Set dst (RShiftI dst shift));
 9259   effect(KILL cr);
 9260 
 9261   format %{ "sarl    $dst, $shift" %}
 9262   ins_encode %{
 9263     __ sarl($dst$$Register, $shift$$constant);
 9264   %}
 9265   ins_pipe(ialu_reg);
 9266 %}
 9267 
 9268 // Arithmetic shift right by one
 9269 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9270 %{
 9271   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9272   effect(KILL cr);
 9273 
 9274   format %{ "sarl    $dst, $shift" %}
 9275   ins_encode %{
 9276     __ sarl($dst$$Address, $shift$$constant);
 9277   %}
 9278   ins_pipe(ialu_mem_imm);
 9279 %}
 9280 
 9281 // Arithmetic Shift Right by 8-bit immediate
 9282 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9283 %{
 9284   match(Set dst (RShiftI dst shift));
 9285   effect(KILL cr);
 9286 
 9287   format %{ "sarl    $dst, $shift" %}
 9288   ins_encode %{
 9289     __ sarl($dst$$Register, $shift$$constant);
 9290   %}
 9291   ins_pipe(ialu_mem_imm);
 9292 %}
 9293 
 9294 // Arithmetic Shift Right by 8-bit immediate
 9295 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9296 %{
 9297   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9298   effect(KILL cr);
 9299 
 9300   format %{ "sarl    $dst, $shift" %}
 9301   ins_encode %{
 9302     __ sarl($dst$$Address, $shift$$constant);
 9303   %}
 9304   ins_pipe(ialu_mem_imm);
 9305 %}
 9306 
 9307 // Arithmetic Shift Right by variable
 9308 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9309 %{
 9310   predicate(!VM_Version::supports_bmi2());
 9311   match(Set dst (RShiftI dst shift));
 9312   effect(KILL cr);
 9313   format %{ "sarl    $dst, $shift" %}
 9314   ins_encode %{
 9315     __ sarl($dst$$Register);
 9316   %}
 9317   ins_pipe(ialu_reg_reg);
 9318 %}
 9319 
 9320 // Arithmetic Shift Right by variable
 9321 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9322 %{
 9323   predicate(!VM_Version::supports_bmi2());
 9324   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9325   effect(KILL cr);
 9326 
 9327   format %{ "sarl    $dst, $shift" %}
 9328   ins_encode %{
 9329     __ sarl($dst$$Address);
 9330   %}
 9331   ins_pipe(ialu_mem_reg);
 9332 %}
 9333 
 9334 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9335 %{
 9336   predicate(VM_Version::supports_bmi2());
 9337   match(Set dst (RShiftI src shift));
 9338 
 9339   format %{ "sarxl   $dst, $src, $shift" %}
 9340   ins_encode %{
 9341     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9342   %}
 9343   ins_pipe(ialu_reg_reg);
 9344 %}
 9345 
 9346 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9347 %{
 9348   predicate(VM_Version::supports_bmi2());
 9349   match(Set dst (RShiftI (LoadI src) shift));
 9350   ins_cost(175);
 9351   format %{ "sarxl   $dst, $src, $shift" %}
 9352   ins_encode %{
 9353     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9354   %}
 9355   ins_pipe(ialu_reg_mem);
 9356 %}
 9357 
 9358 // Logical shift right by one
 9359 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9360 %{
 9361   match(Set dst (URShiftI dst shift));
 9362   effect(KILL cr);
 9363 
 9364   format %{ "shrl    $dst, $shift" %}
 9365   ins_encode %{
 9366     __ shrl($dst$$Register, $shift$$constant);
 9367   %}
 9368   ins_pipe(ialu_reg);
 9369 %}
 9370 
 9371 // Logical shift right by one
 9372 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9373 %{
 9374   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9375   effect(KILL cr);
 9376 
 9377   format %{ "shrl    $dst, $shift" %}
 9378   ins_encode %{
 9379     __ shrl($dst$$Address, $shift$$constant);
 9380   %}
 9381   ins_pipe(ialu_mem_imm);
 9382 %}
 9383 
 9384 // Logical Shift Right by 8-bit immediate
 9385 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9386 %{
 9387   match(Set dst (URShiftI dst shift));
 9388   effect(KILL cr);
 9389 
 9390   format %{ "shrl    $dst, $shift" %}
 9391   ins_encode %{
 9392     __ shrl($dst$$Register, $shift$$constant);
 9393   %}
 9394   ins_pipe(ialu_reg);
 9395 %}
 9396 
 9397 // Logical Shift Right by 8-bit immediate
 9398 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9399 %{
 9400   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9401   effect(KILL cr);
 9402 
 9403   format %{ "shrl    $dst, $shift" %}
 9404   ins_encode %{
 9405     __ shrl($dst$$Address, $shift$$constant);
 9406   %}
 9407   ins_pipe(ialu_mem_imm);
 9408 %}
 9409 
 9410 // Logical Shift Right by variable
 9411 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9412 %{
 9413   predicate(!VM_Version::supports_bmi2());
 9414   match(Set dst (URShiftI dst shift));
 9415   effect(KILL cr);
 9416 
 9417   format %{ "shrl    $dst, $shift" %}
 9418   ins_encode %{
 9419     __ shrl($dst$$Register);
 9420   %}
 9421   ins_pipe(ialu_reg_reg);
 9422 %}
 9423 
 9424 // Logical Shift Right by variable
 9425 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9426 %{
 9427   predicate(!VM_Version::supports_bmi2());
 9428   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9429   effect(KILL cr);
 9430 
 9431   format %{ "shrl    $dst, $shift" %}
 9432   ins_encode %{
 9433     __ shrl($dst$$Address);
 9434   %}
 9435   ins_pipe(ialu_mem_reg);
 9436 %}
 9437 
 9438 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9439 %{
 9440   predicate(VM_Version::supports_bmi2());
 9441   match(Set dst (URShiftI src shift));
 9442 
 9443   format %{ "shrxl   $dst, $src, $shift" %}
 9444   ins_encode %{
 9445     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9446   %}
 9447   ins_pipe(ialu_reg_reg);
 9448 %}
 9449 
 9450 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9451 %{
 9452   predicate(VM_Version::supports_bmi2());
 9453   match(Set dst (URShiftI (LoadI src) shift));
 9454   ins_cost(175);
 9455   format %{ "shrxl   $dst, $src, $shift" %}
 9456   ins_encode %{
 9457     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9458   %}
 9459   ins_pipe(ialu_reg_mem);
 9460 %}
 9461 
 9462 // Long Shift Instructions
 9463 // Shift Left by one
 9464 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9465 %{
 9466   match(Set dst (LShiftL dst shift));
 9467   effect(KILL cr);
 9468 
 9469   format %{ "salq    $dst, $shift" %}
 9470   ins_encode %{
 9471     __ salq($dst$$Register, $shift$$constant);
 9472   %}
 9473   ins_pipe(ialu_reg);
 9474 %}
 9475 
 9476 // Shift Left by one
 9477 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9478 %{
 9479   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9480   effect(KILL cr);
 9481 
 9482   format %{ "salq    $dst, $shift" %}
 9483   ins_encode %{
 9484     __ salq($dst$$Address, $shift$$constant);
 9485   %}
 9486   ins_pipe(ialu_mem_imm);
 9487 %}
 9488 
 9489 // Shift Left by 8-bit immediate
 9490 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9491 %{
 9492   match(Set dst (LShiftL dst shift));
 9493   effect(KILL cr);
 9494 
 9495   format %{ "salq    $dst, $shift" %}
 9496   ins_encode %{
 9497     __ salq($dst$$Register, $shift$$constant);
 9498   %}
 9499   ins_pipe(ialu_reg);
 9500 %}
 9501 
 9502 // Shift Left by 8-bit immediate
 9503 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9504 %{
 9505   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9506   effect(KILL cr);
 9507 
 9508   format %{ "salq    $dst, $shift" %}
 9509   ins_encode %{
 9510     __ salq($dst$$Address, $shift$$constant);
 9511   %}
 9512   ins_pipe(ialu_mem_imm);
 9513 %}
 9514 
 9515 // Shift Left by variable
 9516 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9517 %{
 9518   predicate(!VM_Version::supports_bmi2());
 9519   match(Set dst (LShiftL dst shift));
 9520   effect(KILL cr);
 9521 
 9522   format %{ "salq    $dst, $shift" %}
 9523   ins_encode %{
 9524     __ salq($dst$$Register);
 9525   %}
 9526   ins_pipe(ialu_reg_reg);
 9527 %}
 9528 
 9529 // Shift Left by variable
 9530 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9531 %{
 9532   predicate(!VM_Version::supports_bmi2());
 9533   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9534   effect(KILL cr);
 9535 
 9536   format %{ "salq    $dst, $shift" %}
 9537   ins_encode %{
 9538     __ salq($dst$$Address);
 9539   %}
 9540   ins_pipe(ialu_mem_reg);
 9541 %}
 9542 
 9543 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9544 %{
 9545   predicate(VM_Version::supports_bmi2());
 9546   match(Set dst (LShiftL src shift));
 9547 
 9548   format %{ "shlxq   $dst, $src, $shift" %}
 9549   ins_encode %{
 9550     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9551   %}
 9552   ins_pipe(ialu_reg_reg);
 9553 %}
 9554 
 9555 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9556 %{
 9557   predicate(VM_Version::supports_bmi2());
 9558   match(Set dst (LShiftL (LoadL src) shift));
 9559   ins_cost(175);
 9560   format %{ "shlxq   $dst, $src, $shift" %}
 9561   ins_encode %{
 9562     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9563   %}
 9564   ins_pipe(ialu_reg_mem);
 9565 %}
 9566 
 9567 // Arithmetic shift right by one
 9568 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9569 %{
 9570   match(Set dst (RShiftL dst shift));
 9571   effect(KILL cr);
 9572 
 9573   format %{ "sarq    $dst, $shift" %}
 9574   ins_encode %{
 9575     __ sarq($dst$$Register, $shift$$constant);
 9576   %}
 9577   ins_pipe(ialu_reg);
 9578 %}
 9579 
 9580 // Arithmetic shift right by one
 9581 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9582 %{
 9583   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9584   effect(KILL cr);
 9585 
 9586   format %{ "sarq    $dst, $shift" %}
 9587   ins_encode %{
 9588     __ sarq($dst$$Address, $shift$$constant);
 9589   %}
 9590   ins_pipe(ialu_mem_imm);
 9591 %}
 9592 
 9593 // Arithmetic Shift Right by 8-bit immediate
 9594 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9595 %{
 9596   match(Set dst (RShiftL dst shift));
 9597   effect(KILL cr);
 9598 
 9599   format %{ "sarq    $dst, $shift" %}
 9600   ins_encode %{
 9601     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9602   %}
 9603   ins_pipe(ialu_mem_imm);
 9604 %}
 9605 
 9606 // Arithmetic Shift Right by 8-bit immediate
 9607 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9608 %{
 9609   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9610   effect(KILL cr);
 9611 
 9612   format %{ "sarq    $dst, $shift" %}
 9613   ins_encode %{
 9614     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9615   %}
 9616   ins_pipe(ialu_mem_imm);
 9617 %}
 9618 
 9619 // Arithmetic Shift Right by variable
 9620 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9621 %{
 9622   predicate(!VM_Version::supports_bmi2());
 9623   match(Set dst (RShiftL dst shift));
 9624   effect(KILL cr);
 9625 
 9626   format %{ "sarq    $dst, $shift" %}
 9627   ins_encode %{
 9628     __ sarq($dst$$Register);
 9629   %}
 9630   ins_pipe(ialu_reg_reg);
 9631 %}
 9632 
 9633 // Arithmetic Shift Right by variable
 9634 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9635 %{
 9636   predicate(!VM_Version::supports_bmi2());
 9637   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9638   effect(KILL cr);
 9639 
 9640   format %{ "sarq    $dst, $shift" %}
 9641   ins_encode %{
 9642     __ sarq($dst$$Address);
 9643   %}
 9644   ins_pipe(ialu_mem_reg);
 9645 %}
 9646 
 9647 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9648 %{
 9649   predicate(VM_Version::supports_bmi2());
 9650   match(Set dst (RShiftL src shift));
 9651 
 9652   format %{ "sarxq   $dst, $src, $shift" %}
 9653   ins_encode %{
 9654     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9655   %}
 9656   ins_pipe(ialu_reg_reg);
 9657 %}
 9658 
 9659 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9660 %{
 9661   predicate(VM_Version::supports_bmi2());
 9662   match(Set dst (RShiftL (LoadL src) shift));
 9663   ins_cost(175);
 9664   format %{ "sarxq   $dst, $src, $shift" %}
 9665   ins_encode %{
 9666     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9667   %}
 9668   ins_pipe(ialu_reg_mem);
 9669 %}
 9670 
 9671 // Logical shift right by one
 9672 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9673 %{
 9674   match(Set dst (URShiftL dst shift));
 9675   effect(KILL cr);
 9676 
 9677   format %{ "shrq    $dst, $shift" %}
 9678   ins_encode %{
 9679     __ shrq($dst$$Register, $shift$$constant);
 9680   %}
 9681   ins_pipe(ialu_reg);
 9682 %}
 9683 
 9684 // Logical shift right by one
 9685 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9686 %{
 9687   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9688   effect(KILL cr);
 9689 
 9690   format %{ "shrq    $dst, $shift" %}
 9691   ins_encode %{
 9692     __ shrq($dst$$Address, $shift$$constant);
 9693   %}
 9694   ins_pipe(ialu_mem_imm);
 9695 %}
 9696 
 9697 // Logical Shift Right by 8-bit immediate
 9698 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9699 %{
 9700   match(Set dst (URShiftL dst shift));
 9701   effect(KILL cr);
 9702 
 9703   format %{ "shrq    $dst, $shift" %}
 9704   ins_encode %{
 9705     __ shrq($dst$$Register, $shift$$constant);
 9706   %}
 9707   ins_pipe(ialu_reg);
 9708 %}
 9709 
 9710 // Logical Shift Right by 8-bit immediate
 9711 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9712 %{
 9713   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9714   effect(KILL cr);
 9715 
 9716   format %{ "shrq    $dst, $shift" %}
 9717   ins_encode %{
 9718     __ shrq($dst$$Address, $shift$$constant);
 9719   %}
 9720   ins_pipe(ialu_mem_imm);
 9721 %}
 9722 
 9723 // Logical Shift Right by variable
 9724 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9725 %{
 9726   predicate(!VM_Version::supports_bmi2());
 9727   match(Set dst (URShiftL dst shift));
 9728   effect(KILL cr);
 9729 
 9730   format %{ "shrq    $dst, $shift" %}
 9731   ins_encode %{
 9732     __ shrq($dst$$Register);
 9733   %}
 9734   ins_pipe(ialu_reg_reg);
 9735 %}
 9736 
 9737 // Logical Shift Right by variable
 9738 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9739 %{
 9740   predicate(!VM_Version::supports_bmi2());
 9741   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9742   effect(KILL cr);
 9743 
 9744   format %{ "shrq    $dst, $shift" %}
 9745   ins_encode %{
 9746     __ shrq($dst$$Address);
 9747   %}
 9748   ins_pipe(ialu_mem_reg);
 9749 %}
 9750 
 9751 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9752 %{
 9753   predicate(VM_Version::supports_bmi2());
 9754   match(Set dst (URShiftL src shift));
 9755 
 9756   format %{ "shrxq   $dst, $src, $shift" %}
 9757   ins_encode %{
 9758     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9759   %}
 9760   ins_pipe(ialu_reg_reg);
 9761 %}
 9762 
 9763 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9764 %{
 9765   predicate(VM_Version::supports_bmi2());
 9766   match(Set dst (URShiftL (LoadL src) shift));
 9767   ins_cost(175);
 9768   format %{ "shrxq   $dst, $src, $shift" %}
 9769   ins_encode %{
 9770     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9771   %}
 9772   ins_pipe(ialu_reg_mem);
 9773 %}
 9774 
 9775 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9776 // This idiom is used by the compiler for the i2b bytecode.
 9777 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9778 %{
 9779   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9780 
 9781   format %{ "movsbl  $dst, $src\t# i2b" %}
 9782   ins_encode %{
 9783     __ movsbl($dst$$Register, $src$$Register);
 9784   %}
 9785   ins_pipe(ialu_reg_reg);
 9786 %}
 9787 
 9788 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9789 // This idiom is used by the compiler the i2s bytecode.
 9790 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9791 %{
 9792   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9793 
 9794   format %{ "movswl  $dst, $src\t# i2s" %}
 9795   ins_encode %{
 9796     __ movswl($dst$$Register, $src$$Register);
 9797   %}
 9798   ins_pipe(ialu_reg_reg);
 9799 %}
 9800 
 9801 // ROL/ROR instructions
 9802 
 9803 // Rotate left by constant.
 9804 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9805 %{
 9806   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9807   match(Set dst (RotateLeft dst shift));
 9808   effect(KILL cr);
 9809   format %{ "roll    $dst, $shift" %}
 9810   ins_encode %{
 9811     __ roll($dst$$Register, $shift$$constant);
 9812   %}
 9813   ins_pipe(ialu_reg);
 9814 %}
 9815 
 9816 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9817 %{
 9818   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9819   match(Set dst (RotateLeft src shift));
 9820   format %{ "rolxl   $dst, $src, $shift" %}
 9821   ins_encode %{
 9822     int shift = 32 - ($shift$$constant & 31);
 9823     __ rorxl($dst$$Register, $src$$Register, shift);
 9824   %}
 9825   ins_pipe(ialu_reg_reg);
 9826 %}
 9827 
 9828 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9829 %{
 9830   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9831   match(Set dst (RotateLeft (LoadI src) shift));
 9832   ins_cost(175);
 9833   format %{ "rolxl   $dst, $src, $shift" %}
 9834   ins_encode %{
 9835     int shift = 32 - ($shift$$constant & 31);
 9836     __ rorxl($dst$$Register, $src$$Address, shift);
 9837   %}
 9838   ins_pipe(ialu_reg_mem);
 9839 %}
 9840 
 9841 // Rotate Left by variable
 9842 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9843 %{
 9844   predicate(n->bottom_type()->basic_type() == T_INT);
 9845   match(Set dst (RotateLeft dst shift));
 9846   effect(KILL cr);
 9847   format %{ "roll    $dst, $shift" %}
 9848   ins_encode %{
 9849     __ roll($dst$$Register);
 9850   %}
 9851   ins_pipe(ialu_reg_reg);
 9852 %}
 9853 
 9854 // Rotate Right by constant.
 9855 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9856 %{
 9857   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9858   match(Set dst (RotateRight dst shift));
 9859   effect(KILL cr);
 9860   format %{ "rorl    $dst, $shift" %}
 9861   ins_encode %{
 9862     __ rorl($dst$$Register, $shift$$constant);
 9863   %}
 9864   ins_pipe(ialu_reg);
 9865 %}
 9866 
 9867 // Rotate Right by constant.
 9868 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9869 %{
 9870   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9871   match(Set dst (RotateRight src shift));
 9872   format %{ "rorxl   $dst, $src, $shift" %}
 9873   ins_encode %{
 9874     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9875   %}
 9876   ins_pipe(ialu_reg_reg);
 9877 %}
 9878 
 9879 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9880 %{
 9881   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9882   match(Set dst (RotateRight (LoadI src) shift));
 9883   ins_cost(175);
 9884   format %{ "rorxl   $dst, $src, $shift" %}
 9885   ins_encode %{
 9886     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9887   %}
 9888   ins_pipe(ialu_reg_mem);
 9889 %}
 9890 
 9891 // Rotate Right by variable
 9892 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9893 %{
 9894   predicate(n->bottom_type()->basic_type() == T_INT);
 9895   match(Set dst (RotateRight dst shift));
 9896   effect(KILL cr);
 9897   format %{ "rorl    $dst, $shift" %}
 9898   ins_encode %{
 9899     __ rorl($dst$$Register);
 9900   %}
 9901   ins_pipe(ialu_reg_reg);
 9902 %}
 9903 
 9904 // Rotate Left by constant.
 9905 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9906 %{
 9907   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9908   match(Set dst (RotateLeft dst shift));
 9909   effect(KILL cr);
 9910   format %{ "rolq    $dst, $shift" %}
 9911   ins_encode %{
 9912     __ rolq($dst$$Register, $shift$$constant);
 9913   %}
 9914   ins_pipe(ialu_reg);
 9915 %}
 9916 
 9917 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9918 %{
 9919   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9920   match(Set dst (RotateLeft src shift));
 9921   format %{ "rolxq   $dst, $src, $shift" %}
 9922   ins_encode %{
 9923     int shift = 64 - ($shift$$constant & 63);
 9924     __ rorxq($dst$$Register, $src$$Register, shift);
 9925   %}
 9926   ins_pipe(ialu_reg_reg);
 9927 %}
 9928 
 9929 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9930 %{
 9931   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9932   match(Set dst (RotateLeft (LoadL src) shift));
 9933   ins_cost(175);
 9934   format %{ "rolxq   $dst, $src, $shift" %}
 9935   ins_encode %{
 9936     int shift = 64 - ($shift$$constant & 63);
 9937     __ rorxq($dst$$Register, $src$$Address, shift);
 9938   %}
 9939   ins_pipe(ialu_reg_mem);
 9940 %}
 9941 
 9942 // Rotate Left by variable
 9943 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9944 %{
 9945   predicate(n->bottom_type()->basic_type() == T_LONG);
 9946   match(Set dst (RotateLeft dst shift));
 9947   effect(KILL cr);
 9948   format %{ "rolq    $dst, $shift" %}
 9949   ins_encode %{
 9950     __ rolq($dst$$Register);
 9951   %}
 9952   ins_pipe(ialu_reg_reg);
 9953 %}
 9954 
 9955 // Rotate Right by constant.
 9956 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9957 %{
 9958   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9959   match(Set dst (RotateRight dst shift));
 9960   effect(KILL cr);
 9961   format %{ "rorq    $dst, $shift" %}
 9962   ins_encode %{
 9963     __ rorq($dst$$Register, $shift$$constant);
 9964   %}
 9965   ins_pipe(ialu_reg);
 9966 %}
 9967 
 9968 // Rotate Right by constant
 9969 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9970 %{
 9971   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9972   match(Set dst (RotateRight src shift));
 9973   format %{ "rorxq   $dst, $src, $shift" %}
 9974   ins_encode %{
 9975     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9976   %}
 9977   ins_pipe(ialu_reg_reg);
 9978 %}
 9979 
 9980 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9981 %{
 9982   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9983   match(Set dst (RotateRight (LoadL src) shift));
 9984   ins_cost(175);
 9985   format %{ "rorxq   $dst, $src, $shift" %}
 9986   ins_encode %{
 9987     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9988   %}
 9989   ins_pipe(ialu_reg_mem);
 9990 %}
 9991 
 9992 // Rotate Right by variable
 9993 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9994 %{
 9995   predicate(n->bottom_type()->basic_type() == T_LONG);
 9996   match(Set dst (RotateRight dst shift));
 9997   effect(KILL cr);
 9998   format %{ "rorq    $dst, $shift" %}
 9999   ins_encode %{
10000     __ rorq($dst$$Register);
10001   %}
10002   ins_pipe(ialu_reg_reg);
10003 %}
10004 
10005 //----------------------------- CompressBits/ExpandBits ------------------------
10006 
10007 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10008   predicate(n->bottom_type()->isa_long());
10009   match(Set dst (CompressBits src mask));
10010   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10011   ins_encode %{
10012     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
10013   %}
10014   ins_pipe( pipe_slow );
10015 %}
10016 
10017 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10018   predicate(n->bottom_type()->isa_long());
10019   match(Set dst (ExpandBits src mask));
10020   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10021   ins_encode %{
10022     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10023   %}
10024   ins_pipe( pipe_slow );
10025 %}
10026 
10027 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10028   predicate(n->bottom_type()->isa_long());
10029   match(Set dst (CompressBits src (LoadL mask)));
10030   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10031   ins_encode %{
10032     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10033   %}
10034   ins_pipe( pipe_slow );
10035 %}
10036 
10037 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10038   predicate(n->bottom_type()->isa_long());
10039   match(Set dst (ExpandBits src (LoadL mask)));
10040   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10041   ins_encode %{
10042     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10043   %}
10044   ins_pipe( pipe_slow );
10045 %}
10046 
10047 
10048 // Logical Instructions
10049 
10050 // Integer Logical Instructions
10051 
10052 // And Instructions
10053 // And Register with Register
10054 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10055 %{
10056   match(Set dst (AndI dst src));
10057   effect(KILL cr);
10058 
10059   format %{ "andl    $dst, $src\t# int" %}
10060   ins_encode %{
10061     __ andl($dst$$Register, $src$$Register);
10062   %}
10063   ins_pipe(ialu_reg_reg);
10064 %}
10065 
10066 // And Register with Immediate 255
10067 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
10068 %{
10069   match(Set dst (AndI dst src));
10070 
10071   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
10072   ins_encode %{
10073     __ movzbl($dst$$Register, $dst$$Register);
10074   %}
10075   ins_pipe(ialu_reg);
10076 %}
10077 
10078 // And Register with Immediate 255 and promote to long
10079 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10080 %{
10081   match(Set dst (ConvI2L (AndI src mask)));
10082 
10083   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10084   ins_encode %{
10085     __ movzbl($dst$$Register, $src$$Register);
10086   %}
10087   ins_pipe(ialu_reg);
10088 %}
10089 
10090 // And Register with Immediate 65535
10091 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
10092 %{
10093   match(Set dst (AndI dst src));
10094 
10095   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
10096   ins_encode %{
10097     __ movzwl($dst$$Register, $dst$$Register);
10098   %}
10099   ins_pipe(ialu_reg);
10100 %}
10101 
10102 // And Register with Immediate 65535 and promote to long
10103 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10104 %{
10105   match(Set dst (ConvI2L (AndI src mask)));
10106 
10107   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10108   ins_encode %{
10109     __ movzwl($dst$$Register, $src$$Register);
10110   %}
10111   ins_pipe(ialu_reg);
10112 %}
10113 
10114 // Can skip int2long conversions after AND with small bitmask
10115 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10116 %{
10117   predicate(VM_Version::supports_bmi2());
10118   ins_cost(125);
10119   effect(TEMP tmp, KILL cr);
10120   match(Set dst (ConvI2L (AndI src mask)));
10121   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10122   ins_encode %{
10123     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10124     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10125   %}
10126   ins_pipe(ialu_reg_reg);
10127 %}
10128 
10129 // And Register with Immediate
10130 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10131 %{
10132   match(Set dst (AndI dst src));
10133   effect(KILL cr);
10134 
10135   format %{ "andl    $dst, $src\t# int" %}
10136   ins_encode %{
10137     __ andl($dst$$Register, $src$$constant);
10138   %}
10139   ins_pipe(ialu_reg);
10140 %}
10141 
10142 // And Register with Memory
10143 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10144 %{
10145   match(Set dst (AndI dst (LoadI src)));
10146   effect(KILL cr);
10147 
10148   ins_cost(125);
10149   format %{ "andl    $dst, $src\t# int" %}
10150   ins_encode %{
10151     __ andl($dst$$Register, $src$$Address);
10152   %}
10153   ins_pipe(ialu_reg_mem);
10154 %}
10155 
10156 // And Memory with Register
10157 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10158 %{
10159   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10160   effect(KILL cr);
10161 
10162   ins_cost(150);
10163   format %{ "andb    $dst, $src\t# byte" %}
10164   ins_encode %{
10165     __ andb($dst$$Address, $src$$Register);
10166   %}
10167   ins_pipe(ialu_mem_reg);
10168 %}
10169 
10170 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10171 %{
10172   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10173   effect(KILL cr);
10174 
10175   ins_cost(150);
10176   format %{ "andl    $dst, $src\t# int" %}
10177   ins_encode %{
10178     __ andl($dst$$Address, $src$$Register);
10179   %}
10180   ins_pipe(ialu_mem_reg);
10181 %}
10182 
10183 // And Memory with Immediate
10184 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10185 %{
10186   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10187   effect(KILL cr);
10188 
10189   ins_cost(125);
10190   format %{ "andl    $dst, $src\t# int" %}
10191   ins_encode %{
10192     __ andl($dst$$Address, $src$$constant);
10193   %}
10194   ins_pipe(ialu_mem_imm);
10195 %}
10196 
10197 // BMI1 instructions
10198 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10199   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10200   predicate(UseBMI1Instructions);
10201   effect(KILL cr);
10202 
10203   ins_cost(125);
10204   format %{ "andnl  $dst, $src1, $src2" %}
10205 
10206   ins_encode %{
10207     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10208   %}
10209   ins_pipe(ialu_reg_mem);
10210 %}
10211 
10212 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10213   match(Set dst (AndI (XorI src1 minus_1) src2));
10214   predicate(UseBMI1Instructions);
10215   effect(KILL cr);
10216 
10217   format %{ "andnl  $dst, $src1, $src2" %}
10218 
10219   ins_encode %{
10220     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10221   %}
10222   ins_pipe(ialu_reg);
10223 %}
10224 
10225 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10226   match(Set dst (AndI (SubI imm_zero src) src));
10227   predicate(UseBMI1Instructions);
10228   effect(KILL cr);
10229 
10230   format %{ "blsil  $dst, $src" %}
10231 
10232   ins_encode %{
10233     __ blsil($dst$$Register, $src$$Register);
10234   %}
10235   ins_pipe(ialu_reg);
10236 %}
10237 
10238 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10239   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10240   predicate(UseBMI1Instructions);
10241   effect(KILL cr);
10242 
10243   ins_cost(125);
10244   format %{ "blsil  $dst, $src" %}
10245 
10246   ins_encode %{
10247     __ blsil($dst$$Register, $src$$Address);
10248   %}
10249   ins_pipe(ialu_reg_mem);
10250 %}
10251 
10252 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10253 %{
10254   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10255   predicate(UseBMI1Instructions);
10256   effect(KILL cr);
10257 
10258   ins_cost(125);
10259   format %{ "blsmskl $dst, $src" %}
10260 
10261   ins_encode %{
10262     __ blsmskl($dst$$Register, $src$$Address);
10263   %}
10264   ins_pipe(ialu_reg_mem);
10265 %}
10266 
10267 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10268 %{
10269   match(Set dst (XorI (AddI src minus_1) src));
10270   predicate(UseBMI1Instructions);
10271   effect(KILL cr);
10272 
10273   format %{ "blsmskl $dst, $src" %}
10274 
10275   ins_encode %{
10276     __ blsmskl($dst$$Register, $src$$Register);
10277   %}
10278 
10279   ins_pipe(ialu_reg);
10280 %}
10281 
10282 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10283 %{
10284   match(Set dst (AndI (AddI src minus_1) src) );
10285   predicate(UseBMI1Instructions);
10286   effect(KILL cr);
10287 
10288   format %{ "blsrl  $dst, $src" %}
10289 
10290   ins_encode %{
10291     __ blsrl($dst$$Register, $src$$Register);
10292   %}
10293 
10294   ins_pipe(ialu_reg_mem);
10295 %}
10296 
10297 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10298 %{
10299   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10300   predicate(UseBMI1Instructions);
10301   effect(KILL cr);
10302 
10303   ins_cost(125);
10304   format %{ "blsrl  $dst, $src" %}
10305 
10306   ins_encode %{
10307     __ blsrl($dst$$Register, $src$$Address);
10308   %}
10309 
10310   ins_pipe(ialu_reg);
10311 %}
10312 
10313 // Or Instructions
10314 // Or Register with Register
10315 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10316 %{
10317   match(Set dst (OrI dst src));
10318   effect(KILL cr);
10319 
10320   format %{ "orl     $dst, $src\t# int" %}
10321   ins_encode %{
10322     __ orl($dst$$Register, $src$$Register);
10323   %}
10324   ins_pipe(ialu_reg_reg);
10325 %}
10326 
10327 // Or Register with Immediate
10328 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10329 %{
10330   match(Set dst (OrI dst src));
10331   effect(KILL cr);
10332 
10333   format %{ "orl     $dst, $src\t# int" %}
10334   ins_encode %{
10335     __ orl($dst$$Register, $src$$constant);
10336   %}
10337   ins_pipe(ialu_reg);
10338 %}
10339 
10340 // Or Register with Memory
10341 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10342 %{
10343   match(Set dst (OrI dst (LoadI src)));
10344   effect(KILL cr);
10345 
10346   ins_cost(125);
10347   format %{ "orl     $dst, $src\t# int" %}
10348   ins_encode %{
10349     __ orl($dst$$Register, $src$$Address);
10350   %}
10351   ins_pipe(ialu_reg_mem);
10352 %}
10353 
10354 // Or Memory with Register
10355 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10356 %{
10357   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10358   effect(KILL cr);
10359 
10360   ins_cost(150);
10361   format %{ "orb    $dst, $src\t# byte" %}
10362   ins_encode %{
10363     __ orb($dst$$Address, $src$$Register);
10364   %}
10365   ins_pipe(ialu_mem_reg);
10366 %}
10367 
10368 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10369 %{
10370   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10371   effect(KILL cr);
10372 
10373   ins_cost(150);
10374   format %{ "orl     $dst, $src\t# int" %}
10375   ins_encode %{
10376     __ orl($dst$$Address, $src$$Register);
10377   %}
10378   ins_pipe(ialu_mem_reg);
10379 %}
10380 
10381 // Or Memory with Immediate
10382 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10383 %{
10384   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10385   effect(KILL cr);
10386 
10387   ins_cost(125);
10388   format %{ "orl     $dst, $src\t# int" %}
10389   ins_encode %{
10390     __ orl($dst$$Address, $src$$constant);
10391   %}
10392   ins_pipe(ialu_mem_imm);
10393 %}
10394 
10395 // Xor Instructions
10396 // Xor Register with Register
10397 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10398 %{
10399   match(Set dst (XorI dst src));
10400   effect(KILL cr);
10401 
10402   format %{ "xorl    $dst, $src\t# int" %}
10403   ins_encode %{
10404     __ xorl($dst$$Register, $src$$Register);
10405   %}
10406   ins_pipe(ialu_reg_reg);
10407 %}
10408 
10409 // Xor Register with Immediate -1
10410 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10411   match(Set dst (XorI dst imm));
10412 
10413   format %{ "not    $dst" %}
10414   ins_encode %{
10415      __ notl($dst$$Register);
10416   %}
10417   ins_pipe(ialu_reg);
10418 %}
10419 
10420 // Xor Register with Immediate
10421 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10422 %{
10423   match(Set dst (XorI dst src));
10424   effect(KILL cr);
10425 
10426   format %{ "xorl    $dst, $src\t# int" %}
10427   ins_encode %{
10428     __ xorl($dst$$Register, $src$$constant);
10429   %}
10430   ins_pipe(ialu_reg);
10431 %}
10432 
10433 // Xor Register with Memory
10434 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10435 %{
10436   match(Set dst (XorI dst (LoadI src)));
10437   effect(KILL cr);
10438 
10439   ins_cost(125);
10440   format %{ "xorl    $dst, $src\t# int" %}
10441   ins_encode %{
10442     __ xorl($dst$$Register, $src$$Address);
10443   %}
10444   ins_pipe(ialu_reg_mem);
10445 %}
10446 
10447 // Xor Memory with Register
10448 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10449 %{
10450   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10451   effect(KILL cr);
10452 
10453   ins_cost(150);
10454   format %{ "xorb    $dst, $src\t# byte" %}
10455   ins_encode %{
10456     __ xorb($dst$$Address, $src$$Register);
10457   %}
10458   ins_pipe(ialu_mem_reg);
10459 %}
10460 
10461 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10462 %{
10463   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10464   effect(KILL cr);
10465 
10466   ins_cost(150);
10467   format %{ "xorl    $dst, $src\t# int" %}
10468   ins_encode %{
10469     __ xorl($dst$$Address, $src$$Register);
10470   %}
10471   ins_pipe(ialu_mem_reg);
10472 %}
10473 
10474 // Xor Memory with Immediate
10475 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10476 %{
10477   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10478   effect(KILL cr);
10479 
10480   ins_cost(125);
10481   format %{ "xorl    $dst, $src\t# int" %}
10482   ins_encode %{
10483     __ xorl($dst$$Address, $src$$constant);
10484   %}
10485   ins_pipe(ialu_mem_imm);
10486 %}
10487 
10488 
10489 // Long Logical Instructions
10490 
10491 // And Instructions
10492 // And Register with Register
10493 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10494 %{
10495   match(Set dst (AndL dst src));
10496   effect(KILL cr);
10497 
10498   format %{ "andq    $dst, $src\t# long" %}
10499   ins_encode %{
10500     __ andq($dst$$Register, $src$$Register);
10501   %}
10502   ins_pipe(ialu_reg_reg);
10503 %}
10504 
10505 // And Register with Immediate 255
10506 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
10507 %{
10508   match(Set dst (AndL dst src));
10509 
10510   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
10511   ins_encode %{
10512     __ movzbq($dst$$Register, $dst$$Register);
10513   %}
10514   ins_pipe(ialu_reg);
10515 %}
10516 
10517 // And Register with Immediate 65535
10518 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
10519 %{
10520   match(Set dst (AndL dst src));
10521 
10522   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
10523   ins_encode %{
10524     __ movzwq($dst$$Register, $dst$$Register);
10525   %}
10526   ins_pipe(ialu_reg);
10527 %}
10528 
10529 // And Register with Immediate
10530 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10531 %{
10532   match(Set dst (AndL dst src));
10533   effect(KILL cr);
10534 
10535   format %{ "andq    $dst, $src\t# long" %}
10536   ins_encode %{
10537     __ andq($dst$$Register, $src$$constant);
10538   %}
10539   ins_pipe(ialu_reg);
10540 %}
10541 
10542 // And Register with Memory
10543 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10544 %{
10545   match(Set dst (AndL dst (LoadL src)));
10546   effect(KILL cr);
10547 
10548   ins_cost(125);
10549   format %{ "andq    $dst, $src\t# long" %}
10550   ins_encode %{
10551     __ andq($dst$$Register, $src$$Address);
10552   %}
10553   ins_pipe(ialu_reg_mem);
10554 %}
10555 
10556 // And Memory with Register
10557 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10558 %{
10559   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10560   effect(KILL cr);
10561 
10562   ins_cost(150);
10563   format %{ "andq    $dst, $src\t# long" %}
10564   ins_encode %{
10565     __ andq($dst$$Address, $src$$Register);
10566   %}
10567   ins_pipe(ialu_mem_reg);
10568 %}
10569 
10570 // And Memory with Immediate
10571 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10572 %{
10573   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10574   effect(KILL cr);
10575 
10576   ins_cost(125);
10577   format %{ "andq    $dst, $src\t# long" %}
10578   ins_encode %{
10579     __ andq($dst$$Address, $src$$constant);
10580   %}
10581   ins_pipe(ialu_mem_imm);
10582 %}
10583 
10584 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10585 %{
10586   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10587   // because AND/OR works well enough for 8/32-bit values.
10588   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10589 
10590   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10591   effect(KILL cr);
10592 
10593   ins_cost(125);
10594   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10595   ins_encode %{
10596     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10597   %}
10598   ins_pipe(ialu_mem_imm);
10599 %}
10600 
10601 // BMI1 instructions
10602 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10603   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10604   predicate(UseBMI1Instructions);
10605   effect(KILL cr);
10606 
10607   ins_cost(125);
10608   format %{ "andnq  $dst, $src1, $src2" %}
10609 
10610   ins_encode %{
10611     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10612   %}
10613   ins_pipe(ialu_reg_mem);
10614 %}
10615 
10616 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10617   match(Set dst (AndL (XorL src1 minus_1) src2));
10618   predicate(UseBMI1Instructions);
10619   effect(KILL cr);
10620 
10621   format %{ "andnq  $dst, $src1, $src2" %}
10622 
10623   ins_encode %{
10624   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10625   %}
10626   ins_pipe(ialu_reg_mem);
10627 %}
10628 
10629 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10630   match(Set dst (AndL (SubL imm_zero src) src));
10631   predicate(UseBMI1Instructions);
10632   effect(KILL cr);
10633 
10634   format %{ "blsiq  $dst, $src" %}
10635 
10636   ins_encode %{
10637     __ blsiq($dst$$Register, $src$$Register);
10638   %}
10639   ins_pipe(ialu_reg);
10640 %}
10641 
10642 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10643   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10644   predicate(UseBMI1Instructions);
10645   effect(KILL cr);
10646 
10647   ins_cost(125);
10648   format %{ "blsiq  $dst, $src" %}
10649 
10650   ins_encode %{
10651     __ blsiq($dst$$Register, $src$$Address);
10652   %}
10653   ins_pipe(ialu_reg_mem);
10654 %}
10655 
10656 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10657 %{
10658   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10659   predicate(UseBMI1Instructions);
10660   effect(KILL cr);
10661 
10662   ins_cost(125);
10663   format %{ "blsmskq $dst, $src" %}
10664 
10665   ins_encode %{
10666     __ blsmskq($dst$$Register, $src$$Address);
10667   %}
10668   ins_pipe(ialu_reg_mem);
10669 %}
10670 
10671 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10672 %{
10673   match(Set dst (XorL (AddL src minus_1) src));
10674   predicate(UseBMI1Instructions);
10675   effect(KILL cr);
10676 
10677   format %{ "blsmskq $dst, $src" %}
10678 
10679   ins_encode %{
10680     __ blsmskq($dst$$Register, $src$$Register);
10681   %}
10682 
10683   ins_pipe(ialu_reg);
10684 %}
10685 
10686 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10687 %{
10688   match(Set dst (AndL (AddL src minus_1) src) );
10689   predicate(UseBMI1Instructions);
10690   effect(KILL cr);
10691 
10692   format %{ "blsrq  $dst, $src" %}
10693 
10694   ins_encode %{
10695     __ blsrq($dst$$Register, $src$$Register);
10696   %}
10697 
10698   ins_pipe(ialu_reg);
10699 %}
10700 
10701 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10702 %{
10703   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10704   predicate(UseBMI1Instructions);
10705   effect(KILL cr);
10706 
10707   ins_cost(125);
10708   format %{ "blsrq  $dst, $src" %}
10709 
10710   ins_encode %{
10711     __ blsrq($dst$$Register, $src$$Address);
10712   %}
10713 
10714   ins_pipe(ialu_reg);
10715 %}
10716 
10717 // Or Instructions
10718 // Or Register with Register
10719 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10720 %{
10721   match(Set dst (OrL dst src));
10722   effect(KILL cr);
10723 
10724   format %{ "orq     $dst, $src\t# long" %}
10725   ins_encode %{
10726     __ orq($dst$$Register, $src$$Register);
10727   %}
10728   ins_pipe(ialu_reg_reg);
10729 %}
10730 
10731 // Use any_RegP to match R15 (TLS register) without spilling.
10732 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10733   match(Set dst (OrL dst (CastP2X src)));
10734   effect(KILL cr);
10735 
10736   format %{ "orq     $dst, $src\t# long" %}
10737   ins_encode %{
10738     __ orq($dst$$Register, $src$$Register);
10739   %}
10740   ins_pipe(ialu_reg_reg);
10741 %}
10742 
10743 
10744 // Or Register with Immediate
10745 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10746 %{
10747   match(Set dst (OrL dst src));
10748   effect(KILL cr);
10749 
10750   format %{ "orq     $dst, $src\t# long" %}
10751   ins_encode %{
10752     __ orq($dst$$Register, $src$$constant);
10753   %}
10754   ins_pipe(ialu_reg);
10755 %}
10756 
10757 // Or Register with Memory
10758 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10759 %{
10760   match(Set dst (OrL dst (LoadL src)));
10761   effect(KILL cr);
10762 
10763   ins_cost(125);
10764   format %{ "orq     $dst, $src\t# long" %}
10765   ins_encode %{
10766     __ orq($dst$$Register, $src$$Address);
10767   %}
10768   ins_pipe(ialu_reg_mem);
10769 %}
10770 
10771 // Or Memory with Register
10772 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10773 %{
10774   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10775   effect(KILL cr);
10776 
10777   ins_cost(150);
10778   format %{ "orq     $dst, $src\t# long" %}
10779   ins_encode %{
10780     __ orq($dst$$Address, $src$$Register);
10781   %}
10782   ins_pipe(ialu_mem_reg);
10783 %}
10784 
10785 // Or Memory with Immediate
10786 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10787 %{
10788   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10789   effect(KILL cr);
10790 
10791   ins_cost(125);
10792   format %{ "orq     $dst, $src\t# long" %}
10793   ins_encode %{
10794     __ orq($dst$$Address, $src$$constant);
10795   %}
10796   ins_pipe(ialu_mem_imm);
10797 %}
10798 
10799 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10800 %{
10801   // con should be a pure 64-bit power of 2 immediate
10802   // because AND/OR works well enough for 8/32-bit values.
10803   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10804 
10805   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10806   effect(KILL cr);
10807 
10808   ins_cost(125);
10809   format %{ "btsq    $dst, log2($con)\t# long" %}
10810   ins_encode %{
10811     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10812   %}
10813   ins_pipe(ialu_mem_imm);
10814 %}
10815 
10816 // Xor Instructions
10817 // Xor Register with Register
10818 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10819 %{
10820   match(Set dst (XorL dst src));
10821   effect(KILL cr);
10822 
10823   format %{ "xorq    $dst, $src\t# long" %}
10824   ins_encode %{
10825     __ xorq($dst$$Register, $src$$Register);
10826   %}
10827   ins_pipe(ialu_reg_reg);
10828 %}
10829 
10830 // Xor Register with Immediate -1
10831 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10832   match(Set dst (XorL dst imm));
10833 
10834   format %{ "notq   $dst" %}
10835   ins_encode %{
10836      __ notq($dst$$Register);
10837   %}
10838   ins_pipe(ialu_reg);
10839 %}
10840 
10841 // Xor Register with Immediate
10842 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10843 %{
10844   match(Set dst (XorL dst src));
10845   effect(KILL cr);
10846 
10847   format %{ "xorq    $dst, $src\t# long" %}
10848   ins_encode %{
10849     __ xorq($dst$$Register, $src$$constant);
10850   %}
10851   ins_pipe(ialu_reg);
10852 %}
10853 
10854 // Xor Register with Memory
10855 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10856 %{
10857   match(Set dst (XorL dst (LoadL src)));
10858   effect(KILL cr);
10859 
10860   ins_cost(125);
10861   format %{ "xorq    $dst, $src\t# long" %}
10862   ins_encode %{
10863     __ xorq($dst$$Register, $src$$Address);
10864   %}
10865   ins_pipe(ialu_reg_mem);
10866 %}
10867 
10868 // Xor Memory with Register
10869 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10870 %{
10871   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10872   effect(KILL cr);
10873 
10874   ins_cost(150);
10875   format %{ "xorq    $dst, $src\t# long" %}
10876   ins_encode %{
10877     __ xorq($dst$$Address, $src$$Register);
10878   %}
10879   ins_pipe(ialu_mem_reg);
10880 %}
10881 
10882 // Xor Memory with Immediate
10883 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10884 %{
10885   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10886   effect(KILL cr);
10887 
10888   ins_cost(125);
10889   format %{ "xorq    $dst, $src\t# long" %}
10890   ins_encode %{
10891     __ xorq($dst$$Address, $src$$constant);
10892   %}
10893   ins_pipe(ialu_mem_imm);
10894 %}
10895 
10896 // Convert Int to Boolean
10897 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10898 %{
10899   match(Set dst (Conv2B src));
10900   effect(KILL cr);
10901 
10902   format %{ "testl   $src, $src\t# ci2b\n\t"
10903             "setnz   $dst\n\t"
10904             "movzbl  $dst, $dst" %}
10905   ins_encode %{
10906     __ testl($src$$Register, $src$$Register);
10907     __ set_byte_if_not_zero($dst$$Register);
10908     __ movzbl($dst$$Register, $dst$$Register);
10909   %}
10910   ins_pipe(pipe_slow); // XXX
10911 %}
10912 
10913 // Convert Pointer to Boolean
10914 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10915 %{
10916   match(Set dst (Conv2B src));
10917   effect(KILL cr);
10918 
10919   format %{ "testq   $src, $src\t# cp2b\n\t"
10920             "setnz   $dst\n\t"
10921             "movzbl  $dst, $dst" %}
10922   ins_encode %{
10923     __ testq($src$$Register, $src$$Register);
10924     __ set_byte_if_not_zero($dst$$Register);
10925     __ movzbl($dst$$Register, $dst$$Register);
10926   %}
10927   ins_pipe(pipe_slow); // XXX
10928 %}
10929 
10930 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10931 %{
10932   match(Set dst (CmpLTMask p q));
10933   effect(KILL cr);
10934 
10935   ins_cost(400);
10936   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10937             "setlt   $dst\n\t"
10938             "movzbl  $dst, $dst\n\t"
10939             "negl    $dst" %}
10940   ins_encode %{
10941     __ cmpl($p$$Register, $q$$Register);
10942     __ setl($dst$$Register);
10943     __ movzbl($dst$$Register, $dst$$Register);
10944     __ negl($dst$$Register);
10945   %}
10946   ins_pipe(pipe_slow);
10947 %}
10948 
10949 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10950 %{
10951   match(Set dst (CmpLTMask dst zero));
10952   effect(KILL cr);
10953 
10954   ins_cost(100);
10955   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10956   ins_encode %{
10957     __ sarl($dst$$Register, 31);
10958   %}
10959   ins_pipe(ialu_reg);
10960 %}
10961 
10962 /* Better to save a register than avoid a branch */
10963 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10964 %{
10965   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10966   effect(KILL cr);
10967   ins_cost(300);
10968   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10969             "jge     done\n\t"
10970             "addl    $p,$y\n"
10971             "done:   " %}
10972   ins_encode %{
10973     Register Rp = $p$$Register;
10974     Register Rq = $q$$Register;
10975     Register Ry = $y$$Register;
10976     Label done;
10977     __ subl(Rp, Rq);
10978     __ jccb(Assembler::greaterEqual, done);
10979     __ addl(Rp, Ry);
10980     __ bind(done);
10981   %}
10982   ins_pipe(pipe_cmplt);
10983 %}
10984 
10985 /* Better to save a register than avoid a branch */
10986 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10987 %{
10988   match(Set y (AndI (CmpLTMask p q) y));
10989   effect(KILL cr);
10990 
10991   ins_cost(300);
10992 
10993   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10994             "jlt     done\n\t"
10995             "xorl    $y, $y\n"
10996             "done:   " %}
10997   ins_encode %{
10998     Register Rp = $p$$Register;
10999     Register Rq = $q$$Register;
11000     Register Ry = $y$$Register;
11001     Label done;
11002     __ cmpl(Rp, Rq);
11003     __ jccb(Assembler::less, done);
11004     __ xorl(Ry, Ry);
11005     __ bind(done);
11006   %}
11007   ins_pipe(pipe_cmplt);
11008 %}
11009 
11010 
11011 //---------- FP Instructions------------------------------------------------
11012 
11013 // Really expensive, avoid
11014 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11015 %{
11016   match(Set cr (CmpF src1 src2));
11017 
11018   ins_cost(500);
11019   format %{ "ucomiss $src1, $src2\n\t"
11020             "jnp,s   exit\n\t"
11021             "pushfq\t# saw NaN, set CF\n\t"
11022             "andq    [rsp], #0xffffff2b\n\t"
11023             "popfq\n"
11024     "exit:" %}
11025   ins_encode %{
11026     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11027     emit_cmpfp_fixup(_masm);
11028   %}
11029   ins_pipe(pipe_slow);
11030 %}
11031 
11032 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11033   match(Set cr (CmpF src1 src2));
11034 
11035   ins_cost(100);
11036   format %{ "ucomiss $src1, $src2" %}
11037   ins_encode %{
11038     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11039   %}
11040   ins_pipe(pipe_slow);
11041 %}
11042 
11043 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11044   match(Set cr (CmpF src1 (LoadF src2)));
11045 
11046   ins_cost(100);
11047   format %{ "ucomiss $src1, $src2" %}
11048   ins_encode %{
11049     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11050   %}
11051   ins_pipe(pipe_slow);
11052 %}
11053 
11054 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11055   match(Set cr (CmpF src con));
11056   ins_cost(100);
11057   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11058   ins_encode %{
11059     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11060   %}
11061   ins_pipe(pipe_slow);
11062 %}
11063 
11064 // Really expensive, avoid
11065 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11066 %{
11067   match(Set cr (CmpD src1 src2));
11068 
11069   ins_cost(500);
11070   format %{ "ucomisd $src1, $src2\n\t"
11071             "jnp,s   exit\n\t"
11072             "pushfq\t# saw NaN, set CF\n\t"
11073             "andq    [rsp], #0xffffff2b\n\t"
11074             "popfq\n"
11075     "exit:" %}
11076   ins_encode %{
11077     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11078     emit_cmpfp_fixup(_masm);
11079   %}
11080   ins_pipe(pipe_slow);
11081 %}
11082 
11083 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11084   match(Set cr (CmpD src1 src2));
11085 
11086   ins_cost(100);
11087   format %{ "ucomisd $src1, $src2 test" %}
11088   ins_encode %{
11089     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11090   %}
11091   ins_pipe(pipe_slow);
11092 %}
11093 
11094 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11095   match(Set cr (CmpD src1 (LoadD src2)));
11096 
11097   ins_cost(100);
11098   format %{ "ucomisd $src1, $src2" %}
11099   ins_encode %{
11100     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11101   %}
11102   ins_pipe(pipe_slow);
11103 %}
11104 
11105 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11106   match(Set cr (CmpD src con));
11107   ins_cost(100);
11108   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11109   ins_encode %{
11110     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11111   %}
11112   ins_pipe(pipe_slow);
11113 %}
11114 
11115 // Compare into -1,0,1
11116 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11117 %{
11118   match(Set dst (CmpF3 src1 src2));
11119   effect(KILL cr);
11120 
11121   ins_cost(275);
11122   format %{ "ucomiss $src1, $src2\n\t"
11123             "movl    $dst, #-1\n\t"
11124             "jp,s    done\n\t"
11125             "jb,s    done\n\t"
11126             "setne   $dst\n\t"
11127             "movzbl  $dst, $dst\n"
11128     "done:" %}
11129   ins_encode %{
11130     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11131     emit_cmpfp3(_masm, $dst$$Register);
11132   %}
11133   ins_pipe(pipe_slow);
11134 %}
11135 
11136 // Compare into -1,0,1
11137 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11138 %{
11139   match(Set dst (CmpF3 src1 (LoadF src2)));
11140   effect(KILL cr);
11141 
11142   ins_cost(275);
11143   format %{ "ucomiss $src1, $src2\n\t"
11144             "movl    $dst, #-1\n\t"
11145             "jp,s    done\n\t"
11146             "jb,s    done\n\t"
11147             "setne   $dst\n\t"
11148             "movzbl  $dst, $dst\n"
11149     "done:" %}
11150   ins_encode %{
11151     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11152     emit_cmpfp3(_masm, $dst$$Register);
11153   %}
11154   ins_pipe(pipe_slow);
11155 %}
11156 
11157 // Compare into -1,0,1
11158 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11159   match(Set dst (CmpF3 src con));
11160   effect(KILL cr);
11161 
11162   ins_cost(275);
11163   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11164             "movl    $dst, #-1\n\t"
11165             "jp,s    done\n\t"
11166             "jb,s    done\n\t"
11167             "setne   $dst\n\t"
11168             "movzbl  $dst, $dst\n"
11169     "done:" %}
11170   ins_encode %{
11171     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11172     emit_cmpfp3(_masm, $dst$$Register);
11173   %}
11174   ins_pipe(pipe_slow);
11175 %}
11176 
11177 // Compare into -1,0,1
11178 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11179 %{
11180   match(Set dst (CmpD3 src1 src2));
11181   effect(KILL cr);
11182 
11183   ins_cost(275);
11184   format %{ "ucomisd $src1, $src2\n\t"
11185             "movl    $dst, #-1\n\t"
11186             "jp,s    done\n\t"
11187             "jb,s    done\n\t"
11188             "setne   $dst\n\t"
11189             "movzbl  $dst, $dst\n"
11190     "done:" %}
11191   ins_encode %{
11192     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11193     emit_cmpfp3(_masm, $dst$$Register);
11194   %}
11195   ins_pipe(pipe_slow);
11196 %}
11197 
11198 // Compare into -1,0,1
11199 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11200 %{
11201   match(Set dst (CmpD3 src1 (LoadD src2)));
11202   effect(KILL cr);
11203 
11204   ins_cost(275);
11205   format %{ "ucomisd $src1, $src2\n\t"
11206             "movl    $dst, #-1\n\t"
11207             "jp,s    done\n\t"
11208             "jb,s    done\n\t"
11209             "setne   $dst\n\t"
11210             "movzbl  $dst, $dst\n"
11211     "done:" %}
11212   ins_encode %{
11213     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11214     emit_cmpfp3(_masm, $dst$$Register);
11215   %}
11216   ins_pipe(pipe_slow);
11217 %}
11218 
11219 // Compare into -1,0,1
11220 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11221   match(Set dst (CmpD3 src con));
11222   effect(KILL cr);
11223 
11224   ins_cost(275);
11225   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11226             "movl    $dst, #-1\n\t"
11227             "jp,s    done\n\t"
11228             "jb,s    done\n\t"
11229             "setne   $dst\n\t"
11230             "movzbl  $dst, $dst\n"
11231     "done:" %}
11232   ins_encode %{
11233     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11234     emit_cmpfp3(_masm, $dst$$Register);
11235   %}
11236   ins_pipe(pipe_slow);
11237 %}
11238 
11239 //----------Arithmetic Conversion Instructions---------------------------------
11240 
11241 instruct convF2D_reg_reg(regD dst, regF src)
11242 %{
11243   match(Set dst (ConvF2D src));
11244 
11245   format %{ "cvtss2sd $dst, $src" %}
11246   ins_encode %{
11247     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11248   %}
11249   ins_pipe(pipe_slow); // XXX
11250 %}
11251 
11252 instruct convF2D_reg_mem(regD dst, memory src)
11253 %{
11254   match(Set dst (ConvF2D (LoadF src)));
11255 
11256   format %{ "cvtss2sd $dst, $src" %}
11257   ins_encode %{
11258     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11259   %}
11260   ins_pipe(pipe_slow); // XXX
11261 %}
11262 
11263 instruct convD2F_reg_reg(regF dst, regD src)
11264 %{
11265   match(Set dst (ConvD2F src));
11266 
11267   format %{ "cvtsd2ss $dst, $src" %}
11268   ins_encode %{
11269     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11270   %}
11271   ins_pipe(pipe_slow); // XXX
11272 %}
11273 
11274 instruct convD2F_reg_mem(regF dst, memory src)
11275 %{
11276   match(Set dst (ConvD2F (LoadD src)));
11277 
11278   format %{ "cvtsd2ss $dst, $src" %}
11279   ins_encode %{
11280     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11281   %}
11282   ins_pipe(pipe_slow); // XXX
11283 %}
11284 
11285 // XXX do mem variants
11286 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11287 %{
11288   match(Set dst (ConvF2I src));
11289   effect(KILL cr);
11290   format %{ "convert_f2i $dst,$src" %}
11291   ins_encode %{
11292     __ convert_f2i($dst$$Register, $src$$XMMRegister);
11293   %}
11294   ins_pipe(pipe_slow);
11295 %}
11296 
11297 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11298 %{
11299   match(Set dst (ConvF2L src));
11300   effect(KILL cr);
11301   format %{ "convert_f2l $dst,$src"%}
11302   ins_encode %{
11303     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11304   %}
11305   ins_pipe(pipe_slow);
11306 %}
11307 
11308 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11309 %{
11310   match(Set dst (ConvD2I src));
11311   effect(KILL cr);
11312   format %{ "convert_d2i $dst,$src"%}
11313   ins_encode %{
11314     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11315   %}
11316   ins_pipe(pipe_slow);
11317 %}
11318 
11319 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11320 %{
11321   match(Set dst (ConvD2L src));
11322   effect(KILL cr);
11323   format %{ "convert_d2l $dst,$src"%}
11324   ins_encode %{
11325     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11326   %}
11327   ins_pipe(pipe_slow);
11328 %}
11329 
11330 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11331 %{
11332   match(Set dst (RoundD src));
11333   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11334   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11335   ins_encode %{
11336     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11337   %}
11338   ins_pipe(pipe_slow);
11339 %}
11340 
11341 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11342 %{
11343   match(Set dst (RoundF src));
11344   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11345   format %{ "round_float $dst,$src" %}
11346   ins_encode %{
11347     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11348   %}
11349   ins_pipe(pipe_slow);
11350 %}
11351 
11352 instruct convI2F_reg_reg(regF dst, rRegI src)
11353 %{
11354   predicate(!UseXmmI2F);
11355   match(Set dst (ConvI2F src));
11356 
11357   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11358   ins_encode %{
11359     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11360   %}
11361   ins_pipe(pipe_slow); // XXX
11362 %}
11363 
11364 instruct convI2F_reg_mem(regF dst, memory src)
11365 %{
11366   match(Set dst (ConvI2F (LoadI src)));
11367 
11368   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11369   ins_encode %{
11370     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11371   %}
11372   ins_pipe(pipe_slow); // XXX
11373 %}
11374 
11375 instruct convI2D_reg_reg(regD dst, rRegI src)
11376 %{
11377   predicate(!UseXmmI2D);
11378   match(Set dst (ConvI2D src));
11379 
11380   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11381   ins_encode %{
11382     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11383   %}
11384   ins_pipe(pipe_slow); // XXX
11385 %}
11386 
11387 instruct convI2D_reg_mem(regD dst, memory src)
11388 %{
11389   match(Set dst (ConvI2D (LoadI src)));
11390 
11391   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11392   ins_encode %{
11393     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11394   %}
11395   ins_pipe(pipe_slow); // XXX
11396 %}
11397 
11398 instruct convXI2F_reg(regF dst, rRegI src)
11399 %{
11400   predicate(UseXmmI2F);
11401   match(Set dst (ConvI2F src));
11402 
11403   format %{ "movdl $dst, $src\n\t"
11404             "cvtdq2psl $dst, $dst\t# i2f" %}
11405   ins_encode %{
11406     __ movdl($dst$$XMMRegister, $src$$Register);
11407     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11408   %}
11409   ins_pipe(pipe_slow); // XXX
11410 %}
11411 
11412 instruct convXI2D_reg(regD dst, rRegI src)
11413 %{
11414   predicate(UseXmmI2D);
11415   match(Set dst (ConvI2D src));
11416 
11417   format %{ "movdl $dst, $src\n\t"
11418             "cvtdq2pdl $dst, $dst\t# i2d" %}
11419   ins_encode %{
11420     __ movdl($dst$$XMMRegister, $src$$Register);
11421     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11422   %}
11423   ins_pipe(pipe_slow); // XXX
11424 %}
11425 
11426 instruct convL2F_reg_reg(regF dst, rRegL src)
11427 %{
11428   match(Set dst (ConvL2F src));
11429 
11430   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11431   ins_encode %{
11432     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11433   %}
11434   ins_pipe(pipe_slow); // XXX
11435 %}
11436 
11437 instruct convL2F_reg_mem(regF dst, memory src)
11438 %{
11439   match(Set dst (ConvL2F (LoadL src)));
11440 
11441   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11442   ins_encode %{
11443     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11444   %}
11445   ins_pipe(pipe_slow); // XXX
11446 %}
11447 
11448 instruct convL2D_reg_reg(regD dst, rRegL src)
11449 %{
11450   match(Set dst (ConvL2D src));
11451 
11452   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11453   ins_encode %{
11454     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11455   %}
11456   ins_pipe(pipe_slow); // XXX
11457 %}
11458 
11459 instruct convL2D_reg_mem(regD dst, memory src)
11460 %{
11461   match(Set dst (ConvL2D (LoadL src)));
11462 
11463   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11464   ins_encode %{
11465     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11466   %}
11467   ins_pipe(pipe_slow); // XXX
11468 %}
11469 
11470 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11471 %{
11472   match(Set dst (ConvI2L src));
11473 
11474   ins_cost(125);
11475   format %{ "movslq  $dst, $src\t# i2l" %}
11476   ins_encode %{
11477     __ movslq($dst$$Register, $src$$Register);
11478   %}
11479   ins_pipe(ialu_reg_reg);
11480 %}
11481 
11482 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11483 // %{
11484 //   match(Set dst (ConvI2L src));
11485 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11486 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11487 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11488 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11489 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11490 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11491 
11492 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11493 //   ins_encode(enc_copy(dst, src));
11494 // //   opcode(0x63); // needs REX.W
11495 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11496 //   ins_pipe(ialu_reg_reg);
11497 // %}
11498 
11499 // Zero-extend convert int to long
11500 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11501 %{
11502   match(Set dst (AndL (ConvI2L src) mask));
11503 
11504   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11505   ins_encode %{
11506     if ($dst$$reg != $src$$reg) {
11507       __ movl($dst$$Register, $src$$Register);
11508     }
11509   %}
11510   ins_pipe(ialu_reg_reg);
11511 %}
11512 
11513 // Zero-extend convert int to long
11514 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11515 %{
11516   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11517 
11518   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11519   ins_encode %{
11520     __ movl($dst$$Register, $src$$Address);
11521   %}
11522   ins_pipe(ialu_reg_mem);
11523 %}
11524 
11525 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11526 %{
11527   match(Set dst (AndL src mask));
11528 
11529   format %{ "movl    $dst, $src\t# zero-extend long" %}
11530   ins_encode %{
11531     __ movl($dst$$Register, $src$$Register);
11532   %}
11533   ins_pipe(ialu_reg_reg);
11534 %}
11535 
11536 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11537 %{
11538   match(Set dst (ConvL2I src));
11539 
11540   format %{ "movl    $dst, $src\t# l2i" %}
11541   ins_encode %{
11542     __ movl($dst$$Register, $src$$Register);
11543   %}
11544   ins_pipe(ialu_reg_reg);
11545 %}
11546 
11547 
11548 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11549   match(Set dst (MoveF2I src));
11550   effect(DEF dst, USE src);
11551 
11552   ins_cost(125);
11553   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11554   ins_encode %{
11555     __ movl($dst$$Register, Address(rsp, $src$$disp));
11556   %}
11557   ins_pipe(ialu_reg_mem);
11558 %}
11559 
11560 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11561   match(Set dst (MoveI2F src));
11562   effect(DEF dst, USE src);
11563 
11564   ins_cost(125);
11565   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11566   ins_encode %{
11567     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11568   %}
11569   ins_pipe(pipe_slow);
11570 %}
11571 
11572 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11573   match(Set dst (MoveD2L src));
11574   effect(DEF dst, USE src);
11575 
11576   ins_cost(125);
11577   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11578   ins_encode %{
11579     __ movq($dst$$Register, Address(rsp, $src$$disp));
11580   %}
11581   ins_pipe(ialu_reg_mem);
11582 %}
11583 
11584 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11585   predicate(!UseXmmLoadAndClearUpper);
11586   match(Set dst (MoveL2D src));
11587   effect(DEF dst, USE src);
11588 
11589   ins_cost(125);
11590   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11591   ins_encode %{
11592     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11593   %}
11594   ins_pipe(pipe_slow);
11595 %}
11596 
11597 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11598   predicate(UseXmmLoadAndClearUpper);
11599   match(Set dst (MoveL2D src));
11600   effect(DEF dst, USE src);
11601 
11602   ins_cost(125);
11603   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11604   ins_encode %{
11605     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11606   %}
11607   ins_pipe(pipe_slow);
11608 %}
11609 
11610 
11611 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11612   match(Set dst (MoveF2I src));
11613   effect(DEF dst, USE src);
11614 
11615   ins_cost(95); // XXX
11616   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11617   ins_encode %{
11618     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11619   %}
11620   ins_pipe(pipe_slow);
11621 %}
11622 
11623 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11624   match(Set dst (MoveI2F src));
11625   effect(DEF dst, USE src);
11626 
11627   ins_cost(100);
11628   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11629   ins_encode %{
11630     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11631   %}
11632   ins_pipe( ialu_mem_reg );
11633 %}
11634 
11635 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11636   match(Set dst (MoveD2L src));
11637   effect(DEF dst, USE src);
11638 
11639   ins_cost(95); // XXX
11640   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11641   ins_encode %{
11642     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11643   %}
11644   ins_pipe(pipe_slow);
11645 %}
11646 
11647 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11648   match(Set dst (MoveL2D src));
11649   effect(DEF dst, USE src);
11650 
11651   ins_cost(100);
11652   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11653   ins_encode %{
11654     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11655   %}
11656   ins_pipe(ialu_mem_reg);
11657 %}
11658 
11659 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11660   match(Set dst (MoveF2I src));
11661   effect(DEF dst, USE src);
11662   ins_cost(85);
11663   format %{ "movd    $dst,$src\t# MoveF2I" %}
11664   ins_encode %{
11665     __ movdl($dst$$Register, $src$$XMMRegister);
11666   %}
11667   ins_pipe( pipe_slow );
11668 %}
11669 
11670 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11671   match(Set dst (MoveD2L src));
11672   effect(DEF dst, USE src);
11673   ins_cost(85);
11674   format %{ "movd    $dst,$src\t# MoveD2L" %}
11675   ins_encode %{
11676     __ movdq($dst$$Register, $src$$XMMRegister);
11677   %}
11678   ins_pipe( pipe_slow );
11679 %}
11680 
11681 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11682   match(Set dst (MoveI2F src));
11683   effect(DEF dst, USE src);
11684   ins_cost(100);
11685   format %{ "movd    $dst,$src\t# MoveI2F" %}
11686   ins_encode %{
11687     __ movdl($dst$$XMMRegister, $src$$Register);
11688   %}
11689   ins_pipe( pipe_slow );
11690 %}
11691 
11692 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11693   match(Set dst (MoveL2D src));
11694   effect(DEF dst, USE src);
11695   ins_cost(100);
11696   format %{ "movd    $dst,$src\t# MoveL2D" %}
11697   ins_encode %{
11698      __ movdq($dst$$XMMRegister, $src$$Register);
11699   %}
11700   ins_pipe( pipe_slow );
11701 %}
11702 
11703 // Fast clearing of an array
11704 // Small ClearArray non-AVX512.
11705 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11706                   Universe dummy, rFlagsReg cr)
11707 %{
11708   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11709   match(Set dummy (ClearArray cnt base));
11710   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11711 
11712   format %{ $$template
11713     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11714     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11715     $$emit$$"jg      LARGE\n\t"
11716     $$emit$$"dec     rcx\n\t"
11717     $$emit$$"js      DONE\t# Zero length\n\t"
11718     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11719     $$emit$$"dec     rcx\n\t"
11720     $$emit$$"jge     LOOP\n\t"
11721     $$emit$$"jmp     DONE\n\t"
11722     $$emit$$"# LARGE:\n\t"
11723     if (UseFastStosb) {
11724        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11725        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11726     } else if (UseXMMForObjInit) {
11727        $$emit$$"mov     rdi,rax\n\t"
11728        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11729        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11730        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11731        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11732        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11733        $$emit$$"add     0x40,rax\n\t"
11734        $$emit$$"# L_zero_64_bytes:\n\t"
11735        $$emit$$"sub     0x8,rcx\n\t"
11736        $$emit$$"jge     L_loop\n\t"
11737        $$emit$$"add     0x4,rcx\n\t"
11738        $$emit$$"jl      L_tail\n\t"
11739        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11740        $$emit$$"add     0x20,rax\n\t"
11741        $$emit$$"sub     0x4,rcx\n\t"
11742        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11743        $$emit$$"add     0x4,rcx\n\t"
11744        $$emit$$"jle     L_end\n\t"
11745        $$emit$$"dec     rcx\n\t"
11746        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11747        $$emit$$"vmovq   xmm0,(rax)\n\t"
11748        $$emit$$"add     0x8,rax\n\t"
11749        $$emit$$"dec     rcx\n\t"
11750        $$emit$$"jge     L_sloop\n\t"
11751        $$emit$$"# L_end:\n\t"
11752     } else {
11753        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11754     }
11755     $$emit$$"# DONE"
11756   %}
11757   ins_encode %{
11758     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11759                  $tmp$$XMMRegister, false, knoreg);
11760   %}
11761   ins_pipe(pipe_slow);
11762 %}
11763 
11764 // Small ClearArray AVX512 non-constant length.
11765 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11766                        Universe dummy, rFlagsReg cr)
11767 %{
11768   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11769   match(Set dummy (ClearArray cnt base));
11770   ins_cost(125);
11771   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11772 
11773   format %{ $$template
11774     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11775     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11776     $$emit$$"jg      LARGE\n\t"
11777     $$emit$$"dec     rcx\n\t"
11778     $$emit$$"js      DONE\t# Zero length\n\t"
11779     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11780     $$emit$$"dec     rcx\n\t"
11781     $$emit$$"jge     LOOP\n\t"
11782     $$emit$$"jmp     DONE\n\t"
11783     $$emit$$"# LARGE:\n\t"
11784     if (UseFastStosb) {
11785        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11786        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11787     } else if (UseXMMForObjInit) {
11788        $$emit$$"mov     rdi,rax\n\t"
11789        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11790        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11791        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11792        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11793        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11794        $$emit$$"add     0x40,rax\n\t"
11795        $$emit$$"# L_zero_64_bytes:\n\t"
11796        $$emit$$"sub     0x8,rcx\n\t"
11797        $$emit$$"jge     L_loop\n\t"
11798        $$emit$$"add     0x4,rcx\n\t"
11799        $$emit$$"jl      L_tail\n\t"
11800        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11801        $$emit$$"add     0x20,rax\n\t"
11802        $$emit$$"sub     0x4,rcx\n\t"
11803        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11804        $$emit$$"add     0x4,rcx\n\t"
11805        $$emit$$"jle     L_end\n\t"
11806        $$emit$$"dec     rcx\n\t"
11807        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11808        $$emit$$"vmovq   xmm0,(rax)\n\t"
11809        $$emit$$"add     0x8,rax\n\t"
11810        $$emit$$"dec     rcx\n\t"
11811        $$emit$$"jge     L_sloop\n\t"
11812        $$emit$$"# L_end:\n\t"
11813     } else {
11814        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11815     }
11816     $$emit$$"# DONE"
11817   %}
11818   ins_encode %{
11819     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11820                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11821   %}
11822   ins_pipe(pipe_slow);
11823 %}
11824 
11825 // Large ClearArray non-AVX512.
11826 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11827                         Universe dummy, rFlagsReg cr)
11828 %{
11829   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11830   match(Set dummy (ClearArray cnt base));
11831   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11832 
11833   format %{ $$template
11834     if (UseFastStosb) {
11835        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11836        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11837        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11838     } else if (UseXMMForObjInit) {
11839        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11840        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11841        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11842        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11843        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11844        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11845        $$emit$$"add     0x40,rax\n\t"
11846        $$emit$$"# L_zero_64_bytes:\n\t"
11847        $$emit$$"sub     0x8,rcx\n\t"
11848        $$emit$$"jge     L_loop\n\t"
11849        $$emit$$"add     0x4,rcx\n\t"
11850        $$emit$$"jl      L_tail\n\t"
11851        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11852        $$emit$$"add     0x20,rax\n\t"
11853        $$emit$$"sub     0x4,rcx\n\t"
11854        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11855        $$emit$$"add     0x4,rcx\n\t"
11856        $$emit$$"jle     L_end\n\t"
11857        $$emit$$"dec     rcx\n\t"
11858        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11859        $$emit$$"vmovq   xmm0,(rax)\n\t"
11860        $$emit$$"add     0x8,rax\n\t"
11861        $$emit$$"dec     rcx\n\t"
11862        $$emit$$"jge     L_sloop\n\t"
11863        $$emit$$"# L_end:\n\t"
11864     } else {
11865        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11866        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11867     }
11868   %}
11869   ins_encode %{
11870     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11871                  $tmp$$XMMRegister, true, knoreg);
11872   %}
11873   ins_pipe(pipe_slow);
11874 %}
11875 
11876 // Large ClearArray AVX512.
11877 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11878                              Universe dummy, rFlagsReg cr)
11879 %{
11880   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11881   match(Set dummy (ClearArray cnt base));
11882   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11883 
11884   format %{ $$template
11885     if (UseFastStosb) {
11886        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11887        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11888        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11889     } else if (UseXMMForObjInit) {
11890        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11891        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11892        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11893        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11894        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11895        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11896        $$emit$$"add     0x40,rax\n\t"
11897        $$emit$$"# L_zero_64_bytes:\n\t"
11898        $$emit$$"sub     0x8,rcx\n\t"
11899        $$emit$$"jge     L_loop\n\t"
11900        $$emit$$"add     0x4,rcx\n\t"
11901        $$emit$$"jl      L_tail\n\t"
11902        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11903        $$emit$$"add     0x20,rax\n\t"
11904        $$emit$$"sub     0x4,rcx\n\t"
11905        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11906        $$emit$$"add     0x4,rcx\n\t"
11907        $$emit$$"jle     L_end\n\t"
11908        $$emit$$"dec     rcx\n\t"
11909        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11910        $$emit$$"vmovq   xmm0,(rax)\n\t"
11911        $$emit$$"add     0x8,rax\n\t"
11912        $$emit$$"dec     rcx\n\t"
11913        $$emit$$"jge     L_sloop\n\t"
11914        $$emit$$"# L_end:\n\t"
11915     } else {
11916        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11917        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11918     }
11919   %}
11920   ins_encode %{
11921     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11922                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11923   %}
11924   ins_pipe(pipe_slow);
11925 %}
11926 
11927 // Small ClearArray AVX512 constant length.
11928 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11929 %{
11930   predicate(!((ClearArrayNode*)n)->is_large() &&
11931               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11932   match(Set dummy (ClearArray cnt base));
11933   ins_cost(100);
11934   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11935   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11936   ins_encode %{
11937    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11938   %}
11939   ins_pipe(pipe_slow);
11940 %}
11941 
11942 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11943                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11944 %{
11945   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11946   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11947   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11948 
11949   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11950   ins_encode %{
11951     __ string_compare($str1$$Register, $str2$$Register,
11952                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11953                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11954   %}
11955   ins_pipe( pipe_slow );
11956 %}
11957 
11958 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11959                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11960 %{
11961   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11962   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11963   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11964 
11965   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11966   ins_encode %{
11967     __ string_compare($str1$$Register, $str2$$Register,
11968                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11969                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11970   %}
11971   ins_pipe( pipe_slow );
11972 %}
11973 
11974 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11975                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11976 %{
11977   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11978   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11979   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11980 
11981   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11982   ins_encode %{
11983     __ string_compare($str1$$Register, $str2$$Register,
11984                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11985                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11986   %}
11987   ins_pipe( pipe_slow );
11988 %}
11989 
11990 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11991                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11992 %{
11993   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11994   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11995   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11996 
11997   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11998   ins_encode %{
11999     __ string_compare($str1$$Register, $str2$$Register,
12000                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12001                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12002   %}
12003   ins_pipe( pipe_slow );
12004 %}
12005 
12006 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12007                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12008 %{
12009   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12010   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12011   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12012 
12013   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12014   ins_encode %{
12015     __ string_compare($str1$$Register, $str2$$Register,
12016                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12017                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12018   %}
12019   ins_pipe( pipe_slow );
12020 %}
12021 
12022 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12023                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12024 %{
12025   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12026   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12027   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12028 
12029   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12030   ins_encode %{
12031     __ string_compare($str1$$Register, $str2$$Register,
12032                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12033                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12034   %}
12035   ins_pipe( pipe_slow );
12036 %}
12037 
12038 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12039                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12040 %{
12041   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12042   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12043   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12044 
12045   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12046   ins_encode %{
12047     __ string_compare($str2$$Register, $str1$$Register,
12048                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12049                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12050   %}
12051   ins_pipe( pipe_slow );
12052 %}
12053 
12054 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12055                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12056 %{
12057   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12058   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12059   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12060 
12061   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12062   ins_encode %{
12063     __ string_compare($str2$$Register, $str1$$Register,
12064                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12065                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12066   %}
12067   ins_pipe( pipe_slow );
12068 %}
12069 
12070 // fast search of substring with known size.
12071 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12072                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12073 %{
12074   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12075   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12076   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12077 
12078   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12079   ins_encode %{
12080     int icnt2 = (int)$int_cnt2$$constant;
12081     if (icnt2 >= 16) {
12082       // IndexOf for constant substrings with size >= 16 elements
12083       // which don't need to be loaded through stack.
12084       __ string_indexofC8($str1$$Register, $str2$$Register,
12085                           $cnt1$$Register, $cnt2$$Register,
12086                           icnt2, $result$$Register,
12087                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12088     } else {
12089       // Small strings are loaded through stack if they cross page boundary.
12090       __ string_indexof($str1$$Register, $str2$$Register,
12091                         $cnt1$$Register, $cnt2$$Register,
12092                         icnt2, $result$$Register,
12093                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12094     }
12095   %}
12096   ins_pipe( pipe_slow );
12097 %}
12098 
12099 // fast search of substring with known size.
12100 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12101                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12102 %{
12103   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12104   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12105   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12106 
12107   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12108   ins_encode %{
12109     int icnt2 = (int)$int_cnt2$$constant;
12110     if (icnt2 >= 8) {
12111       // IndexOf for constant substrings with size >= 8 elements
12112       // which don't need to be loaded through stack.
12113       __ string_indexofC8($str1$$Register, $str2$$Register,
12114                           $cnt1$$Register, $cnt2$$Register,
12115                           icnt2, $result$$Register,
12116                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12117     } else {
12118       // Small strings are loaded through stack if they cross page boundary.
12119       __ string_indexof($str1$$Register, $str2$$Register,
12120                         $cnt1$$Register, $cnt2$$Register,
12121                         icnt2, $result$$Register,
12122                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12123     }
12124   %}
12125   ins_pipe( pipe_slow );
12126 %}
12127 
12128 // fast search of substring with known size.
12129 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12130                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12131 %{
12132   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12133   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12134   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12135 
12136   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12137   ins_encode %{
12138     int icnt2 = (int)$int_cnt2$$constant;
12139     if (icnt2 >= 8) {
12140       // IndexOf for constant substrings with size >= 8 elements
12141       // which don't need to be loaded through stack.
12142       __ string_indexofC8($str1$$Register, $str2$$Register,
12143                           $cnt1$$Register, $cnt2$$Register,
12144                           icnt2, $result$$Register,
12145                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12146     } else {
12147       // Small strings are loaded through stack if they cross page boundary.
12148       __ string_indexof($str1$$Register, $str2$$Register,
12149                         $cnt1$$Register, $cnt2$$Register,
12150                         icnt2, $result$$Register,
12151                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12152     }
12153   %}
12154   ins_pipe( pipe_slow );
12155 %}
12156 
12157 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12158                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12159 %{
12160   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12161   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12162   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12163 
12164   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12165   ins_encode %{
12166     __ string_indexof($str1$$Register, $str2$$Register,
12167                       $cnt1$$Register, $cnt2$$Register,
12168                       (-1), $result$$Register,
12169                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12170   %}
12171   ins_pipe( pipe_slow );
12172 %}
12173 
12174 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12175                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12176 %{
12177   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12178   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12179   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12180 
12181   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12182   ins_encode %{
12183     __ string_indexof($str1$$Register, $str2$$Register,
12184                       $cnt1$$Register, $cnt2$$Register,
12185                       (-1), $result$$Register,
12186                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12187   %}
12188   ins_pipe( pipe_slow );
12189 %}
12190 
12191 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12192                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12193 %{
12194   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12195   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12196   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12197 
12198   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12199   ins_encode %{
12200     __ string_indexof($str1$$Register, $str2$$Register,
12201                       $cnt1$$Register, $cnt2$$Register,
12202                       (-1), $result$$Register,
12203                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12204   %}
12205   ins_pipe( pipe_slow );
12206 %}
12207 
12208 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12209                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12210 %{
12211   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12212   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12213   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12214   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12215   ins_encode %{
12216     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12217                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12218   %}
12219   ins_pipe( pipe_slow );
12220 %}
12221 
12222 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12223                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12224 %{
12225   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12226   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12227   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12228   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12229   ins_encode %{
12230     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12231                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12232   %}
12233   ins_pipe( pipe_slow );
12234 %}
12235 
12236 // fast string equals
12237 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12238                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12239 %{
12240   predicate(!VM_Version::supports_avx512vlbw());
12241   match(Set result (StrEquals (Binary str1 str2) cnt));
12242   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12243 
12244   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12245   ins_encode %{
12246     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12247                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12248                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12249   %}
12250   ins_pipe( pipe_slow );
12251 %}
12252 
12253 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12254                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12255 %{
12256   predicate(VM_Version::supports_avx512vlbw());
12257   match(Set result (StrEquals (Binary str1 str2) cnt));
12258   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12259 
12260   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12261   ins_encode %{
12262     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12263                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12264                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12265   %}
12266   ins_pipe( pipe_slow );
12267 %}
12268 
12269 // fast array equals
12270 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12271                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12272 %{
12273   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12274   match(Set result (AryEq ary1 ary2));
12275   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12276 
12277   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12278   ins_encode %{
12279     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12280                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12281                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12282   %}
12283   ins_pipe( pipe_slow );
12284 %}
12285 
12286 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12287                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12288 %{
12289   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12290   match(Set result (AryEq ary1 ary2));
12291   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12292 
12293   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12294   ins_encode %{
12295     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12296                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12297                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12298   %}
12299   ins_pipe( pipe_slow );
12300 %}
12301 
12302 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12303                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12304 %{
12305   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12306   match(Set result (AryEq ary1 ary2));
12307   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12308 
12309   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12310   ins_encode %{
12311     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12312                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12313                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12314   %}
12315   ins_pipe( pipe_slow );
12316 %}
12317 
12318 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12319                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12320 %{
12321   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12322   match(Set result (AryEq ary1 ary2));
12323   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12324 
12325   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12326   ins_encode %{
12327     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12328                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12329                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12330   %}
12331   ins_pipe( pipe_slow );
12332 %}
12333 
12334 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12335                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12336 %{
12337   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12338   match(Set result (CountPositives ary1 len));
12339   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12340 
12341   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12342   ins_encode %{
12343     __ count_positives($ary1$$Register, $len$$Register,
12344                        $result$$Register, $tmp3$$Register,
12345                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12346   %}
12347   ins_pipe( pipe_slow );
12348 %}
12349 
12350 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12351                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12352 %{
12353   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12354   match(Set result (CountPositives ary1 len));
12355   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12356 
12357   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12358   ins_encode %{
12359     __ count_positives($ary1$$Register, $len$$Register,
12360                        $result$$Register, $tmp3$$Register,
12361                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12362   %}
12363   ins_pipe( pipe_slow );
12364 %}
12365 
12366 // fast char[] to byte[] compression
12367 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12368                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12369   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12370   match(Set result (StrCompressedCopy src (Binary dst len)));
12371   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12372          USE_KILL len, KILL tmp5, KILL cr);
12373 
12374   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12375   ins_encode %{
12376     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12377                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12378                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12379                            knoreg, knoreg);
12380   %}
12381   ins_pipe( pipe_slow );
12382 %}
12383 
12384 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12385                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12386   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12387   match(Set result (StrCompressedCopy src (Binary dst len)));
12388   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12389          USE_KILL len, KILL tmp5, KILL cr);
12390 
12391   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12392   ins_encode %{
12393     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12394                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12395                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12396                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12397   %}
12398   ins_pipe( pipe_slow );
12399 %}
12400 // fast byte[] to char[] inflation
12401 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12402                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12403   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12404   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12405   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12406 
12407   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12408   ins_encode %{
12409     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12410                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12411   %}
12412   ins_pipe( pipe_slow );
12413 %}
12414 
12415 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12416                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12417   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12418   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12419   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12420 
12421   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12422   ins_encode %{
12423     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12424                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12425   %}
12426   ins_pipe( pipe_slow );
12427 %}
12428 
12429 // encode char[] to byte[] in ISO_8859_1
12430 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12431                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12432                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12433   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12434   match(Set result (EncodeISOArray src (Binary dst len)));
12435   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12436 
12437   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12438   ins_encode %{
12439     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12440                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12441                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12442   %}
12443   ins_pipe( pipe_slow );
12444 %}
12445 
12446 // encode char[] to byte[] in ASCII
12447 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12448                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12449                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12450   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12451   match(Set result (EncodeISOArray src (Binary dst len)));
12452   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12453 
12454   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12455   ins_encode %{
12456     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12457                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12458                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12459   %}
12460   ins_pipe( pipe_slow );
12461 %}
12462 
12463 //----------Overflow Math Instructions-----------------------------------------
12464 
12465 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12466 %{
12467   match(Set cr (OverflowAddI op1 op2));
12468   effect(DEF cr, USE_KILL op1, USE op2);
12469 
12470   format %{ "addl    $op1, $op2\t# overflow check int" %}
12471 
12472   ins_encode %{
12473     __ addl($op1$$Register, $op2$$Register);
12474   %}
12475   ins_pipe(ialu_reg_reg);
12476 %}
12477 
12478 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12479 %{
12480   match(Set cr (OverflowAddI op1 op2));
12481   effect(DEF cr, USE_KILL op1, USE op2);
12482 
12483   format %{ "addl    $op1, $op2\t# overflow check int" %}
12484 
12485   ins_encode %{
12486     __ addl($op1$$Register, $op2$$constant);
12487   %}
12488   ins_pipe(ialu_reg_reg);
12489 %}
12490 
12491 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12492 %{
12493   match(Set cr (OverflowAddL op1 op2));
12494   effect(DEF cr, USE_KILL op1, USE op2);
12495 
12496   format %{ "addq    $op1, $op2\t# overflow check long" %}
12497   ins_encode %{
12498     __ addq($op1$$Register, $op2$$Register);
12499   %}
12500   ins_pipe(ialu_reg_reg);
12501 %}
12502 
12503 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12504 %{
12505   match(Set cr (OverflowAddL op1 op2));
12506   effect(DEF cr, USE_KILL op1, USE op2);
12507 
12508   format %{ "addq    $op1, $op2\t# overflow check long" %}
12509   ins_encode %{
12510     __ addq($op1$$Register, $op2$$constant);
12511   %}
12512   ins_pipe(ialu_reg_reg);
12513 %}
12514 
12515 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12516 %{
12517   match(Set cr (OverflowSubI op1 op2));
12518 
12519   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12520   ins_encode %{
12521     __ cmpl($op1$$Register, $op2$$Register);
12522   %}
12523   ins_pipe(ialu_reg_reg);
12524 %}
12525 
12526 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12527 %{
12528   match(Set cr (OverflowSubI op1 op2));
12529 
12530   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12531   ins_encode %{
12532     __ cmpl($op1$$Register, $op2$$constant);
12533   %}
12534   ins_pipe(ialu_reg_reg);
12535 %}
12536 
12537 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12538 %{
12539   match(Set cr (OverflowSubL op1 op2));
12540 
12541   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12542   ins_encode %{
12543     __ cmpq($op1$$Register, $op2$$Register);
12544   %}
12545   ins_pipe(ialu_reg_reg);
12546 %}
12547 
12548 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12549 %{
12550   match(Set cr (OverflowSubL op1 op2));
12551 
12552   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12553   ins_encode %{
12554     __ cmpq($op1$$Register, $op2$$constant);
12555   %}
12556   ins_pipe(ialu_reg_reg);
12557 %}
12558 
12559 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12560 %{
12561   match(Set cr (OverflowSubI zero op2));
12562   effect(DEF cr, USE_KILL op2);
12563 
12564   format %{ "negl    $op2\t# overflow check int" %}
12565   ins_encode %{
12566     __ negl($op2$$Register);
12567   %}
12568   ins_pipe(ialu_reg_reg);
12569 %}
12570 
12571 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12572 %{
12573   match(Set cr (OverflowSubL zero op2));
12574   effect(DEF cr, USE_KILL op2);
12575 
12576   format %{ "negq    $op2\t# overflow check long" %}
12577   ins_encode %{
12578     __ negq($op2$$Register);
12579   %}
12580   ins_pipe(ialu_reg_reg);
12581 %}
12582 
12583 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12584 %{
12585   match(Set cr (OverflowMulI op1 op2));
12586   effect(DEF cr, USE_KILL op1, USE op2);
12587 
12588   format %{ "imull    $op1, $op2\t# overflow check int" %}
12589   ins_encode %{
12590     __ imull($op1$$Register, $op2$$Register);
12591   %}
12592   ins_pipe(ialu_reg_reg_alu0);
12593 %}
12594 
12595 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12596 %{
12597   match(Set cr (OverflowMulI op1 op2));
12598   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12599 
12600   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12601   ins_encode %{
12602     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12603   %}
12604   ins_pipe(ialu_reg_reg_alu0);
12605 %}
12606 
12607 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12608 %{
12609   match(Set cr (OverflowMulL op1 op2));
12610   effect(DEF cr, USE_KILL op1, USE op2);
12611 
12612   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12613   ins_encode %{
12614     __ imulq($op1$$Register, $op2$$Register);
12615   %}
12616   ins_pipe(ialu_reg_reg_alu0);
12617 %}
12618 
12619 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12620 %{
12621   match(Set cr (OverflowMulL op1 op2));
12622   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12623 
12624   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12625   ins_encode %{
12626     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12627   %}
12628   ins_pipe(ialu_reg_reg_alu0);
12629 %}
12630 
12631 
12632 //----------Control Flow Instructions------------------------------------------
12633 // Signed compare Instructions
12634 
12635 // XXX more variants!!
12636 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12637 %{
12638   match(Set cr (CmpI op1 op2));
12639   effect(DEF cr, USE op1, USE op2);
12640 
12641   format %{ "cmpl    $op1, $op2" %}
12642   ins_encode %{
12643     __ cmpl($op1$$Register, $op2$$Register);
12644   %}
12645   ins_pipe(ialu_cr_reg_reg);
12646 %}
12647 
12648 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12649 %{
12650   match(Set cr (CmpI op1 op2));
12651 
12652   format %{ "cmpl    $op1, $op2" %}
12653   ins_encode %{
12654     __ cmpl($op1$$Register, $op2$$constant);
12655   %}
12656   ins_pipe(ialu_cr_reg_imm);
12657 %}
12658 
12659 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12660 %{
12661   match(Set cr (CmpI op1 (LoadI op2)));
12662 
12663   ins_cost(500); // XXX
12664   format %{ "cmpl    $op1, $op2" %}
12665   ins_encode %{
12666     __ cmpl($op1$$Register, $op2$$Address);
12667   %}
12668   ins_pipe(ialu_cr_reg_mem);
12669 %}
12670 
12671 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12672 %{
12673   match(Set cr (CmpI src zero));
12674 
12675   format %{ "testl   $src, $src" %}
12676   ins_encode %{
12677     __ testl($src$$Register, $src$$Register);
12678   %}
12679   ins_pipe(ialu_cr_reg_imm);
12680 %}
12681 
12682 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12683 %{
12684   match(Set cr (CmpI (AndI src con) zero));
12685 
12686   format %{ "testl   $src, $con" %}
12687   ins_encode %{
12688     __ testl($src$$Register, $con$$constant);
12689   %}
12690   ins_pipe(ialu_cr_reg_imm);
12691 %}
12692 
12693 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12694 %{
12695   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12696 
12697   format %{ "testl   $src, $mem" %}
12698   ins_encode %{
12699     __ testl($src$$Register, $mem$$Address);
12700   %}
12701   ins_pipe(ialu_cr_reg_mem);
12702 %}
12703 
12704 // Unsigned compare Instructions; really, same as signed except they
12705 // produce an rFlagsRegU instead of rFlagsReg.
12706 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12707 %{
12708   match(Set cr (CmpU op1 op2));
12709 
12710   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12711   ins_encode %{
12712     __ cmpl($op1$$Register, $op2$$Register);
12713   %}
12714   ins_pipe(ialu_cr_reg_reg);
12715 %}
12716 
12717 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12718 %{
12719   match(Set cr (CmpU op1 op2));
12720 
12721   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12722   ins_encode %{
12723     __ cmpl($op1$$Register, $op2$$constant);
12724   %}
12725   ins_pipe(ialu_cr_reg_imm);
12726 %}
12727 
12728 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12729 %{
12730   match(Set cr (CmpU op1 (LoadI op2)));
12731 
12732   ins_cost(500); // XXX
12733   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12734   ins_encode %{
12735     __ cmpl($op1$$Register, $op2$$Address);
12736   %}
12737   ins_pipe(ialu_cr_reg_mem);
12738 %}
12739 
12740 // // // Cisc-spilled version of cmpU_rReg
12741 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12742 // //%{
12743 // //  match(Set cr (CmpU (LoadI op1) op2));
12744 // //
12745 // //  format %{ "CMPu   $op1,$op2" %}
12746 // //  ins_cost(500);
12747 // //  opcode(0x39);  /* Opcode 39 /r */
12748 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12749 // //%}
12750 
12751 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12752 %{
12753   match(Set cr (CmpU src zero));
12754 
12755   format %{ "testl   $src, $src\t# unsigned" %}
12756   ins_encode %{
12757     __ testl($src$$Register, $src$$Register);
12758   %}
12759   ins_pipe(ialu_cr_reg_imm);
12760 %}
12761 
12762 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12763 %{
12764   match(Set cr (CmpP op1 op2));
12765 
12766   format %{ "cmpq    $op1, $op2\t# ptr" %}
12767   ins_encode %{
12768     __ cmpq($op1$$Register, $op2$$Register);
12769   %}
12770   ins_pipe(ialu_cr_reg_reg);
12771 %}
12772 
12773 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12774 %{
12775   match(Set cr (CmpP op1 (LoadP op2)));
12776   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12777 
12778   ins_cost(500); // XXX
12779   format %{ "cmpq    $op1, $op2\t# ptr" %}
12780   ins_encode %{
12781     __ cmpq($op1$$Register, $op2$$Address);
12782   %}
12783   ins_pipe(ialu_cr_reg_mem);
12784 %}
12785 
12786 // // // Cisc-spilled version of cmpP_rReg
12787 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12788 // //%{
12789 // //  match(Set cr (CmpP (LoadP op1) op2));
12790 // //
12791 // //  format %{ "CMPu   $op1,$op2" %}
12792 // //  ins_cost(500);
12793 // //  opcode(0x39);  /* Opcode 39 /r */
12794 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12795 // //%}
12796 
12797 // XXX this is generalized by compP_rReg_mem???
12798 // Compare raw pointer (used in out-of-heap check).
12799 // Only works because non-oop pointers must be raw pointers
12800 // and raw pointers have no anti-dependencies.
12801 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12802 %{
12803   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12804             n->in(2)->as_Load()->barrier_data() == 0);
12805   match(Set cr (CmpP op1 (LoadP op2)));
12806 
12807   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12808   ins_encode %{
12809     __ cmpq($op1$$Register, $op2$$Address);
12810   %}
12811   ins_pipe(ialu_cr_reg_mem);
12812 %}
12813 
12814 // This will generate a signed flags result. This should be OK since
12815 // any compare to a zero should be eq/neq.
12816 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12817 %{
12818   match(Set cr (CmpP src zero));
12819 
12820   format %{ "testq   $src, $src\t# ptr" %}
12821   ins_encode %{
12822     __ testq($src$$Register, $src$$Register);
12823   %}
12824   ins_pipe(ialu_cr_reg_imm);
12825 %}
12826 
12827 // This will generate a signed flags result. This should be OK since
12828 // any compare to a zero should be eq/neq.
12829 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12830 %{
12831   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12832             n->in(1)->as_Load()->barrier_data() == 0);
12833   match(Set cr (CmpP (LoadP op) zero));
12834 
12835   ins_cost(500); // XXX
12836   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12837   ins_encode %{
12838     __ testq($op$$Address, 0xFFFFFFFF);
12839   %}
12840   ins_pipe(ialu_cr_reg_imm);
12841 %}
12842 
12843 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12844 %{
12845   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12846             n->in(1)->as_Load()->barrier_data() == 0);
12847   match(Set cr (CmpP (LoadP mem) zero));
12848 
12849   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12850   ins_encode %{
12851     __ cmpq(r12, $mem$$Address);
12852   %}
12853   ins_pipe(ialu_cr_reg_mem);
12854 %}
12855 
12856 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12857 %{
12858   match(Set cr (CmpN op1 op2));
12859 
12860   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12861   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12862   ins_pipe(ialu_cr_reg_reg);
12863 %}
12864 
12865 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12866 %{
12867   match(Set cr (CmpN src (LoadN mem)));
12868 
12869   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12870   ins_encode %{
12871     __ cmpl($src$$Register, $mem$$Address);
12872   %}
12873   ins_pipe(ialu_cr_reg_mem);
12874 %}
12875 
12876 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12877   match(Set cr (CmpN op1 op2));
12878 
12879   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12880   ins_encode %{
12881     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12882   %}
12883   ins_pipe(ialu_cr_reg_imm);
12884 %}
12885 
12886 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12887 %{
12888   match(Set cr (CmpN src (LoadN mem)));
12889 
12890   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12891   ins_encode %{
12892     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12893   %}
12894   ins_pipe(ialu_cr_reg_mem);
12895 %}
12896 
12897 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12898   match(Set cr (CmpN op1 op2));
12899 
12900   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12901   ins_encode %{
12902     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12903   %}
12904   ins_pipe(ialu_cr_reg_imm);
12905 %}
12906 



12907 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12908 %{

12909   match(Set cr (CmpN src (LoadNKlass mem)));
12910 
12911   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12912   ins_encode %{
12913     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12914   %}
12915   ins_pipe(ialu_cr_reg_mem);
12916 %}
12917 
12918 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12919   match(Set cr (CmpN src zero));
12920 
12921   format %{ "testl   $src, $src\t# compressed ptr" %}
12922   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12923   ins_pipe(ialu_cr_reg_imm);
12924 %}
12925 
12926 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12927 %{
12928   predicate(CompressedOops::base() != NULL);
12929   match(Set cr (CmpN (LoadN mem) zero));
12930 
12931   ins_cost(500); // XXX
12932   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12933   ins_encode %{
12934     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12935   %}
12936   ins_pipe(ialu_cr_reg_mem);
12937 %}
12938 
12939 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12940 %{
12941   predicate(CompressedOops::base() == NULL);
12942   match(Set cr (CmpN (LoadN mem) zero));
12943 
12944   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12945   ins_encode %{
12946     __ cmpl(r12, $mem$$Address);
12947   %}
12948   ins_pipe(ialu_cr_reg_mem);
12949 %}
12950 
12951 // Yanked all unsigned pointer compare operations.
12952 // Pointer compares are done with CmpP which is already unsigned.
12953 
12954 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12955 %{
12956   match(Set cr (CmpL op1 op2));
12957 
12958   format %{ "cmpq    $op1, $op2" %}
12959   ins_encode %{
12960     __ cmpq($op1$$Register, $op2$$Register);
12961   %}
12962   ins_pipe(ialu_cr_reg_reg);
12963 %}
12964 
12965 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12966 %{
12967   match(Set cr (CmpL op1 op2));
12968 
12969   format %{ "cmpq    $op1, $op2" %}
12970   ins_encode %{
12971     __ cmpq($op1$$Register, $op2$$constant);
12972   %}
12973   ins_pipe(ialu_cr_reg_imm);
12974 %}
12975 
12976 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12977 %{
12978   match(Set cr (CmpL op1 (LoadL op2)));
12979 
12980   format %{ "cmpq    $op1, $op2" %}
12981   ins_encode %{
12982     __ cmpq($op1$$Register, $op2$$Address);
12983   %}
12984   ins_pipe(ialu_cr_reg_mem);
12985 %}
12986 
12987 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12988 %{
12989   match(Set cr (CmpL src zero));
12990 
12991   format %{ "testq   $src, $src" %}
12992   ins_encode %{
12993     __ testq($src$$Register, $src$$Register);
12994   %}
12995   ins_pipe(ialu_cr_reg_imm);
12996 %}
12997 
12998 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12999 %{
13000   match(Set cr (CmpL (AndL src con) zero));
13001 
13002   format %{ "testq   $src, $con\t# long" %}
13003   ins_encode %{
13004     __ testq($src$$Register, $con$$constant);
13005   %}
13006   ins_pipe(ialu_cr_reg_imm);
13007 %}
13008 
13009 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13010 %{
13011   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13012 
13013   format %{ "testq   $src, $mem" %}
13014   ins_encode %{
13015     __ testq($src$$Register, $mem$$Address);
13016   %}
13017   ins_pipe(ialu_cr_reg_mem);
13018 %}
13019 
13020 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13021 %{
13022   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13023 
13024   format %{ "testq   $src, $mem" %}
13025   ins_encode %{
13026     __ testq($src$$Register, $mem$$Address);
13027   %}
13028   ins_pipe(ialu_cr_reg_mem);
13029 %}
13030 
13031 // Manifest a CmpU result in an integer register.  Very painful.
13032 // This is the test to avoid.
13033 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13034 %{
13035   match(Set dst (CmpU3 src1 src2));
13036   effect(KILL flags);
13037 
13038   ins_cost(275); // XXX
13039   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13040             "movl    $dst, -1\n\t"
13041             "jb,u    done\n\t"
13042             "setne   $dst\n\t"
13043             "movzbl  $dst, $dst\n\t"
13044     "done:" %}
13045   ins_encode %{
13046     Label done;
13047     __ cmpl($src1$$Register, $src2$$Register);
13048     __ movl($dst$$Register, -1);
13049     __ jccb(Assembler::below, done);
13050     __ setne($dst$$Register);
13051     __ movzbl($dst$$Register, $dst$$Register);
13052     __ bind(done);
13053   %}
13054   ins_pipe(pipe_slow);
13055 %}
13056 
13057 // Manifest a CmpL result in an integer register.  Very painful.
13058 // This is the test to avoid.
13059 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13060 %{
13061   match(Set dst (CmpL3 src1 src2));
13062   effect(KILL flags);
13063 
13064   ins_cost(275); // XXX
13065   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13066             "movl    $dst, -1\n\t"
13067             "jl,s    done\n\t"
13068             "setne   $dst\n\t"
13069             "movzbl  $dst, $dst\n\t"
13070     "done:" %}
13071   ins_encode %{
13072     Label done;
13073     __ cmpq($src1$$Register, $src2$$Register);
13074     __ movl($dst$$Register, -1);
13075     __ jccb(Assembler::less, done);
13076     __ setne($dst$$Register);
13077     __ movzbl($dst$$Register, $dst$$Register);
13078     __ bind(done);
13079   %}
13080   ins_pipe(pipe_slow);
13081 %}
13082 
13083 // Manifest a CmpUL result in an integer register.  Very painful.
13084 // This is the test to avoid.
13085 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13086 %{
13087   match(Set dst (CmpUL3 src1 src2));
13088   effect(KILL flags);
13089 
13090   ins_cost(275); // XXX
13091   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13092             "movl    $dst, -1\n\t"
13093             "jb,u    done\n\t"
13094             "setne   $dst\n\t"
13095             "movzbl  $dst, $dst\n\t"
13096     "done:" %}
13097   ins_encode %{
13098     Label done;
13099     __ cmpq($src1$$Register, $src2$$Register);
13100     __ movl($dst$$Register, -1);
13101     __ jccb(Assembler::below, done);
13102     __ setne($dst$$Register);
13103     __ movzbl($dst$$Register, $dst$$Register);
13104     __ bind(done);
13105   %}
13106   ins_pipe(pipe_slow);
13107 %}
13108 
13109 // Unsigned long compare Instructions; really, same as signed long except they
13110 // produce an rFlagsRegU instead of rFlagsReg.
13111 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13112 %{
13113   match(Set cr (CmpUL op1 op2));
13114 
13115   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13116   ins_encode %{
13117     __ cmpq($op1$$Register, $op2$$Register);
13118   %}
13119   ins_pipe(ialu_cr_reg_reg);
13120 %}
13121 
13122 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13123 %{
13124   match(Set cr (CmpUL op1 op2));
13125 
13126   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13127   ins_encode %{
13128     __ cmpq($op1$$Register, $op2$$constant);
13129   %}
13130   ins_pipe(ialu_cr_reg_imm);
13131 %}
13132 
13133 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13134 %{
13135   match(Set cr (CmpUL op1 (LoadL op2)));
13136 
13137   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13138   ins_encode %{
13139     __ cmpq($op1$$Register, $op2$$Address);
13140   %}
13141   ins_pipe(ialu_cr_reg_mem);
13142 %}
13143 
13144 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13145 %{
13146   match(Set cr (CmpUL src zero));
13147 
13148   format %{ "testq   $src, $src\t# unsigned" %}
13149   ins_encode %{
13150     __ testq($src$$Register, $src$$Register);
13151   %}
13152   ins_pipe(ialu_cr_reg_imm);
13153 %}
13154 
13155 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13156 %{
13157   match(Set cr (CmpI (LoadB mem) imm));
13158 
13159   ins_cost(125);
13160   format %{ "cmpb    $mem, $imm" %}
13161   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13162   ins_pipe(ialu_cr_reg_mem);
13163 %}
13164 
13165 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13166 %{
13167   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13168 
13169   ins_cost(125);
13170   format %{ "testb   $mem, $imm\t# ubyte" %}
13171   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13172   ins_pipe(ialu_cr_reg_mem);
13173 %}
13174 
13175 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13176 %{
13177   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13178 
13179   ins_cost(125);
13180   format %{ "testb   $mem, $imm\t# byte" %}
13181   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13182   ins_pipe(ialu_cr_reg_mem);
13183 %}
13184 
13185 //----------Max and Min--------------------------------------------------------
13186 // Min Instructions
13187 
13188 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13189 %{
13190   effect(USE_DEF dst, USE src, USE cr);
13191 
13192   format %{ "cmovlgt $dst, $src\t# min" %}
13193   ins_encode %{
13194     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13195   %}
13196   ins_pipe(pipe_cmov_reg);
13197 %}
13198 
13199 
13200 instruct minI_rReg(rRegI dst, rRegI src)
13201 %{
13202   match(Set dst (MinI dst src));
13203 
13204   ins_cost(200);
13205   expand %{
13206     rFlagsReg cr;
13207     compI_rReg(cr, dst, src);
13208     cmovI_reg_g(dst, src, cr);
13209   %}
13210 %}
13211 
13212 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13213 %{
13214   effect(USE_DEF dst, USE src, USE cr);
13215 
13216   format %{ "cmovllt $dst, $src\t# max" %}
13217   ins_encode %{
13218     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13219   %}
13220   ins_pipe(pipe_cmov_reg);
13221 %}
13222 
13223 
13224 instruct maxI_rReg(rRegI dst, rRegI src)
13225 %{
13226   match(Set dst (MaxI dst src));
13227 
13228   ins_cost(200);
13229   expand %{
13230     rFlagsReg cr;
13231     compI_rReg(cr, dst, src);
13232     cmovI_reg_l(dst, src, cr);
13233   %}
13234 %}
13235 
13236 // ============================================================================
13237 // Branch Instructions
13238 
13239 // Jump Direct - Label defines a relative address from JMP+1
13240 instruct jmpDir(label labl)
13241 %{
13242   match(Goto);
13243   effect(USE labl);
13244 
13245   ins_cost(300);
13246   format %{ "jmp     $labl" %}
13247   size(5);
13248   ins_encode %{
13249     Label* L = $labl$$label;
13250     __ jmp(*L, false); // Always long jump
13251   %}
13252   ins_pipe(pipe_jmp);
13253 %}
13254 
13255 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13256 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13257 %{
13258   match(If cop cr);
13259   effect(USE labl);
13260 
13261   ins_cost(300);
13262   format %{ "j$cop     $labl" %}
13263   size(6);
13264   ins_encode %{
13265     Label* L = $labl$$label;
13266     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13267   %}
13268   ins_pipe(pipe_jcc);
13269 %}
13270 
13271 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13272 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13273 %{
13274   match(CountedLoopEnd cop cr);
13275   effect(USE labl);
13276 
13277   ins_cost(300);
13278   format %{ "j$cop     $labl\t# loop end" %}
13279   size(6);
13280   ins_encode %{
13281     Label* L = $labl$$label;
13282     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13283   %}
13284   ins_pipe(pipe_jcc);
13285 %}
13286 
13287 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13288 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13289   match(CountedLoopEnd cop cmp);
13290   effect(USE labl);
13291 
13292   ins_cost(300);
13293   format %{ "j$cop,u   $labl\t# loop end" %}
13294   size(6);
13295   ins_encode %{
13296     Label* L = $labl$$label;
13297     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13298   %}
13299   ins_pipe(pipe_jcc);
13300 %}
13301 
13302 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13303   match(CountedLoopEnd cop cmp);
13304   effect(USE labl);
13305 
13306   ins_cost(200);
13307   format %{ "j$cop,u   $labl\t# loop end" %}
13308   size(6);
13309   ins_encode %{
13310     Label* L = $labl$$label;
13311     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13312   %}
13313   ins_pipe(pipe_jcc);
13314 %}
13315 
13316 // Jump Direct Conditional - using unsigned comparison
13317 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13318   match(If cop cmp);
13319   effect(USE labl);
13320 
13321   ins_cost(300);
13322   format %{ "j$cop,u   $labl" %}
13323   size(6);
13324   ins_encode %{
13325     Label* L = $labl$$label;
13326     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13327   %}
13328   ins_pipe(pipe_jcc);
13329 %}
13330 
13331 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13332   match(If cop cmp);
13333   effect(USE labl);
13334 
13335   ins_cost(200);
13336   format %{ "j$cop,u   $labl" %}
13337   size(6);
13338   ins_encode %{
13339     Label* L = $labl$$label;
13340     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13341   %}
13342   ins_pipe(pipe_jcc);
13343 %}
13344 
13345 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13346   match(If cop cmp);
13347   effect(USE labl);
13348 
13349   ins_cost(200);
13350   format %{ $$template
13351     if ($cop$$cmpcode == Assembler::notEqual) {
13352       $$emit$$"jp,u    $labl\n\t"
13353       $$emit$$"j$cop,u   $labl"
13354     } else {
13355       $$emit$$"jp,u    done\n\t"
13356       $$emit$$"j$cop,u   $labl\n\t"
13357       $$emit$$"done:"
13358     }
13359   %}
13360   ins_encode %{
13361     Label* l = $labl$$label;
13362     if ($cop$$cmpcode == Assembler::notEqual) {
13363       __ jcc(Assembler::parity, *l, false);
13364       __ jcc(Assembler::notEqual, *l, false);
13365     } else if ($cop$$cmpcode == Assembler::equal) {
13366       Label done;
13367       __ jccb(Assembler::parity, done);
13368       __ jcc(Assembler::equal, *l, false);
13369       __ bind(done);
13370     } else {
13371        ShouldNotReachHere();
13372     }
13373   %}
13374   ins_pipe(pipe_jcc);
13375 %}
13376 
13377 // ============================================================================
13378 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13379 // superklass array for an instance of the superklass.  Set a hidden
13380 // internal cache on a hit (cache is checked with exposed code in
13381 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13382 // encoding ALSO sets flags.
13383 
13384 instruct partialSubtypeCheck(rdi_RegP result,
13385                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13386                              rFlagsReg cr)
13387 %{
13388   match(Set result (PartialSubtypeCheck sub super));
13389   effect(KILL rcx, KILL cr);
13390 
13391   ins_cost(1100);  // slightly larger than the next version
13392   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13393             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13394             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13395             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13396             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13397             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13398             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13399     "miss:\t" %}
13400 
13401   opcode(0x1); // Force a XOR of RDI
13402   ins_encode(enc_PartialSubtypeCheck());
13403   ins_pipe(pipe_slow);
13404 %}
13405 
13406 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13407                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13408                                      immP0 zero,
13409                                      rdi_RegP result)
13410 %{
13411   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13412   effect(KILL rcx, KILL result);
13413 
13414   ins_cost(1000);
13415   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13416             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13417             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13418             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13419             "jne,s   miss\t\t# Missed: flags nz\n\t"
13420             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13421     "miss:\t" %}
13422 
13423   opcode(0x0); // No need to XOR RDI
13424   ins_encode(enc_PartialSubtypeCheck());
13425   ins_pipe(pipe_slow);
13426 %}
13427 
13428 // ============================================================================
13429 // Branch Instructions -- short offset versions
13430 //
13431 // These instructions are used to replace jumps of a long offset (the default
13432 // match) with jumps of a shorter offset.  These instructions are all tagged
13433 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13434 // match rules in general matching.  Instead, the ADLC generates a conversion
13435 // method in the MachNode which can be used to do in-place replacement of the
13436 // long variant with the shorter variant.  The compiler will determine if a
13437 // branch can be taken by the is_short_branch_offset() predicate in the machine
13438 // specific code section of the file.
13439 
13440 // Jump Direct - Label defines a relative address from JMP+1
13441 instruct jmpDir_short(label labl) %{
13442   match(Goto);
13443   effect(USE labl);
13444 
13445   ins_cost(300);
13446   format %{ "jmp,s   $labl" %}
13447   size(2);
13448   ins_encode %{
13449     Label* L = $labl$$label;
13450     __ jmpb(*L);
13451   %}
13452   ins_pipe(pipe_jmp);
13453   ins_short_branch(1);
13454 %}
13455 
13456 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13457 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13458   match(If cop cr);
13459   effect(USE labl);
13460 
13461   ins_cost(300);
13462   format %{ "j$cop,s   $labl" %}
13463   size(2);
13464   ins_encode %{
13465     Label* L = $labl$$label;
13466     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13467   %}
13468   ins_pipe(pipe_jcc);
13469   ins_short_branch(1);
13470 %}
13471 
13472 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13473 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13474   match(CountedLoopEnd cop cr);
13475   effect(USE labl);
13476 
13477   ins_cost(300);
13478   format %{ "j$cop,s   $labl\t# loop end" %}
13479   size(2);
13480   ins_encode %{
13481     Label* L = $labl$$label;
13482     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13483   %}
13484   ins_pipe(pipe_jcc);
13485   ins_short_branch(1);
13486 %}
13487 
13488 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13489 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13490   match(CountedLoopEnd cop cmp);
13491   effect(USE labl);
13492 
13493   ins_cost(300);
13494   format %{ "j$cop,us  $labl\t# loop end" %}
13495   size(2);
13496   ins_encode %{
13497     Label* L = $labl$$label;
13498     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13499   %}
13500   ins_pipe(pipe_jcc);
13501   ins_short_branch(1);
13502 %}
13503 
13504 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13505   match(CountedLoopEnd cop cmp);
13506   effect(USE labl);
13507 
13508   ins_cost(300);
13509   format %{ "j$cop,us  $labl\t# loop end" %}
13510   size(2);
13511   ins_encode %{
13512     Label* L = $labl$$label;
13513     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13514   %}
13515   ins_pipe(pipe_jcc);
13516   ins_short_branch(1);
13517 %}
13518 
13519 // Jump Direct Conditional - using unsigned comparison
13520 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13521   match(If cop cmp);
13522   effect(USE labl);
13523 
13524   ins_cost(300);
13525   format %{ "j$cop,us  $labl" %}
13526   size(2);
13527   ins_encode %{
13528     Label* L = $labl$$label;
13529     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13530   %}
13531   ins_pipe(pipe_jcc);
13532   ins_short_branch(1);
13533 %}
13534 
13535 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13536   match(If cop cmp);
13537   effect(USE labl);
13538 
13539   ins_cost(300);
13540   format %{ "j$cop,us  $labl" %}
13541   size(2);
13542   ins_encode %{
13543     Label* L = $labl$$label;
13544     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13545   %}
13546   ins_pipe(pipe_jcc);
13547   ins_short_branch(1);
13548 %}
13549 
13550 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13551   match(If cop cmp);
13552   effect(USE labl);
13553 
13554   ins_cost(300);
13555   format %{ $$template
13556     if ($cop$$cmpcode == Assembler::notEqual) {
13557       $$emit$$"jp,u,s  $labl\n\t"
13558       $$emit$$"j$cop,u,s  $labl"
13559     } else {
13560       $$emit$$"jp,u,s  done\n\t"
13561       $$emit$$"j$cop,u,s  $labl\n\t"
13562       $$emit$$"done:"
13563     }
13564   %}
13565   size(4);
13566   ins_encode %{
13567     Label* l = $labl$$label;
13568     if ($cop$$cmpcode == Assembler::notEqual) {
13569       __ jccb(Assembler::parity, *l);
13570       __ jccb(Assembler::notEqual, *l);
13571     } else if ($cop$$cmpcode == Assembler::equal) {
13572       Label done;
13573       __ jccb(Assembler::parity, done);
13574       __ jccb(Assembler::equal, *l);
13575       __ bind(done);
13576     } else {
13577        ShouldNotReachHere();
13578     }
13579   %}
13580   ins_pipe(pipe_jcc);
13581   ins_short_branch(1);
13582 %}
13583 
13584 // ============================================================================
13585 // inlined locking and unlocking
13586 
13587 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13588   predicate(Compile::current()->use_rtm());
13589   match(Set cr (FastLock object box));
13590   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13591   ins_cost(300);
13592   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13593   ins_encode %{
13594     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13595                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13596                  _rtm_counters, _stack_rtm_counters,
13597                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13598                  true, ra_->C->profile_rtm());
13599   %}
13600   ins_pipe(pipe_slow);
13601 %}
13602 
13603 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13604   predicate(!Compile::current()->use_rtm());
13605   match(Set cr (FastLock object box));
13606   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13607   ins_cost(300);
13608   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13609   ins_encode %{
13610     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13611                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13612   %}
13613   ins_pipe(pipe_slow);
13614 %}
13615 
13616 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13617   match(Set cr (FastUnlock object box));
13618   effect(TEMP tmp, USE_KILL box);
13619   ins_cost(300);
13620   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13621   ins_encode %{
13622     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13623   %}
13624   ins_pipe(pipe_slow);
13625 %}
13626 
13627 
13628 // ============================================================================
13629 // Safepoint Instructions
13630 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13631 %{
13632   match(SafePoint poll);
13633   effect(KILL cr, USE poll);
13634 
13635   format %{ "testl   rax, [$poll]\t"
13636             "# Safepoint: poll for GC" %}
13637   ins_cost(125);
13638   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13639   ins_encode %{
13640     __ relocate(relocInfo::poll_type);
13641     address pre_pc = __ pc();
13642     __ testl(rax, Address($poll$$Register, 0));
13643     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13644   %}
13645   ins_pipe(ialu_reg_mem);
13646 %}
13647 
13648 instruct mask_all_evexL(kReg dst, rRegL src) %{
13649   match(Set dst (MaskAll src));
13650   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13651   ins_encode %{
13652     int mask_len = Matcher::vector_length(this);
13653     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13654   %}
13655   ins_pipe( pipe_slow );
13656 %}
13657 
13658 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13659   predicate(Matcher::vector_length(n) > 32);
13660   match(Set dst (MaskAll src));
13661   effect(TEMP tmp);
13662   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13663   ins_encode %{
13664     int mask_len = Matcher::vector_length(this);
13665     __ movslq($tmp$$Register, $src$$Register);
13666     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13667   %}
13668   ins_pipe( pipe_slow );
13669 %}
13670 
13671 // ============================================================================
13672 // Procedure Call/Return Instructions
13673 // Call Java Static Instruction
13674 // Note: If this code changes, the corresponding ret_addr_offset() and
13675 //       compute_padding() functions will have to be adjusted.
13676 instruct CallStaticJavaDirect(method meth) %{
13677   match(CallStaticJava);
13678   effect(USE meth);
13679 
13680   ins_cost(300);
13681   format %{ "call,static " %}
13682   opcode(0xE8); /* E8 cd */
13683   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13684   ins_pipe(pipe_slow);
13685   ins_alignment(4);
13686 %}
13687 
13688 // Call Java Dynamic Instruction
13689 // Note: If this code changes, the corresponding ret_addr_offset() and
13690 //       compute_padding() functions will have to be adjusted.
13691 instruct CallDynamicJavaDirect(method meth)
13692 %{
13693   match(CallDynamicJava);
13694   effect(USE meth);
13695 
13696   ins_cost(300);
13697   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13698             "call,dynamic " %}
13699   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13700   ins_pipe(pipe_slow);
13701   ins_alignment(4);
13702 %}
13703 
13704 // Call Runtime Instruction
13705 instruct CallRuntimeDirect(method meth)
13706 %{
13707   match(CallRuntime);
13708   effect(USE meth);
13709 
13710   ins_cost(300);
13711   format %{ "call,runtime " %}
13712   ins_encode(clear_avx, Java_To_Runtime(meth));
13713   ins_pipe(pipe_slow);
13714 %}
13715 
13716 // Call runtime without safepoint
13717 instruct CallLeafDirect(method meth)
13718 %{
13719   match(CallLeaf);
13720   effect(USE meth);
13721 
13722   ins_cost(300);
13723   format %{ "call_leaf,runtime " %}
13724   ins_encode(clear_avx, Java_To_Runtime(meth));
13725   ins_pipe(pipe_slow);
13726 %}
13727 
13728 // Call runtime without safepoint and with vector arguments
13729 instruct CallLeafDirectVector(method meth)
13730 %{
13731   match(CallLeafVector);
13732   effect(USE meth);
13733 
13734   ins_cost(300);
13735   format %{ "call_leaf,vector " %}
13736   ins_encode(Java_To_Runtime(meth));
13737   ins_pipe(pipe_slow);
13738 %}
13739 
13740 // Call runtime without safepoint
13741 instruct CallLeafNoFPDirect(method meth)
13742 %{
13743   match(CallLeafNoFP);
13744   effect(USE meth);
13745 
13746   ins_cost(300);
13747   format %{ "call_leaf_nofp,runtime " %}
13748   ins_encode(clear_avx, Java_To_Runtime(meth));
13749   ins_pipe(pipe_slow);
13750 %}
13751 
13752 // Return Instruction
13753 // Remove the return address & jump to it.
13754 // Notice: We always emit a nop after a ret to make sure there is room
13755 // for safepoint patching
13756 instruct Ret()
13757 %{
13758   match(Return);
13759 
13760   format %{ "ret" %}
13761   ins_encode %{
13762     __ ret(0);
13763   %}
13764   ins_pipe(pipe_jmp);
13765 %}
13766 
13767 // Tail Call; Jump from runtime stub to Java code.
13768 // Also known as an 'interprocedural jump'.
13769 // Target of jump will eventually return to caller.
13770 // TailJump below removes the return address.
13771 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13772 %{
13773   match(TailCall jump_target method_ptr);
13774 
13775   ins_cost(300);
13776   format %{ "jmp     $jump_target\t# rbx holds method" %}
13777   ins_encode %{
13778     __ jmp($jump_target$$Register);
13779   %}
13780   ins_pipe(pipe_jmp);
13781 %}
13782 
13783 // Tail Jump; remove the return address; jump to target.
13784 // TailCall above leaves the return address around.
13785 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13786 %{
13787   match(TailJump jump_target ex_oop);
13788 
13789   ins_cost(300);
13790   format %{ "popq    rdx\t# pop return address\n\t"
13791             "jmp     $jump_target" %}
13792   ins_encode %{
13793     __ popq(as_Register(RDX_enc));
13794     __ jmp($jump_target$$Register);
13795   %}
13796   ins_pipe(pipe_jmp);
13797 %}
13798 
13799 // Create exception oop: created by stack-crawling runtime code.
13800 // Created exception is now available to this handler, and is setup
13801 // just prior to jumping to this handler.  No code emitted.
13802 instruct CreateException(rax_RegP ex_oop)
13803 %{
13804   match(Set ex_oop (CreateEx));
13805 
13806   size(0);
13807   // use the following format syntax
13808   format %{ "# exception oop is in rax; no code emitted" %}
13809   ins_encode();
13810   ins_pipe(empty);
13811 %}
13812 
13813 // Rethrow exception:
13814 // The exception oop will come in the first argument position.
13815 // Then JUMP (not call) to the rethrow stub code.
13816 instruct RethrowException()
13817 %{
13818   match(Rethrow);
13819 
13820   // use the following format syntax
13821   format %{ "jmp     rethrow_stub" %}
13822   ins_encode(enc_rethrow);
13823   ins_pipe(pipe_jmp);
13824 %}
13825 
13826 // ============================================================================
13827 // This name is KNOWN by the ADLC and cannot be changed.
13828 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13829 // for this guy.
13830 instruct tlsLoadP(r15_RegP dst) %{
13831   match(Set dst (ThreadLocal));
13832   effect(DEF dst);
13833 
13834   size(0);
13835   format %{ "# TLS is in R15" %}
13836   ins_encode( /*empty encoding*/ );
13837   ins_pipe(ialu_reg_reg);
13838 %}
13839 
13840 
13841 //----------PEEPHOLE RULES-----------------------------------------------------
13842 // These must follow all instruction definitions as they use the names
13843 // defined in the instructions definitions.
13844 //
13845 // peepmatch ( root_instr_name [preceding_instruction]* );
13846 //
13847 // peepconstraint %{
13848 // (instruction_number.operand_name relational_op instruction_number.operand_name
13849 //  [, ...] );
13850 // // instruction numbers are zero-based using left to right order in peepmatch
13851 //
13852 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13853 // // provide an instruction_number.operand_name for each operand that appears
13854 // // in the replacement instruction's match rule
13855 //
13856 // ---------VM FLAGS---------------------------------------------------------
13857 //
13858 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13859 //
13860 // Each peephole rule is given an identifying number starting with zero and
13861 // increasing by one in the order seen by the parser.  An individual peephole
13862 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13863 // on the command-line.
13864 //
13865 // ---------CURRENT LIMITATIONS----------------------------------------------
13866 //
13867 // Only match adjacent instructions in same basic block
13868 // Only equality constraints
13869 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13870 // Only one replacement instruction
13871 //
13872 // ---------EXAMPLE----------------------------------------------------------
13873 //
13874 // // pertinent parts of existing instructions in architecture description
13875 // instruct movI(rRegI dst, rRegI src)
13876 // %{
13877 //   match(Set dst (CopyI src));
13878 // %}
13879 //
13880 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13881 // %{
13882 //   match(Set dst (AddI dst src));
13883 //   effect(KILL cr);
13884 // %}
13885 //
13886 // // Change (inc mov) to lea
13887 // peephole %{
13888 //   // increment preceded by register-register move
13889 //   peepmatch ( incI_rReg movI );
13890 //   // require that the destination register of the increment
13891 //   // match the destination register of the move
13892 //   peepconstraint ( 0.dst == 1.dst );
13893 //   // construct a replacement instruction that sets
13894 //   // the destination to ( move's source register + one )
13895 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13896 // %}
13897 //
13898 
13899 // Implementation no longer uses movX instructions since
13900 // machine-independent system no longer uses CopyX nodes.
13901 //
13902 // peephole
13903 // %{
13904 //   peepmatch (incI_rReg movI);
13905 //   peepconstraint (0.dst == 1.dst);
13906 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13907 // %}
13908 
13909 // peephole
13910 // %{
13911 //   peepmatch (decI_rReg movI);
13912 //   peepconstraint (0.dst == 1.dst);
13913 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13914 // %}
13915 
13916 // peephole
13917 // %{
13918 //   peepmatch (addI_rReg_imm movI);
13919 //   peepconstraint (0.dst == 1.dst);
13920 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13921 // %}
13922 
13923 // peephole
13924 // %{
13925 //   peepmatch (incL_rReg movL);
13926 //   peepconstraint (0.dst == 1.dst);
13927 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13928 // %}
13929 
13930 // peephole
13931 // %{
13932 //   peepmatch (decL_rReg movL);
13933 //   peepconstraint (0.dst == 1.dst);
13934 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13935 // %}
13936 
13937 // peephole
13938 // %{
13939 //   peepmatch (addL_rReg_imm movL);
13940 //   peepconstraint (0.dst == 1.dst);
13941 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13942 // %}
13943 
13944 // peephole
13945 // %{
13946 //   peepmatch (addP_rReg_imm movP);
13947 //   peepconstraint (0.dst == 1.dst);
13948 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13949 // %}
13950 
13951 // // Change load of spilled value to only a spill
13952 // instruct storeI(memory mem, rRegI src)
13953 // %{
13954 //   match(Set mem (StoreI mem src));
13955 // %}
13956 //
13957 // instruct loadI(rRegI dst, memory mem)
13958 // %{
13959 //   match(Set dst (LoadI mem));
13960 // %}
13961 //
13962 
13963 peephole
13964 %{
13965   peepmatch (loadI storeI);
13966   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13967   peepreplace (storeI(1.mem 1.mem 1.src));
13968 %}
13969 
13970 peephole
13971 %{
13972   peepmatch (loadL storeL);
13973   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13974   peepreplace (storeL(1.mem 1.mem 1.src));
13975 %}
13976 
13977 //----------SMARTSPILL RULES---------------------------------------------------
13978 // These must follow all instruction definitions as they use the names
13979 // defined in the instructions definitions.
--- EOF ---