1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 }
  458 
  459 static bool generate_vzeroupper(Compile* C) {
  460   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  461 }
  462 
  463 static int clear_avx_size() {
  464   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  465 }
  466 
  467 // !!!!! Special hack to get all types of calls to specify the byte offset
  468 //       from the start of the call to the point where the return address
  469 //       will point.
  470 int MachCallStaticJavaNode::ret_addr_offset()
  471 {
  472   int offset = 5; // 5 bytes from start of call to where return address points
  473   offset += clear_avx_size();
  474   return offset;
  475 }
  476 
  477 int MachCallDynamicJavaNode::ret_addr_offset()
  478 {
  479   int offset = 15; // 15 bytes from start of call to where return address points
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 
  484 int MachCallRuntimeNode::ret_addr_offset() {
  485   int offset = 13; // movq r10,#addr; callq (r10)
  486   if (this->ideal_Opcode() != Op_CallLeafVector) {
  487     offset += clear_avx_size();
  488   }
  489   return offset;
  490 }
  491 //
  492 // Compute padding required for nodes which need alignment
  493 //
  494 
  495 // The address of the call instruction needs to be 4-byte aligned to
  496 // ensure that it does not span a cache line so that it can be patched.
  497 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  498 {
  499   current_offset += clear_avx_size(); // skip vzeroupper
  500   current_offset += 1; // skip call opcode byte
  501   return align_up(current_offset, alignment_required()) - current_offset;
  502 }
  503 
  504 // The address of the call instruction needs to be 4-byte aligned to
  505 // ensure that it does not span a cache line so that it can be patched.
  506 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  507 {
  508   current_offset += clear_avx_size(); // skip vzeroupper
  509   current_offset += 11; // skip movq instruction + call opcode byte
  510   return align_up(current_offset, alignment_required()) - current_offset;
  511 }
  512 
  513 // EMIT_RM()
  514 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  515   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  516   cbuf.insts()->emit_int8(c);
  517 }
  518 
  519 // EMIT_CC()
  520 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  521   unsigned char c = (unsigned char) (f1 | f2);
  522   cbuf.insts()->emit_int8(c);
  523 }
  524 
  525 // EMIT_OPCODE()
  526 void emit_opcode(CodeBuffer &cbuf, int code) {
  527   cbuf.insts()->emit_int8((unsigned char) code);
  528 }
  529 
  530 // EMIT_OPCODE() w/ relocation information
  531 void emit_opcode(CodeBuffer &cbuf,
  532                  int code, relocInfo::relocType reloc, int offset, int format)
  533 {
  534   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  535   emit_opcode(cbuf, code);
  536 }
  537 
  538 // EMIT_D8()
  539 void emit_d8(CodeBuffer &cbuf, int d8) {
  540   cbuf.insts()->emit_int8((unsigned char) d8);
  541 }
  542 
  543 // EMIT_D16()
  544 void emit_d16(CodeBuffer &cbuf, int d16) {
  545   cbuf.insts()->emit_int16(d16);
  546 }
  547 
  548 // EMIT_D32()
  549 void emit_d32(CodeBuffer &cbuf, int d32) {
  550   cbuf.insts()->emit_int32(d32);
  551 }
  552 
  553 // EMIT_D64()
  554 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  555   cbuf.insts()->emit_int64(d64);
  556 }
  557 
  558 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  559 void emit_d32_reloc(CodeBuffer& cbuf,
  560                     int d32,
  561                     relocInfo::relocType reloc,
  562                     int format)
  563 {
  564   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  565   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  566   cbuf.insts()->emit_int32(d32);
  567 }
  568 
  569 // emit 32 bit value and construct relocation entry from RelocationHolder
  570 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  571 #ifdef ASSERT
  572   if (rspec.reloc()->type() == relocInfo::oop_type &&
  573       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  574     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  575     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  576   }
  577 #endif
  578   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  579   cbuf.insts()->emit_int32(d32);
  580 }
  581 
  582 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  583   address next_ip = cbuf.insts_end() + 4;
  584   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  585                  external_word_Relocation::spec(addr),
  586                  RELOC_DISP32);
  587 }
  588 
  589 
  590 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  591 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  592   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  593   cbuf.insts()->emit_int64(d64);
  594 }
  595 
  596 // emit 64 bit value and construct relocation entry from RelocationHolder
  597 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  598 #ifdef ASSERT
  599   if (rspec.reloc()->type() == relocInfo::oop_type &&
  600       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  601     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  602     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  603   }
  604 #endif
  605   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  606   cbuf.insts()->emit_int64(d64);
  607 }
  608 
  609 // Access stack slot for load or store
  610 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  611 {
  612   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  613   if (-0x80 <= disp && disp < 0x80) {
  614     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  615     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  616     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  617   } else {
  618     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  619     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  620     emit_d32(cbuf, disp);     // Displacement // R/M byte
  621   }
  622 }
  623 
  624    // rRegI ereg, memory mem) %{    // emit_reg_mem
  625 void encode_RegMem(CodeBuffer &cbuf,
  626                    int reg,
  627                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  628 {
  629   assert(disp_reloc == relocInfo::none, "cannot have disp");
  630   int regenc = reg & 7;
  631   int baseenc = base & 7;
  632   int indexenc = index & 7;
  633 
  634   // There is no index & no scale, use form without SIB byte
  635   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  636     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  637     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  638       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  639     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  640       // If 8-bit displacement, mode 0x1
  641       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  642       emit_d8(cbuf, disp);
  643     } else {
  644       // If 32-bit displacement
  645       if (base == -1) { // Special flag for absolute address
  646         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       } else {
  653         // Normal base + offset
  654         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       }
  661     }
  662   } else {
  663     // Else, encode with the SIB byte
  664     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  665     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  666       // If no displacement
  667       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  668       emit_rm(cbuf, scale, indexenc, baseenc);
  669     } else {
  670       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  671         // If 8-bit displacement, mode 0x1
  672         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  673         emit_rm(cbuf, scale, indexenc, baseenc);
  674         emit_d8(cbuf, disp);
  675       } else {
  676         // If 32-bit displacement
  677         if (base == 0x04 ) {
  678           emit_rm(cbuf, 0x2, regenc, 0x4);
  679           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  680         } else {
  681           emit_rm(cbuf, 0x2, regenc, 0x4);
  682           emit_rm(cbuf, scale, indexenc, baseenc); // *
  683         }
  684         if (disp_reloc != relocInfo::none) {
  685           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  686         } else {
  687           emit_d32(cbuf, disp);
  688         }
  689       }
  690     }
  691   }
  692 }
  693 
  694 // This could be in MacroAssembler but it's fairly C2 specific
  695 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  696   Label exit;
  697   __ jccb(Assembler::noParity, exit);
  698   __ pushf();
  699   //
  700   // comiss/ucomiss instructions set ZF,PF,CF flags and
  701   // zero OF,AF,SF for NaN values.
  702   // Fixup flags by zeroing ZF,PF so that compare of NaN
  703   // values returns 'less than' result (CF is set).
  704   // Leave the rest of flags unchanged.
  705   //
  706   //    7 6 5 4 3 2 1 0
  707   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  708   //    0 0 1 0 1 0 1 1   (0x2B)
  709   //
  710   __ andq(Address(rsp, 0), 0xffffff2b);
  711   __ popf();
  712   __ bind(exit);
  713 }
  714 
  715 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  716   Label done;
  717   __ movl(dst, -1);
  718   __ jcc(Assembler::parity, done);
  719   __ jcc(Assembler::below, done);
  720   __ setb(Assembler::notEqual, dst);
  721   __ movzbl(dst, dst);
  722   __ bind(done);
  723 }
  724 
  725 // Math.min()    # Math.max()
  726 // --------------------------
  727 // ucomis[s/d]   #
  728 // ja   -> b     # a
  729 // jp   -> NaN   # NaN
  730 // jb   -> a     # b
  731 // je            #
  732 // |-jz -> a | b # a & b
  733 // |    -> a     #
  734 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  735                      XMMRegister a, XMMRegister b,
  736                      XMMRegister xmmt, Register rt,
  737                      bool min, bool single) {
  738 
  739   Label nan, zero, below, above, done;
  740 
  741   if (single)
  742     __ ucomiss(a, b);
  743   else
  744     __ ucomisd(a, b);
  745 
  746   if (dst->encoding() != (min ? b : a)->encoding())
  747     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  748   else
  749     __ jccb(Assembler::above, done);
  750 
  751   __ jccb(Assembler::parity, nan);  // PF=1
  752   __ jccb(Assembler::below, below); // CF=1
  753 
  754   // equal
  755   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  756   if (single) {
  757     __ ucomiss(a, xmmt);
  758     __ jccb(Assembler::equal, zero);
  759 
  760     __ movflt(dst, a);
  761     __ jmp(done);
  762   }
  763   else {
  764     __ ucomisd(a, xmmt);
  765     __ jccb(Assembler::equal, zero);
  766 
  767     __ movdbl(dst, a);
  768     __ jmp(done);
  769   }
  770 
  771   __ bind(zero);
  772   if (min)
  773     __ vpor(dst, a, b, Assembler::AVX_128bit);
  774   else
  775     __ vpand(dst, a, b, Assembler::AVX_128bit);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(above);
  780   if (single)
  781     __ movflt(dst, min ? b : a);
  782   else
  783     __ movdbl(dst, min ? b : a);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(nan);
  788   if (single) {
  789     __ movl(rt, 0x7fc00000); // Float.NaN
  790     __ movdl(dst, rt);
  791   }
  792   else {
  793     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  794     __ movdq(dst, rt);
  795   }
  796   __ jmp(done);
  797 
  798   __ bind(below);
  799   if (single)
  800     __ movflt(dst, min ? a : b);
  801   else
  802     __ movdbl(dst, min ? a : b);
  803 
  804   __ bind(done);
  805 }
  806 
  807 //=============================================================================
  808 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  809 
  810 int ConstantTable::calculate_table_base_offset() const {
  811   return 0;  // absolute addressing, no offset
  812 }
  813 
  814 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  815 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  816   ShouldNotReachHere();
  817 }
  818 
  819 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  820   // Empty encoding
  821 }
  822 
  823 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  824   return 0;
  825 }
  826 
  827 #ifndef PRODUCT
  828 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   st->print("# MachConstantBaseNode (empty encoding)");
  830 }
  831 #endif
  832 
  833 
  834 //=============================================================================
  835 #ifndef PRODUCT
  836 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   Compile* C = ra_->C;
  838 
  839   int framesize = C->output()->frame_size_in_bytes();
  840   int bangsize = C->output()->bang_size_in_bytes();
  841   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  842   // Remove wordSize for return addr which is already pushed.
  843   framesize -= wordSize;
  844 
  845   if (C->output()->need_stack_bang(bangsize)) {
  846     framesize -= wordSize;
  847     st->print("# stack bang (%d bytes)", bangsize);
  848     st->print("\n\t");
  849     st->print("pushq   rbp\t# Save rbp");
  850     if (PreserveFramePointer) {
  851         st->print("\n\t");
  852         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  853     }
  854     if (framesize) {
  855       st->print("\n\t");
  856       st->print("subq    rsp, #%d\t# Create frame",framesize);
  857     }
  858   } else {
  859     st->print("subq    rsp, #%d\t# Create frame",framesize);
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  863     if (PreserveFramePointer) {
  864       st->print("\n\t");
  865       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  866       if (framesize > 0) {
  867         st->print("\n\t");
  868         st->print("addq    rbp, #%d", framesize);
  869       }
  870     }
  871   }
  872 
  873   if (VerifyStackAtCalls) {
  874     st->print("\n\t");
  875     framesize -= wordSize;
  876     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  877 #ifdef ASSERT
  878     st->print("\n\t");
  879     st->print("# stack alignment check");
  880 #endif
  881   }
  882   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  883     st->print("\n\t");
  884     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  885     st->print("\n\t");
  886     st->print("je      fast_entry\t");
  887     st->print("\n\t");
  888     st->print("call    #nmethod_entry_barrier_stub\t");
  889     st->print("\n\tfast_entry:");
  890   }
  891   st->cr();
  892 }
  893 #endif
  894 
  895 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  896   Compile* C = ra_->C;
  897   C2_MacroAssembler _masm(&cbuf);
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   int bangsize = C->output()->bang_size_in_bytes();
  901 
  902   if (C->clinit_barrier_on_entry()) {
  903     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  904     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  905 
  906     Label L_skip_barrier;
  907     Register klass = rscratch1;
  908 
  909     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  910     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  911 
  912     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  913 
  914     __ bind(L_skip_barrier);
  915   }
  916 
  917   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  918 
  919   C->output()->set_frame_complete(cbuf.insts_size());
  920 
  921   if (C->has_mach_constant_base_node()) {
  922     // NOTE: We set the table base offset here because users might be
  923     // emitted before MachConstantBaseNode.
  924     ConstantTable& constant_table = C->output()->constant_table();
  925     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  926   }
  927 }
  928 
  929 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  930 {
  931   return MachNode::size(ra_); // too many variables; just compute it
  932                               // the hard way
  933 }
  934 
  935 int MachPrologNode::reloc() const
  936 {
  937   return 0; // a large enough number
  938 }
  939 
  940 //=============================================================================
  941 #ifndef PRODUCT
  942 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  943 {
  944   Compile* C = ra_->C;
  945   if (generate_vzeroupper(C)) {
  946     st->print("vzeroupper");
  947     st->cr(); st->print("\t");
  948   }
  949 
  950   int framesize = C->output()->frame_size_in_bytes();
  951   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  952   // Remove word for return adr already pushed
  953   // and RBP
  954   framesize -= 2*wordSize;
  955 
  956   if (framesize) {
  957     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  958     st->print("\t");
  959   }
  960 
  961   st->print_cr("popq    rbp");
  962   if (do_polling() && C->is_method_compilation()) {
  963     st->print("\t");
  964     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  965                  "ja       #safepoint_stub\t"
  966                  "# Safepoint: poll for GC");
  967   }
  968 }
  969 #endif
  970 
  971 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  972 {
  973   Compile* C = ra_->C;
  974   MacroAssembler _masm(&cbuf);
  975 
  976   if (generate_vzeroupper(C)) {
  977     // Clear upper bits of YMM registers when current compiled code uses
  978     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  979     __ vzeroupper();
  980   }
  981 
  982   int framesize = C->output()->frame_size_in_bytes();
  983   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  984   // Remove word for return adr already pushed
  985   // and RBP
  986   framesize -= 2*wordSize;
  987 
  988   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  989 
  990   if (framesize) {
  991     emit_opcode(cbuf, Assembler::REX_W);
  992     if (framesize < 0x80) {
  993       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  995       emit_d8(cbuf, framesize);
  996     } else {
  997       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  998       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  999       emit_d32(cbuf, framesize);
 1000     }
 1001   }
 1002 
 1003   // popq rbp
 1004   emit_opcode(cbuf, 0x58 | RBP_enc);
 1005 
 1006   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1007     __ reserved_stack_check();
 1008   }
 1009 
 1010   if (do_polling() && C->is_method_compilation()) {
 1011     MacroAssembler _masm(&cbuf);
 1012     Label dummy_label;
 1013     Label* code_stub = &dummy_label;
 1014     if (!C->output()->in_scratch_emit_size()) {
 1015       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1016     }
 1017     __ relocate(relocInfo::poll_return_type);
 1018     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1019   }
 1020 }
 1021 
 1022 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1023 {
 1024   return MachNode::size(ra_); // too many variables; just compute it
 1025                               // the hard way
 1026 }
 1027 
 1028 int MachEpilogNode::reloc() const
 1029 {
 1030   return 2; // a large enough number
 1031 }
 1032 
 1033 const Pipeline* MachEpilogNode::pipeline() const
 1034 {
 1035   return MachNode::pipeline_class();
 1036 }
 1037 
 1038 //=============================================================================
 1039 
 1040 enum RC {
 1041   rc_bad,
 1042   rc_int,
 1043   rc_kreg,
 1044   rc_float,
 1045   rc_stack
 1046 };
 1047 
 1048 static enum RC rc_class(OptoReg::Name reg)
 1049 {
 1050   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1051 
 1052   if (OptoReg::is_stack(reg)) return rc_stack;
 1053 
 1054   VMReg r = OptoReg::as_VMReg(reg);
 1055 
 1056   if (r->is_Register()) return rc_int;
 1057 
 1058   if (r->is_KRegister()) return rc_kreg;
 1059 
 1060   assert(r->is_XMMRegister(), "must be");
 1061   return rc_float;
 1062 }
 1063 
 1064 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1065 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1066                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1067 
 1068 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1069                      int stack_offset, int reg, uint ireg, outputStream* st);
 1070 
 1071 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1072                                       int dst_offset, uint ireg, outputStream* st) {
 1073   if (cbuf) {
 1074     MacroAssembler _masm(cbuf);
 1075     switch (ireg) {
 1076     case Op_VecS:
 1077       __ movq(Address(rsp, -8), rax);
 1078       __ movl(rax, Address(rsp, src_offset));
 1079       __ movl(Address(rsp, dst_offset), rax);
 1080       __ movq(rax, Address(rsp, -8));
 1081       break;
 1082     case Op_VecD:
 1083       __ pushq(Address(rsp, src_offset));
 1084       __ popq (Address(rsp, dst_offset));
 1085       break;
 1086     case Op_VecX:
 1087       __ pushq(Address(rsp, src_offset));
 1088       __ popq (Address(rsp, dst_offset));
 1089       __ pushq(Address(rsp, src_offset+8));
 1090       __ popq (Address(rsp, dst_offset+8));
 1091       break;
 1092     case Op_VecY:
 1093       __ vmovdqu(Address(rsp, -32), xmm0);
 1094       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1095       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1096       __ vmovdqu(xmm0, Address(rsp, -32));
 1097       break;
 1098     case Op_VecZ:
 1099       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1100       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1101       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1102       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1103       break;
 1104     default:
 1105       ShouldNotReachHere();
 1106     }
 1107 #ifndef PRODUCT
 1108   } else {
 1109     switch (ireg) {
 1110     case Op_VecS:
 1111       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1112                 "movl    rax, [rsp + #%d]\n\t"
 1113                 "movl    [rsp + #%d], rax\n\t"
 1114                 "movq    rax, [rsp - #8]",
 1115                 src_offset, dst_offset);
 1116       break;
 1117     case Op_VecD:
 1118       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1119                 "popq    [rsp + #%d]",
 1120                 src_offset, dst_offset);
 1121       break;
 1122      case Op_VecX:
 1123       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1124                 "popq    [rsp + #%d]\n\t"
 1125                 "pushq   [rsp + #%d]\n\t"
 1126                 "popq    [rsp + #%d]",
 1127                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1128       break;
 1129     case Op_VecY:
 1130       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1133                 "vmovdqu xmm0, [rsp - #32]",
 1134                 src_offset, dst_offset);
 1135       break;
 1136     case Op_VecZ:
 1137       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1138                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1139                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1140                 "vmovdqu xmm0, [rsp - #64]",
 1141                 src_offset, dst_offset);
 1142       break;
 1143     default:
 1144       ShouldNotReachHere();
 1145     }
 1146 #endif
 1147   }
 1148 }
 1149 
 1150 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1151                                        PhaseRegAlloc* ra_,
 1152                                        bool do_size,
 1153                                        outputStream* st) const {
 1154   assert(cbuf != NULL || st  != NULL, "sanity");
 1155   // Get registers to move
 1156   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1157   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1158   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1159   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1160 
 1161   enum RC src_second_rc = rc_class(src_second);
 1162   enum RC src_first_rc = rc_class(src_first);
 1163   enum RC dst_second_rc = rc_class(dst_second);
 1164   enum RC dst_first_rc = rc_class(dst_first);
 1165 
 1166   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1167          "must move at least 1 register" );
 1168 
 1169   if (src_first == dst_first && src_second == dst_second) {
 1170     // Self copy, no move
 1171     return 0;
 1172   }
 1173   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1174     uint ireg = ideal_reg();
 1175     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1176     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1177     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1178       // mem -> mem
 1179       int src_offset = ra_->reg2offset(src_first);
 1180       int dst_offset = ra_->reg2offset(dst_first);
 1181       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1182     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1183       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1184     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1185       int stack_offset = ra_->reg2offset(dst_first);
 1186       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1187     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1188       int stack_offset = ra_->reg2offset(src_first);
 1189       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1190     } else {
 1191       ShouldNotReachHere();
 1192     }
 1193     return 0;
 1194   }
 1195   if (src_first_rc == rc_stack) {
 1196     // mem ->
 1197     if (dst_first_rc == rc_stack) {
 1198       // mem -> mem
 1199       assert(src_second != dst_first, "overlap");
 1200       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1201           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1202         // 64-bit
 1203         int src_offset = ra_->reg2offset(src_first);
 1204         int dst_offset = ra_->reg2offset(dst_first);
 1205         if (cbuf) {
 1206           MacroAssembler _masm(cbuf);
 1207           __ pushq(Address(rsp, src_offset));
 1208           __ popq (Address(rsp, dst_offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1212                     "popq    [rsp + #%d]",
 1213                      src_offset, dst_offset);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         // No pushl/popl, so:
 1221         int src_offset = ra_->reg2offset(src_first);
 1222         int dst_offset = ra_->reg2offset(dst_first);
 1223         if (cbuf) {
 1224           MacroAssembler _masm(cbuf);
 1225           __ movq(Address(rsp, -8), rax);
 1226           __ movl(rax, Address(rsp, src_offset));
 1227           __ movl(Address(rsp, dst_offset), rax);
 1228           __ movq(rax, Address(rsp, -8));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1232                     "movl    rax, [rsp + #%d]\n\t"
 1233                     "movl    [rsp + #%d], rax\n\t"
 1234                     "movq    rax, [rsp - #8]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_int) {
 1241       // mem -> gpr
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (cbuf) {
 1247           MacroAssembler _masm(cbuf);
 1248           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1249 #ifndef PRODUCT
 1250         } else {
 1251           st->print("movq    %s, [rsp + #%d]\t# spill",
 1252                      Matcher::regName[dst_first],
 1253                      offset);
 1254 #endif
 1255         }
 1256       } else {
 1257         // 32-bit
 1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1260         int offset = ra_->reg2offset(src_first);
 1261         if (cbuf) {
 1262           MacroAssembler _masm(cbuf);
 1263           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1264 #ifndef PRODUCT
 1265         } else {
 1266           st->print("movl    %s, [rsp + #%d]\t# spill",
 1267                      Matcher::regName[dst_first],
 1268                      offset);
 1269 #endif
 1270         }
 1271       }
 1272       return 0;
 1273     } else if (dst_first_rc == rc_float) {
 1274       // mem-> xmm
 1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1277         // 64-bit
 1278         int offset = ra_->reg2offset(src_first);
 1279         if (cbuf) {
 1280           MacroAssembler _masm(cbuf);
 1281           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("%s  %s, [rsp + #%d]\t# spill",
 1285                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1286                      Matcher::regName[dst_first],
 1287                      offset);
 1288 #endif
 1289         }
 1290       } else {
 1291         // 32-bit
 1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1294         int offset = ra_->reg2offset(src_first);
 1295         if (cbuf) {
 1296           MacroAssembler _masm(cbuf);
 1297           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movss   %s, [rsp + #%d]\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      offset);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_kreg) {
 1308       // mem -> kreg
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         int offset = ra_->reg2offset(src_first);
 1313         if (cbuf) {
 1314           MacroAssembler _masm(cbuf);
 1315           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1316 #ifndef PRODUCT
 1317         } else {
 1318           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1319                      Matcher::regName[dst_first],
 1320                      offset);
 1321 #endif
 1322         }
 1323       }
 1324       return 0;
 1325     }
 1326   } else if (src_first_rc == rc_int) {
 1327     // gpr ->
 1328     if (dst_first_rc == rc_stack) {
 1329       // gpr -> mem
 1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1332         // 64-bit
 1333         int offset = ra_->reg2offset(dst_first);
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("movq    [rsp + #%d], %s\t# spill",
 1340                      offset,
 1341                      Matcher::regName[src_first]);
 1342 #endif
 1343         }
 1344       } else {
 1345         // 32-bit
 1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("movl    [rsp + #%d], %s\t# spill",
 1355                      offset,
 1356                      Matcher::regName[src_first]);
 1357 #endif
 1358         }
 1359       }
 1360       return 0;
 1361     } else if (dst_first_rc == rc_int) {
 1362       // gpr -> gpr
 1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1365         // 64-bit
 1366         if (cbuf) {
 1367           MacroAssembler _masm(cbuf);
 1368           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1369                   as_Register(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movq    %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377         return 0;
 1378       } else {
 1379         // 32-bit
 1380         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1381         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1382         if (cbuf) {
 1383           MacroAssembler _masm(cbuf);
 1384           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1385                   as_Register(Matcher::_regEncode[src_first]));
 1386 #ifndef PRODUCT
 1387         } else {
 1388           st->print("movl    %s, %s\t# spill",
 1389                      Matcher::regName[dst_first],
 1390                      Matcher::regName[src_first]);
 1391 #endif
 1392         }
 1393         return 0;
 1394       }
 1395     } else if (dst_first_rc == rc_float) {
 1396       // gpr -> xmm
 1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1399         // 64-bit
 1400         if (cbuf) {
 1401           MacroAssembler _masm(cbuf);
 1402           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1403 #ifndef PRODUCT
 1404         } else {
 1405           st->print("movdq   %s, %s\t# spill",
 1406                      Matcher::regName[dst_first],
 1407                      Matcher::regName[src_first]);
 1408 #endif
 1409         }
 1410       } else {
 1411         // 32-bit
 1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1414         if (cbuf) {
 1415           MacroAssembler _masm(cbuf);
 1416           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1417 #ifndef PRODUCT
 1418         } else {
 1419           st->print("movdl   %s, %s\t# spill",
 1420                      Matcher::regName[dst_first],
 1421                      Matcher::regName[src_first]);
 1422 #endif
 1423         }
 1424       }
 1425       return 0;
 1426     } else if (dst_first_rc == rc_kreg) {
 1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1429         // 64-bit
 1430         if (cbuf) {
 1431           MacroAssembler _masm(cbuf);
 1432           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1433   #ifndef PRODUCT
 1434         } else {
 1435            st->print("kmovq   %s, %s\t# spill",
 1436                        Matcher::regName[dst_first],
 1437                        Matcher::regName[src_first]);
 1438   #endif
 1439         }
 1440       }
 1441       Unimplemented();
 1442       return 0;
 1443     }
 1444   } else if (src_first_rc == rc_float) {
 1445     // xmm ->
 1446     if (dst_first_rc == rc_stack) {
 1447       // xmm -> mem
 1448       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1449           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1450         // 64-bit
 1451         int offset = ra_->reg2offset(dst_first);
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1455 #ifndef PRODUCT
 1456         } else {
 1457           st->print("movsd   [rsp + #%d], %s\t# spill",
 1458                      offset,
 1459                      Matcher::regName[src_first]);
 1460 #endif
 1461         }
 1462       } else {
 1463         // 32-bit
 1464         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1465         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1466         int offset = ra_->reg2offset(dst_first);
 1467         if (cbuf) {
 1468           MacroAssembler _masm(cbuf);
 1469           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1470 #ifndef PRODUCT
 1471         } else {
 1472           st->print("movss   [rsp + #%d], %s\t# spill",
 1473                      offset,
 1474                      Matcher::regName[src_first]);
 1475 #endif
 1476         }
 1477       }
 1478       return 0;
 1479     } else if (dst_first_rc == rc_int) {
 1480       // xmm -> gpr
 1481       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1482           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1483         // 64-bit
 1484         if (cbuf) {
 1485           MacroAssembler _masm(cbuf);
 1486           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1487 #ifndef PRODUCT
 1488         } else {
 1489           st->print("movdq   %s, %s\t# spill",
 1490                      Matcher::regName[dst_first],
 1491                      Matcher::regName[src_first]);
 1492 #endif
 1493         }
 1494       } else {
 1495         // 32-bit
 1496         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1497         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1498         if (cbuf) {
 1499           MacroAssembler _masm(cbuf);
 1500           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1501 #ifndef PRODUCT
 1502         } else {
 1503           st->print("movdl   %s, %s\t# spill",
 1504                      Matcher::regName[dst_first],
 1505                      Matcher::regName[src_first]);
 1506 #endif
 1507         }
 1508       }
 1509       return 0;
 1510     } else if (dst_first_rc == rc_float) {
 1511       // xmm -> xmm
 1512       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1513           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1514         // 64-bit
 1515         if (cbuf) {
 1516           MacroAssembler _masm(cbuf);
 1517           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1518 #ifndef PRODUCT
 1519         } else {
 1520           st->print("%s  %s, %s\t# spill",
 1521                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1522                      Matcher::regName[dst_first],
 1523                      Matcher::regName[src_first]);
 1524 #endif
 1525         }
 1526       } else {
 1527         // 32-bit
 1528         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1529         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1530         if (cbuf) {
 1531           MacroAssembler _masm(cbuf);
 1532           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1533 #ifndef PRODUCT
 1534         } else {
 1535           st->print("%s  %s, %s\t# spill",
 1536                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1537                      Matcher::regName[dst_first],
 1538                      Matcher::regName[src_first]);
 1539 #endif
 1540         }
 1541       }
 1542       return 0;
 1543     } else if (dst_first_rc == rc_kreg) {
 1544       assert(false, "Illegal spilling");
 1545       return 0;
 1546     }
 1547   } else if (src_first_rc == rc_kreg) {
 1548     if (dst_first_rc == rc_stack) {
 1549       // mem -> kreg
 1550       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1551           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1552         // 64-bit
 1553         int offset = ra_->reg2offset(dst_first);
 1554         if (cbuf) {
 1555           MacroAssembler _masm(cbuf);
 1556           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1557 #ifndef PRODUCT
 1558         } else {
 1559           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1560                      offset,
 1561                      Matcher::regName[src_first]);
 1562 #endif
 1563         }
 1564       }
 1565       return 0;
 1566     } else if (dst_first_rc == rc_int) {
 1567       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1568           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1569         // 64-bit
 1570         if (cbuf) {
 1571           MacroAssembler _masm(cbuf);
 1572           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1573 #ifndef PRODUCT
 1574         } else {
 1575          st->print("kmovq   %s, %s\t# spill",
 1576                      Matcher::regName[dst_first],
 1577                      Matcher::regName[src_first]);
 1578 #endif
 1579         }
 1580       }
 1581       Unimplemented();
 1582       return 0;
 1583     } else if (dst_first_rc == rc_kreg) {
 1584       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1585           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1586         // 64-bit
 1587         if (cbuf) {
 1588           MacroAssembler _masm(cbuf);
 1589           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1590 #ifndef PRODUCT
 1591         } else {
 1592          st->print("kmovq   %s, %s\t# spill",
 1593                      Matcher::regName[dst_first],
 1594                      Matcher::regName[src_first]);
 1595 #endif
 1596         }
 1597       }
 1598       return 0;
 1599     } else if (dst_first_rc == rc_float) {
 1600       assert(false, "Illegal spill");
 1601       return 0;
 1602     }
 1603   }
 1604 
 1605   assert(0," foo ");
 1606   Unimplemented();
 1607   return 0;
 1608 }
 1609 
 1610 #ifndef PRODUCT
 1611 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1612   implementation(NULL, ra_, false, st);
 1613 }
 1614 #endif
 1615 
 1616 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1617   implementation(&cbuf, ra_, false, NULL);
 1618 }
 1619 
 1620 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1621   return MachNode::size(ra_);
 1622 }
 1623 
 1624 //=============================================================================
 1625 #ifndef PRODUCT
 1626 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1627 {
 1628   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1629   int reg = ra_->get_reg_first(this);
 1630   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1631             Matcher::regName[reg], offset);
 1632 }
 1633 #endif
 1634 
 1635 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1636 {
 1637   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1638   int reg = ra_->get_encode(this);
 1639   if (offset >= 0x80) {
 1640     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1641     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1642     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1643     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1644     emit_d32(cbuf, offset);
 1645   } else {
 1646     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1647     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1648     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1649     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1650     emit_d8(cbuf, offset);
 1651   }
 1652 }
 1653 
 1654 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1655 {
 1656   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1657   return (offset < 0x80) ? 5 : 8; // REX
 1658 }
 1659 
 1660 //=============================================================================
 1661 #ifndef PRODUCT
 1662 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1663 {
 1664   if (UseCompressedClassPointers) {
 1665     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1666     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1667     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1668   } else {
 1669     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1670                  "# Inline cache check");
 1671   }
 1672   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1673   st->print_cr("\tnop\t# nops to align entry point");
 1674 }
 1675 #endif
 1676 
 1677 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1678 {
 1679   MacroAssembler masm(&cbuf);
 1680   uint insts_size = cbuf.insts_size();
 1681   if (UseCompressedClassPointers) {
 1682     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1683     masm.cmpptr(rax, rscratch1);
 1684   } else {
 1685     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1686   }
 1687 
 1688   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1689 
 1690   /* WARNING these NOPs are critical so that verified entry point is properly
 1691      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1692   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1693   if (OptoBreakpoint) {
 1694     // Leave space for int3
 1695     nops_cnt -= 1;
 1696   }
 1697   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1698   if (nops_cnt > 0)
 1699     masm.nop(nops_cnt);
 1700 }
 1701 
 1702 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1703 {
 1704   return MachNode::size(ra_); // too many variables; just compute it
 1705                               // the hard way
 1706 }
 1707 
 1708 
 1709 //=============================================================================
 1710 
 1711 const bool Matcher::supports_vector_calling_convention(void) {
 1712   if (EnableVectorSupport && UseVectorStubs) {
 1713     return true;
 1714   }
 1715   return false;
 1716 }
 1717 
 1718 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1719   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1720   int lo = XMM0_num;
 1721   int hi = XMM0b_num;
 1722   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1723   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1724   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1725   return OptoRegPair(hi, lo);
 1726 }
 1727 
 1728 // Is this branch offset short enough that a short branch can be used?
 1729 //
 1730 // NOTE: If the platform does not provide any short branch variants, then
 1731 //       this method should return false for offset 0.
 1732 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1733   // The passed offset is relative to address of the branch.
 1734   // On 86 a branch displacement is calculated relative to address
 1735   // of a next instruction.
 1736   offset -= br_size;
 1737 
 1738   // the short version of jmpConUCF2 contains multiple branches,
 1739   // making the reach slightly less
 1740   if (rule == jmpConUCF2_rule)
 1741     return (-126 <= offset && offset <= 125);
 1742   return (-128 <= offset && offset <= 127);
 1743 }
 1744 
 1745 // Return whether or not this register is ever used as an argument.
 1746 // This function is used on startup to build the trampoline stubs in
 1747 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1748 // call in the trampoline, and arguments in those registers not be
 1749 // available to the callee.
 1750 bool Matcher::can_be_java_arg(int reg)
 1751 {
 1752   return
 1753     reg ==  RDI_num || reg == RDI_H_num ||
 1754     reg ==  RSI_num || reg == RSI_H_num ||
 1755     reg ==  RDX_num || reg == RDX_H_num ||
 1756     reg ==  RCX_num || reg == RCX_H_num ||
 1757     reg ==   R8_num || reg ==  R8_H_num ||
 1758     reg ==   R9_num || reg ==  R9_H_num ||
 1759     reg ==  R12_num || reg == R12_H_num ||
 1760     reg == XMM0_num || reg == XMM0b_num ||
 1761     reg == XMM1_num || reg == XMM1b_num ||
 1762     reg == XMM2_num || reg == XMM2b_num ||
 1763     reg == XMM3_num || reg == XMM3b_num ||
 1764     reg == XMM4_num || reg == XMM4b_num ||
 1765     reg == XMM5_num || reg == XMM5b_num ||
 1766     reg == XMM6_num || reg == XMM6b_num ||
 1767     reg == XMM7_num || reg == XMM7b_num;
 1768 }
 1769 
 1770 bool Matcher::is_spillable_arg(int reg)
 1771 {
 1772   return can_be_java_arg(reg);
 1773 }
 1774 
 1775 uint Matcher::int_pressure_limit()
 1776 {
 1777   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1778 }
 1779 
 1780 uint Matcher::float_pressure_limit()
 1781 {
 1782   // After experiment around with different values, the following default threshold
 1783   // works best for LCM's register pressure scheduling on x64.
 1784   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1785   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1786   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1787 }
 1788 
 1789 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1790   // In 64 bit mode a code which use multiply when
 1791   // devisor is constant is faster than hardware
 1792   // DIV instruction (it uses MulHiL).
 1793   return false;
 1794 }
 1795 
 1796 // Register for DIVI projection of divmodI
 1797 RegMask Matcher::divI_proj_mask() {
 1798   return INT_RAX_REG_mask();
 1799 }
 1800 
 1801 // Register for MODI projection of divmodI
 1802 RegMask Matcher::modI_proj_mask() {
 1803   return INT_RDX_REG_mask();
 1804 }
 1805 
 1806 // Register for DIVL projection of divmodL
 1807 RegMask Matcher::divL_proj_mask() {
 1808   return LONG_RAX_REG_mask();
 1809 }
 1810 
 1811 // Register for MODL projection of divmodL
 1812 RegMask Matcher::modL_proj_mask() {
 1813   return LONG_RDX_REG_mask();
 1814 }
 1815 
 1816 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1817 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1818     return NO_REG_mask();
 1819 }
 1820 
 1821 %}
 1822 
 1823 //----------ENCODING BLOCK-----------------------------------------------------
 1824 // This block specifies the encoding classes used by the compiler to
 1825 // output byte streams.  Encoding classes are parameterized macros
 1826 // used by Machine Instruction Nodes in order to generate the bit
 1827 // encoding of the instruction.  Operands specify their base encoding
 1828 // interface with the interface keyword.  There are currently
 1829 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1830 // COND_INTER.  REG_INTER causes an operand to generate a function
 1831 // which returns its register number when queried.  CONST_INTER causes
 1832 // an operand to generate a function which returns the value of the
 1833 // constant when queried.  MEMORY_INTER causes an operand to generate
 1834 // four functions which return the Base Register, the Index Register,
 1835 // the Scale Value, and the Offset Value of the operand when queried.
 1836 // COND_INTER causes an operand to generate six functions which return
 1837 // the encoding code (ie - encoding bits for the instruction)
 1838 // associated with each basic boolean condition for a conditional
 1839 // instruction.
 1840 //
 1841 // Instructions specify two basic values for encoding.  Again, a
 1842 // function is available to check if the constant displacement is an
 1843 // oop. They use the ins_encode keyword to specify their encoding
 1844 // classes (which must be a sequence of enc_class names, and their
 1845 // parameters, specified in the encoding block), and they use the
 1846 // opcode keyword to specify, in order, their primary, secondary, and
 1847 // tertiary opcode.  Only the opcode sections which a particular
 1848 // instruction needs for encoding need to be specified.
 1849 encode %{
 1850   // Build emit functions for each basic byte or larger field in the
 1851   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1852   // from C++ code in the enc_class source block.  Emit functions will
 1853   // live in the main source block for now.  In future, we can
 1854   // generalize this by adding a syntax that specifies the sizes of
 1855   // fields in an order, so that the adlc can build the emit functions
 1856   // automagically
 1857 
 1858   // Emit primary opcode
 1859   enc_class OpcP
 1860   %{
 1861     emit_opcode(cbuf, $primary);
 1862   %}
 1863 
 1864   // Emit secondary opcode
 1865   enc_class OpcS
 1866   %{
 1867     emit_opcode(cbuf, $secondary);
 1868   %}
 1869 
 1870   // Emit tertiary opcode
 1871   enc_class OpcT
 1872   %{
 1873     emit_opcode(cbuf, $tertiary);
 1874   %}
 1875 
 1876   // Emit opcode directly
 1877   enc_class Opcode(immI d8)
 1878   %{
 1879     emit_opcode(cbuf, $d8$$constant);
 1880   %}
 1881 
 1882   // Emit size prefix
 1883   enc_class SizePrefix
 1884   %{
 1885     emit_opcode(cbuf, 0x66);
 1886   %}
 1887 
 1888   enc_class reg(rRegI reg)
 1889   %{
 1890     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1891   %}
 1892 
 1893   enc_class reg_reg(rRegI dst, rRegI src)
 1894   %{
 1895     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1896   %}
 1897 
 1898   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1899   %{
 1900     emit_opcode(cbuf, $opcode$$constant);
 1901     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1902   %}
 1903 
 1904   enc_class cdql_enc(no_rax_rdx_RegI div)
 1905   %{
 1906     // Full implementation of Java idiv and irem; checks for
 1907     // special case as described in JVM spec., p.243 & p.271.
 1908     //
 1909     //         normal case                           special case
 1910     //
 1911     // input : rax: dividend                         min_int
 1912     //         reg: divisor                          -1
 1913     //
 1914     // output: rax: quotient  (= rax idiv reg)       min_int
 1915     //         rdx: remainder (= rax irem reg)       0
 1916     //
 1917     //  Code sequnce:
 1918     //
 1919     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1920     //    5:   75 07/08                jne    e <normal>
 1921     //    7:   33 d2                   xor    %edx,%edx
 1922     //  [div >= 8 -> offset + 1]
 1923     //  [REX_B]
 1924     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1925     //    c:   74 03/04                je     11 <done>
 1926     // 000000000000000e <normal>:
 1927     //    e:   99                      cltd
 1928     //  [div >= 8 -> offset + 1]
 1929     //  [REX_B]
 1930     //    f:   f7 f9                   idiv   $div
 1931     // 0000000000000011 <done>:
 1932     MacroAssembler _masm(&cbuf);
 1933     Label normal;
 1934     Label done;
 1935 
 1936     // cmp    $0x80000000,%eax
 1937     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1938 
 1939     // jne    e <normal>
 1940     __ jccb(Assembler::notEqual, normal);
 1941 
 1942     // xor    %edx,%edx
 1943     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1944 
 1945     // cmp    $0xffffffffffffffff,%ecx
 1946     __ cmpl($div$$Register, -1);
 1947 
 1948     // je     11 <done>
 1949     __ jccb(Assembler::equal, done);
 1950 
 1951     // <normal>
 1952     // cltd
 1953     __ bind(normal);
 1954     __ cdql();
 1955 
 1956     // idivl
 1957     // <done>
 1958     __ idivl($div$$Register);
 1959     __ bind(done);
 1960   %}
 1961 
 1962   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1963   %{
 1964     // Full implementation of Java ldiv and lrem; checks for
 1965     // special case as described in JVM spec., p.243 & p.271.
 1966     //
 1967     //         normal case                           special case
 1968     //
 1969     // input : rax: dividend                         min_long
 1970     //         reg: divisor                          -1
 1971     //
 1972     // output: rax: quotient  (= rax idiv reg)       min_long
 1973     //         rdx: remainder (= rax irem reg)       0
 1974     //
 1975     //  Code sequnce:
 1976     //
 1977     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1978     //    7:   00 00 80
 1979     //    a:   48 39 d0                cmp    %rdx,%rax
 1980     //    d:   75 08                   jne    17 <normal>
 1981     //    f:   33 d2                   xor    %edx,%edx
 1982     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1983     //   15:   74 05                   je     1c <done>
 1984     // 0000000000000017 <normal>:
 1985     //   17:   48 99                   cqto
 1986     //   19:   48 f7 f9                idiv   $div
 1987     // 000000000000001c <done>:
 1988     MacroAssembler _masm(&cbuf);
 1989     Label normal;
 1990     Label done;
 1991 
 1992     // mov    $0x8000000000000000,%rdx
 1993     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1994 
 1995     // cmp    %rdx,%rax
 1996     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1997 
 1998     // jne    17 <normal>
 1999     __ jccb(Assembler::notEqual, normal);
 2000 
 2001     // xor    %edx,%edx
 2002     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2003 
 2004     // cmp    $0xffffffffffffffff,$div
 2005     __ cmpq($div$$Register, -1);
 2006 
 2007     // je     1e <done>
 2008     __ jccb(Assembler::equal, done);
 2009 
 2010     // <normal>
 2011     // cqto
 2012     __ bind(normal);
 2013     __ cdqq();
 2014 
 2015     // idivq (note: must be emitted by the user of this rule)
 2016     // <done>
 2017     __ idivq($div$$Register);
 2018     __ bind(done);
 2019   %}
 2020 
 2021   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2022   enc_class OpcSE(immI imm)
 2023   %{
 2024     // Emit primary opcode and set sign-extend bit
 2025     // Check for 8-bit immediate, and set sign extend bit in opcode
 2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2027       emit_opcode(cbuf, $primary | 0x02);
 2028     } else {
 2029       // 32-bit immediate
 2030       emit_opcode(cbuf, $primary);
 2031     }
 2032   %}
 2033 
 2034   enc_class OpcSErm(rRegI dst, immI imm)
 2035   %{
 2036     // OpcSEr/m
 2037     int dstenc = $dst$$reg;
 2038     if (dstenc >= 8) {
 2039       emit_opcode(cbuf, Assembler::REX_B);
 2040       dstenc -= 8;
 2041     }
 2042     // Emit primary opcode and set sign-extend bit
 2043     // Check for 8-bit immediate, and set sign extend bit in opcode
 2044     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2045       emit_opcode(cbuf, $primary | 0x02);
 2046     } else {
 2047       // 32-bit immediate
 2048       emit_opcode(cbuf, $primary);
 2049     }
 2050     // Emit r/m byte with secondary opcode, after primary opcode.
 2051     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2052   %}
 2053 
 2054   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2055   %{
 2056     // OpcSEr/m
 2057     int dstenc = $dst$$reg;
 2058     if (dstenc < 8) {
 2059       emit_opcode(cbuf, Assembler::REX_W);
 2060     } else {
 2061       emit_opcode(cbuf, Assembler::REX_WB);
 2062       dstenc -= 8;
 2063     }
 2064     // Emit primary opcode and set sign-extend bit
 2065     // Check for 8-bit immediate, and set sign extend bit in opcode
 2066     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2067       emit_opcode(cbuf, $primary | 0x02);
 2068     } else {
 2069       // 32-bit immediate
 2070       emit_opcode(cbuf, $primary);
 2071     }
 2072     // Emit r/m byte with secondary opcode, after primary opcode.
 2073     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2074   %}
 2075 
 2076   enc_class Con8or32(immI imm)
 2077   %{
 2078     // Check for 8-bit immediate, and set sign extend bit in opcode
 2079     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2080       $$$emit8$imm$$constant;
 2081     } else {
 2082       // 32-bit immediate
 2083       $$$emit32$imm$$constant;
 2084     }
 2085   %}
 2086 
 2087   enc_class opc2_reg(rRegI dst)
 2088   %{
 2089     // BSWAP
 2090     emit_cc(cbuf, $secondary, $dst$$reg);
 2091   %}
 2092 
 2093   enc_class opc3_reg(rRegI dst)
 2094   %{
 2095     // BSWAP
 2096     emit_cc(cbuf, $tertiary, $dst$$reg);
 2097   %}
 2098 
 2099   enc_class reg_opc(rRegI div)
 2100   %{
 2101     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2102     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2103   %}
 2104 
 2105   enc_class enc_cmov(cmpOp cop)
 2106   %{
 2107     // CMOV
 2108     $$$emit8$primary;
 2109     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2110   %}
 2111 
 2112   enc_class enc_PartialSubtypeCheck()
 2113   %{
 2114     Register Rrdi = as_Register(RDI_enc); // result register
 2115     Register Rrax = as_Register(RAX_enc); // super class
 2116     Register Rrcx = as_Register(RCX_enc); // killed
 2117     Register Rrsi = as_Register(RSI_enc); // sub class
 2118     Label miss;
 2119     const bool set_cond_codes = true;
 2120 
 2121     MacroAssembler _masm(&cbuf);
 2122     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2123                                      NULL, &miss,
 2124                                      /*set_cond_codes:*/ true);
 2125     if ($primary) {
 2126       __ xorptr(Rrdi, Rrdi);
 2127     }
 2128     __ bind(miss);
 2129   %}
 2130 
 2131   enc_class clear_avx %{
 2132     debug_only(int off0 = cbuf.insts_size());
 2133     if (generate_vzeroupper(Compile::current())) {
 2134       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2135       // Clear upper bits of YMM registers when current compiled code uses
 2136       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2137       MacroAssembler _masm(&cbuf);
 2138       __ vzeroupper();
 2139     }
 2140     debug_only(int off1 = cbuf.insts_size());
 2141     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2142   %}
 2143 
 2144   enc_class Java_To_Runtime(method meth) %{
 2145     // No relocation needed
 2146     MacroAssembler _masm(&cbuf);
 2147     __ mov64(r10, (int64_t) $meth$$method);
 2148     __ call(r10);
 2149     __ post_call_nop();
 2150   %}
 2151 
 2152   enc_class Java_Static_Call(method meth)
 2153   %{
 2154     // JAVA STATIC CALL
 2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2156     // determine who we intended to call.
 2157     MacroAssembler _masm(&cbuf);
 2158     cbuf.set_insts_mark();
 2159     $$$emit8$primary;
 2160 
 2161     if (!_method) {
 2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2163                      runtime_call_Relocation::spec(),
 2164                      RELOC_DISP32);
 2165     } else {
 2166       int method_index = resolved_method_index(cbuf);
 2167       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2168                                                   : static_call_Relocation::spec(method_index);
 2169       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2170                      rspec, RELOC_DISP32);
 2171       address mark = cbuf.insts_mark();
 2172       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2173         // Calls of the same statically bound method can share
 2174         // a stub to the interpreter.
 2175         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2176       } else {
 2177         // Emit stubs for static call.
 2178         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2179         if (stub == NULL) {
 2180           ciEnv::current()->record_failure("CodeCache is full");
 2181           return;
 2182         }
 2183       }
 2184     }
 2185     _masm.clear_inst_mark();
 2186     __ post_call_nop();
 2187   %}
 2188 
 2189   enc_class Java_Dynamic_Call(method meth) %{
 2190     MacroAssembler _masm(&cbuf);
 2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2192     __ post_call_nop();
 2193   %}
 2194 
 2195   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2196   %{
 2197     // SAL, SAR, SHR
 2198     int dstenc = $dst$$reg;
 2199     if (dstenc >= 8) {
 2200       emit_opcode(cbuf, Assembler::REX_B);
 2201       dstenc -= 8;
 2202     }
 2203     $$$emit8$primary;
 2204     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2205     $$$emit8$shift$$constant;
 2206   %}
 2207 
 2208   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2209   %{
 2210     // SAL, SAR, SHR
 2211     int dstenc = $dst$$reg;
 2212     if (dstenc < 8) {
 2213       emit_opcode(cbuf, Assembler::REX_W);
 2214     } else {
 2215       emit_opcode(cbuf, Assembler::REX_WB);
 2216       dstenc -= 8;
 2217     }
 2218     $$$emit8$primary;
 2219     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2220     $$$emit8$shift$$constant;
 2221   %}
 2222 
 2223   enc_class load_immI(rRegI dst, immI src)
 2224   %{
 2225     int dstenc = $dst$$reg;
 2226     if (dstenc >= 8) {
 2227       emit_opcode(cbuf, Assembler::REX_B);
 2228       dstenc -= 8;
 2229     }
 2230     emit_opcode(cbuf, 0xB8 | dstenc);
 2231     $$$emit32$src$$constant;
 2232   %}
 2233 
 2234   enc_class load_immL(rRegL dst, immL src)
 2235   %{
 2236     int dstenc = $dst$$reg;
 2237     if (dstenc < 8) {
 2238       emit_opcode(cbuf, Assembler::REX_W);
 2239     } else {
 2240       emit_opcode(cbuf, Assembler::REX_WB);
 2241       dstenc -= 8;
 2242     }
 2243     emit_opcode(cbuf, 0xB8 | dstenc);
 2244     emit_d64(cbuf, $src$$constant);
 2245   %}
 2246 
 2247   enc_class load_immUL32(rRegL dst, immUL32 src)
 2248   %{
 2249     // same as load_immI, but this time we care about zeroes in the high word
 2250     int dstenc = $dst$$reg;
 2251     if (dstenc >= 8) {
 2252       emit_opcode(cbuf, Assembler::REX_B);
 2253       dstenc -= 8;
 2254     }
 2255     emit_opcode(cbuf, 0xB8 | dstenc);
 2256     $$$emit32$src$$constant;
 2257   %}
 2258 
 2259   enc_class load_immL32(rRegL dst, immL32 src)
 2260   %{
 2261     int dstenc = $dst$$reg;
 2262     if (dstenc < 8) {
 2263       emit_opcode(cbuf, Assembler::REX_W);
 2264     } else {
 2265       emit_opcode(cbuf, Assembler::REX_WB);
 2266       dstenc -= 8;
 2267     }
 2268     emit_opcode(cbuf, 0xC7);
 2269     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2270     $$$emit32$src$$constant;
 2271   %}
 2272 
 2273   enc_class load_immP31(rRegP dst, immP32 src)
 2274   %{
 2275     // same as load_immI, but this time we care about zeroes in the high word
 2276     int dstenc = $dst$$reg;
 2277     if (dstenc >= 8) {
 2278       emit_opcode(cbuf, Assembler::REX_B);
 2279       dstenc -= 8;
 2280     }
 2281     emit_opcode(cbuf, 0xB8 | dstenc);
 2282     $$$emit32$src$$constant;
 2283   %}
 2284 
 2285   enc_class load_immP(rRegP dst, immP src)
 2286   %{
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc < 8) {
 2289       emit_opcode(cbuf, Assembler::REX_W);
 2290     } else {
 2291       emit_opcode(cbuf, Assembler::REX_WB);
 2292       dstenc -= 8;
 2293     }
 2294     emit_opcode(cbuf, 0xB8 | dstenc);
 2295     // This next line should be generated from ADLC
 2296     if ($src->constant_reloc() != relocInfo::none) {
 2297       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2298     } else {
 2299       emit_d64(cbuf, $src$$constant);
 2300     }
 2301   %}
 2302 
 2303   enc_class Con32(immI src)
 2304   %{
 2305     // Output immediate
 2306     $$$emit32$src$$constant;
 2307   %}
 2308 
 2309   enc_class Con32F_as_bits(immF src)
 2310   %{
 2311     // Output Float immediate bits
 2312     jfloat jf = $src$$constant;
 2313     jint jf_as_bits = jint_cast(jf);
 2314     emit_d32(cbuf, jf_as_bits);
 2315   %}
 2316 
 2317   enc_class Con16(immI src)
 2318   %{
 2319     // Output immediate
 2320     $$$emit16$src$$constant;
 2321   %}
 2322 
 2323   // How is this different from Con32??? XXX
 2324   enc_class Con_d32(immI src)
 2325   %{
 2326     emit_d32(cbuf,$src$$constant);
 2327   %}
 2328 
 2329   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2330     // Output immediate memory reference
 2331     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2332     emit_d32(cbuf, 0x00);
 2333   %}
 2334 
 2335   enc_class lock_prefix()
 2336   %{
 2337     emit_opcode(cbuf, 0xF0); // lock
 2338   %}
 2339 
 2340   enc_class REX_mem(memory mem)
 2341   %{
 2342     if ($mem$$base >= 8) {
 2343       if ($mem$$index < 8) {
 2344         emit_opcode(cbuf, Assembler::REX_B);
 2345       } else {
 2346         emit_opcode(cbuf, Assembler::REX_XB);
 2347       }
 2348     } else {
 2349       if ($mem$$index >= 8) {
 2350         emit_opcode(cbuf, Assembler::REX_X);
 2351       }
 2352     }
 2353   %}
 2354 
 2355   enc_class REX_mem_wide(memory mem)
 2356   %{
 2357     if ($mem$$base >= 8) {
 2358       if ($mem$$index < 8) {
 2359         emit_opcode(cbuf, Assembler::REX_WB);
 2360       } else {
 2361         emit_opcode(cbuf, Assembler::REX_WXB);
 2362       }
 2363     } else {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_W);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_WX);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   // for byte regs
 2373   enc_class REX_breg(rRegI reg)
 2374   %{
 2375     if ($reg$$reg >= 4) {
 2376       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2382   %{
 2383     if ($dst$$reg < 8) {
 2384       if ($src$$reg >= 4) {
 2385         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2386       }
 2387     } else {
 2388       if ($src$$reg < 8) {
 2389         emit_opcode(cbuf, Assembler::REX_R);
 2390       } else {
 2391         emit_opcode(cbuf, Assembler::REX_RB);
 2392       }
 2393     }
 2394   %}
 2395 
 2396   // for byte regs
 2397   enc_class REX_breg_mem(rRegI reg, memory mem)
 2398   %{
 2399     if ($reg$$reg < 8) {
 2400       if ($mem$$base < 8) {
 2401         if ($mem$$index >= 8) {
 2402           emit_opcode(cbuf, Assembler::REX_X);
 2403         } else if ($reg$$reg >= 4) {
 2404           emit_opcode(cbuf, Assembler::REX);
 2405         }
 2406       } else {
 2407         if ($mem$$index < 8) {
 2408           emit_opcode(cbuf, Assembler::REX_B);
 2409         } else {
 2410           emit_opcode(cbuf, Assembler::REX_XB);
 2411         }
 2412       }
 2413     } else {
 2414       if ($mem$$base < 8) {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_R);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_RX);
 2419         }
 2420       } else {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_RB);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RXB);
 2425         }
 2426       }
 2427     }
 2428   %}
 2429 
 2430   enc_class REX_reg(rRegI reg)
 2431   %{
 2432     if ($reg$$reg >= 8) {
 2433       emit_opcode(cbuf, Assembler::REX_B);
 2434     }
 2435   %}
 2436 
 2437   enc_class REX_reg_wide(rRegI reg)
 2438   %{
 2439     if ($reg$$reg < 8) {
 2440       emit_opcode(cbuf, Assembler::REX_W);
 2441     } else {
 2442       emit_opcode(cbuf, Assembler::REX_WB);
 2443     }
 2444   %}
 2445 
 2446   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2447   %{
 2448     if ($dst$$reg < 8) {
 2449       if ($src$$reg >= 8) {
 2450         emit_opcode(cbuf, Assembler::REX_B);
 2451       }
 2452     } else {
 2453       if ($src$$reg < 8) {
 2454         emit_opcode(cbuf, Assembler::REX_R);
 2455       } else {
 2456         emit_opcode(cbuf, Assembler::REX_RB);
 2457       }
 2458     }
 2459   %}
 2460 
 2461   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2462   %{
 2463     if ($dst$$reg < 8) {
 2464       if ($src$$reg < 8) {
 2465         emit_opcode(cbuf, Assembler::REX_W);
 2466       } else {
 2467         emit_opcode(cbuf, Assembler::REX_WB);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_WR);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_WRB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_mem(rRegI reg, memory mem)
 2479   %{
 2480     if ($reg$$reg < 8) {
 2481       if ($mem$$base < 8) {
 2482         if ($mem$$index >= 8) {
 2483           emit_opcode(cbuf, Assembler::REX_X);
 2484         }
 2485       } else {
 2486         if ($mem$$index < 8) {
 2487           emit_opcode(cbuf, Assembler::REX_B);
 2488         } else {
 2489           emit_opcode(cbuf, Assembler::REX_XB);
 2490         }
 2491       }
 2492     } else {
 2493       if ($mem$$base < 8) {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_R);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_RX);
 2498         }
 2499       } else {
 2500         if ($mem$$index < 8) {
 2501           emit_opcode(cbuf, Assembler::REX_RB);
 2502         } else {
 2503           emit_opcode(cbuf, Assembler::REX_RXB);
 2504         }
 2505       }
 2506     }
 2507   %}
 2508 
 2509   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2510   %{
 2511     if ($reg$$reg < 8) {
 2512       if ($mem$$base < 8) {
 2513         if ($mem$$index < 8) {
 2514           emit_opcode(cbuf, Assembler::REX_W);
 2515         } else {
 2516           emit_opcode(cbuf, Assembler::REX_WX);
 2517         }
 2518       } else {
 2519         if ($mem$$index < 8) {
 2520           emit_opcode(cbuf, Assembler::REX_WB);
 2521         } else {
 2522           emit_opcode(cbuf, Assembler::REX_WXB);
 2523         }
 2524       }
 2525     } else {
 2526       if ($mem$$base < 8) {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WR);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WRX);
 2531         }
 2532       } else {
 2533         if ($mem$$index < 8) {
 2534           emit_opcode(cbuf, Assembler::REX_WRB);
 2535         } else {
 2536           emit_opcode(cbuf, Assembler::REX_WRXB);
 2537         }
 2538       }
 2539     }
 2540   %}
 2541 
 2542   enc_class reg_mem(rRegI ereg, memory mem)
 2543   %{
 2544     // High registers handle in encode_RegMem
 2545     int reg = $ereg$$reg;
 2546     int base = $mem$$base;
 2547     int index = $mem$$index;
 2548     int scale = $mem$$scale;
 2549     int disp = $mem$$disp;
 2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2551 
 2552     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2553   %}
 2554 
 2555   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2556   %{
 2557     int rm_byte_opcode = $rm_opcode$$constant;
 2558 
 2559     // High registers handle in encode_RegMem
 2560     int base = $mem$$base;
 2561     int index = $mem$$index;
 2562     int scale = $mem$$scale;
 2563     int displace = $mem$$disp;
 2564 
 2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2566                                             // working with static
 2567                                             // globals
 2568     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2569                   disp_reloc);
 2570   %}
 2571 
 2572   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2573   %{
 2574     int reg_encoding = $dst$$reg;
 2575     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2576     int index        = 0x04;            // 0x04 indicates no index
 2577     int scale        = 0x00;            // 0x00 indicates no scale
 2578     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2579     relocInfo::relocType disp_reloc = relocInfo::none;
 2580     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2581                   disp_reloc);
 2582   %}
 2583 
 2584   enc_class neg_reg(rRegI dst)
 2585   %{
 2586     int dstenc = $dst$$reg;
 2587     if (dstenc >= 8) {
 2588       emit_opcode(cbuf, Assembler::REX_B);
 2589       dstenc -= 8;
 2590     }
 2591     // NEG $dst
 2592     emit_opcode(cbuf, 0xF7);
 2593     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2594   %}
 2595 
 2596   enc_class neg_reg_wide(rRegI dst)
 2597   %{
 2598     int dstenc = $dst$$reg;
 2599     if (dstenc < 8) {
 2600       emit_opcode(cbuf, Assembler::REX_W);
 2601     } else {
 2602       emit_opcode(cbuf, Assembler::REX_WB);
 2603       dstenc -= 8;
 2604     }
 2605     // NEG $dst
 2606     emit_opcode(cbuf, 0xF7);
 2607     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2608   %}
 2609 
 2610   enc_class setLT_reg(rRegI dst)
 2611   %{
 2612     int dstenc = $dst$$reg;
 2613     if (dstenc >= 8) {
 2614       emit_opcode(cbuf, Assembler::REX_B);
 2615       dstenc -= 8;
 2616     } else if (dstenc >= 4) {
 2617       emit_opcode(cbuf, Assembler::REX);
 2618     }
 2619     // SETLT $dst
 2620     emit_opcode(cbuf, 0x0F);
 2621     emit_opcode(cbuf, 0x9C);
 2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2623   %}
 2624 
 2625   enc_class setNZ_reg(rRegI dst)
 2626   %{
 2627     int dstenc = $dst$$reg;
 2628     if (dstenc >= 8) {
 2629       emit_opcode(cbuf, Assembler::REX_B);
 2630       dstenc -= 8;
 2631     } else if (dstenc >= 4) {
 2632       emit_opcode(cbuf, Assembler::REX);
 2633     }
 2634     // SETNZ $dst
 2635     emit_opcode(cbuf, 0x0F);
 2636     emit_opcode(cbuf, 0x95);
 2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2638   %}
 2639 
 2640 
 2641   // Compare the lonogs and set -1, 0, or 1 into dst
 2642   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2643   %{
 2644     int src1enc = $src1$$reg;
 2645     int src2enc = $src2$$reg;
 2646     int dstenc = $dst$$reg;
 2647 
 2648     // cmpq $src1, $src2
 2649     if (src1enc < 8) {
 2650       if (src2enc < 8) {
 2651         emit_opcode(cbuf, Assembler::REX_W);
 2652       } else {
 2653         emit_opcode(cbuf, Assembler::REX_WB);
 2654       }
 2655     } else {
 2656       if (src2enc < 8) {
 2657         emit_opcode(cbuf, Assembler::REX_WR);
 2658       } else {
 2659         emit_opcode(cbuf, Assembler::REX_WRB);
 2660       }
 2661     }
 2662     emit_opcode(cbuf, 0x3B);
 2663     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2664 
 2665     // movl $dst, -1
 2666     if (dstenc >= 8) {
 2667       emit_opcode(cbuf, Assembler::REX_B);
 2668     }
 2669     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2670     emit_d32(cbuf, -1);
 2671 
 2672     // jl,s done
 2673     emit_opcode(cbuf, 0x7C);
 2674     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2675 
 2676     // setne $dst
 2677     if (dstenc >= 4) {
 2678       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2679     }
 2680     emit_opcode(cbuf, 0x0F);
 2681     emit_opcode(cbuf, 0x95);
 2682     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2683 
 2684     // movzbl $dst, $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0xB6);
 2690     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2691   %}
 2692 
 2693   enc_class Push_ResultXD(regD dst) %{
 2694     MacroAssembler _masm(&cbuf);
 2695     __ fstp_d(Address(rsp, 0));
 2696     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2697     __ addptr(rsp, 8);
 2698   %}
 2699 
 2700   enc_class Push_SrcXD(regD src) %{
 2701     MacroAssembler _masm(&cbuf);
 2702     __ subptr(rsp, 8);
 2703     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2704     __ fld_d(Address(rsp, 0));
 2705   %}
 2706 
 2707 
 2708   enc_class enc_rethrow()
 2709   %{
 2710     cbuf.set_insts_mark();
 2711     emit_opcode(cbuf, 0xE9); // jmp entry
 2712     emit_d32_reloc(cbuf,
 2713                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2714                    runtime_call_Relocation::spec(),
 2715                    RELOC_DISP32);
 2716   %}
 2717 
 2718 %}
 2719 
 2720 
 2721 
 2722 //----------FRAME--------------------------------------------------------------
 2723 // Definition of frame structure and management information.
 2724 //
 2725 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2726 //                             |   (to get allocators register number
 2727 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2728 //  r   CALLER     |        |
 2729 //  o     |        +--------+      pad to even-align allocators stack-slot
 2730 //  w     V        |  pad0  |        numbers; owned by CALLER
 2731 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2732 //  h     ^        |   in   |  5
 2733 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2734 //  |     |        |        |  3
 2735 //  |     |        +--------+
 2736 //  V     |        | old out|      Empty on Intel, window on Sparc
 2737 //        |    old |preserve|      Must be even aligned.
 2738 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2739 //        |        |   in   |  3   area for Intel ret address
 2740 //     Owned by    |preserve|      Empty on Sparc.
 2741 //       SELF      +--------+
 2742 //        |        |  pad2  |  2   pad to align old SP
 2743 //        |        +--------+  1
 2744 //        |        | locks  |  0
 2745 //        |        +--------+----> OptoReg::stack0(), even aligned
 2746 //        |        |  pad1  | 11   pad to align new SP
 2747 //        |        +--------+
 2748 //        |        |        | 10
 2749 //        |        | spills |  9   spills
 2750 //        V        |        |  8   (pad0 slot for callee)
 2751 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2752 //        ^        |  out   |  7
 2753 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2754 //     Owned by    +--------+
 2755 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2756 //        |    new |preserve|      Must be even-aligned.
 2757 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2758 //        |        |        |
 2759 //
 2760 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2761 //         known from SELF's arguments and the Java calling convention.
 2762 //         Region 6-7 is determined per call site.
 2763 // Note 2: If the calling convention leaves holes in the incoming argument
 2764 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2765 //         are owned by the CALLEE.  Holes should not be necessary in the
 2766 //         incoming area, as the Java calling convention is completely under
 2767 //         the control of the AD file.  Doubles can be sorted and packed to
 2768 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2769 //         varargs C calling conventions.
 2770 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2771 //         even aligned with pad0 as needed.
 2772 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2773 //         region 6-11 is even aligned; it may be padded out more so that
 2774 //         the region from SP to FP meets the minimum stack alignment.
 2775 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2776 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2777 //         SP meets the minimum alignment.
 2778 
 2779 frame
 2780 %{
 2781   // These three registers define part of the calling convention
 2782   // between compiled code and the interpreter.
 2783   inline_cache_reg(RAX);                // Inline Cache Register
 2784 
 2785   // Optional: name the operand used by cisc-spilling to access
 2786   // [stack_pointer + offset]
 2787   cisc_spilling_operand_name(indOffset32);
 2788 
 2789   // Number of stack slots consumed by locking an object
 2790   sync_stack_slots(2);
 2791 
 2792   // Compiled code's Frame Pointer
 2793   frame_pointer(RSP);
 2794 
 2795   // Interpreter stores its frame pointer in a register which is
 2796   // stored to the stack by I2CAdaptors.
 2797   // I2CAdaptors convert from interpreted java to compiled java.
 2798   interpreter_frame_pointer(RBP);
 2799 
 2800   // Stack alignment requirement
 2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2802 
 2803   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2804   // for calls to C.  Supports the var-args backing area for register parms.
 2805   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2806 
 2807   // The after-PROLOG location of the return address.  Location of
 2808   // return address specifies a type (REG or STACK) and a number
 2809   // representing the register number (i.e. - use a register name) or
 2810   // stack slot.
 2811   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2812   // Otherwise, it is above the locks and verification slot and alignment word
 2813   return_addr(STACK - 2 +
 2814               align_up((Compile::current()->in_preserve_stack_slots() +
 2815                         Compile::current()->fixed_slots()),
 2816                        stack_alignment_in_slots()));
 2817 
 2818   // Location of compiled Java return values.  Same as C for now.
 2819   return_value
 2820   %{
 2821     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2822            "only return normal values");
 2823 
 2824     static const int lo[Op_RegL + 1] = {
 2825       0,
 2826       0,
 2827       RAX_num,  // Op_RegN
 2828       RAX_num,  // Op_RegI
 2829       RAX_num,  // Op_RegP
 2830       XMM0_num, // Op_RegF
 2831       XMM0_num, // Op_RegD
 2832       RAX_num   // Op_RegL
 2833     };
 2834     static const int hi[Op_RegL + 1] = {
 2835       0,
 2836       0,
 2837       OptoReg::Bad, // Op_RegN
 2838       OptoReg::Bad, // Op_RegI
 2839       RAX_H_num,    // Op_RegP
 2840       OptoReg::Bad, // Op_RegF
 2841       XMM0b_num,    // Op_RegD
 2842       RAX_H_num     // Op_RegL
 2843     };
 2844     // Excluded flags and vector registers.
 2845     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2846     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2847   %}
 2848 %}
 2849 
 2850 //----------ATTRIBUTES---------------------------------------------------------
 2851 //----------Operand Attributes-------------------------------------------------
 2852 op_attrib op_cost(0);        // Required cost attribute
 2853 
 2854 //----------Instruction Attributes---------------------------------------------
 2855 ins_attrib ins_cost(100);       // Required cost attribute
 2856 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2857 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2858                                 // a non-matching short branch variant
 2859                                 // of some long branch?
 2860 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2861                                 // be a power of 2) specifies the
 2862                                 // alignment that some part of the
 2863                                 // instruction (not necessarily the
 2864                                 // start) requires.  If > 1, a
 2865                                 // compute_padding() function must be
 2866                                 // provided for the instruction
 2867 
 2868 //----------OPERANDS-----------------------------------------------------------
 2869 // Operand definitions must precede instruction definitions for correct parsing
 2870 // in the ADLC because operands constitute user defined types which are used in
 2871 // instruction definitions.
 2872 
 2873 //----------Simple Operands----------------------------------------------------
 2874 // Immediate Operands
 2875 // Integer Immediate
 2876 operand immI()
 2877 %{
 2878   match(ConI);
 2879 
 2880   op_cost(10);
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 // Constant for test vs zero
 2886 operand immI_0()
 2887 %{
 2888   predicate(n->get_int() == 0);
 2889   match(ConI);
 2890 
 2891   op_cost(0);
 2892   format %{ %}
 2893   interface(CONST_INTER);
 2894 %}
 2895 
 2896 // Constant for increment
 2897 operand immI_1()
 2898 %{
 2899   predicate(n->get_int() == 1);
 2900   match(ConI);
 2901 
 2902   op_cost(0);
 2903   format %{ %}
 2904   interface(CONST_INTER);
 2905 %}
 2906 
 2907 // Constant for decrement
 2908 operand immI_M1()
 2909 %{
 2910   predicate(n->get_int() == -1);
 2911   match(ConI);
 2912 
 2913   op_cost(0);
 2914   format %{ %}
 2915   interface(CONST_INTER);
 2916 %}
 2917 
 2918 operand immI_2()
 2919 %{
 2920   predicate(n->get_int() == 2);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 operand immI_4()
 2929 %{
 2930   predicate(n->get_int() == 4);
 2931   match(ConI);
 2932 
 2933   op_cost(0);
 2934   format %{ %}
 2935   interface(CONST_INTER);
 2936 %}
 2937 
 2938 operand immI_8()
 2939 %{
 2940   predicate(n->get_int() == 8);
 2941   match(ConI);
 2942 
 2943   op_cost(0);
 2944   format %{ %}
 2945   interface(CONST_INTER);
 2946 %}
 2947 
 2948 // Valid scale values for addressing modes
 2949 operand immI2()
 2950 %{
 2951   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2952   match(ConI);
 2953 
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 operand immU7()
 2959 %{
 2960   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2961   match(ConI);
 2962 
 2963   op_cost(5);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 operand immI8()
 2969 %{
 2970   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2971   match(ConI);
 2972 
 2973   op_cost(5);
 2974   format %{ %}
 2975   interface(CONST_INTER);
 2976 %}
 2977 
 2978 operand immU8()
 2979 %{
 2980   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2981   match(ConI);
 2982 
 2983   op_cost(5);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immI16()
 2989 %{
 2990   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2991   match(ConI);
 2992 
 2993   op_cost(10);
 2994   format %{ %}
 2995   interface(CONST_INTER);
 2996 %}
 2997 
 2998 // Int Immediate non-negative
 2999 operand immU31()
 3000 %{
 3001   predicate(n->get_int() >= 0);
 3002   match(ConI);
 3003 
 3004   op_cost(0);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 // Constant for long shifts
 3010 operand immI_32()
 3011 %{
 3012   predicate( n->get_int() == 32 );
 3013   match(ConI);
 3014 
 3015   op_cost(0);
 3016   format %{ %}
 3017   interface(CONST_INTER);
 3018 %}
 3019 
 3020 // Constant for long shifts
 3021 operand immI_64()
 3022 %{
 3023   predicate( n->get_int() == 64 );
 3024   match(ConI);
 3025 
 3026   op_cost(0);
 3027   format %{ %}
 3028   interface(CONST_INTER);
 3029 %}
 3030 
 3031 // Pointer Immediate
 3032 operand immP()
 3033 %{
 3034   match(ConP);
 3035 
 3036   op_cost(10);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // NULL Pointer Immediate
 3042 operand immP0()
 3043 %{
 3044   predicate(n->get_ptr() == 0);
 3045   match(ConP);
 3046 
 3047   op_cost(5);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immN() %{
 3054   match(ConN);
 3055 
 3056   op_cost(10);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immNKlass() %{
 3062   match(ConNKlass);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // NULL Pointer Immediate
 3070 operand immN0() %{
 3071   predicate(n->get_narrowcon() == 0);
 3072   match(ConN);
 3073 
 3074   op_cost(5);
 3075   format %{ %}
 3076   interface(CONST_INTER);
 3077 %}
 3078 
 3079 operand immP31()
 3080 %{
 3081   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3082             && (n->get_ptr() >> 31) == 0);
 3083   match(ConP);
 3084 
 3085   op_cost(5);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 
 3091 // Long Immediate
 3092 operand immL()
 3093 %{
 3094   match(ConL);
 3095 
 3096   op_cost(20);
 3097   format %{ %}
 3098   interface(CONST_INTER);
 3099 %}
 3100 
 3101 // Long Immediate 8-bit
 3102 operand immL8()
 3103 %{
 3104   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3105   match(ConL);
 3106 
 3107   op_cost(5);
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate 32-bit unsigned
 3113 operand immUL32()
 3114 %{
 3115   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3116   match(ConL);
 3117 
 3118   op_cost(10);
 3119   format %{ %}
 3120   interface(CONST_INTER);
 3121 %}
 3122 
 3123 // Long Immediate 32-bit signed
 3124 operand immL32()
 3125 %{
 3126   predicate(n->get_long() == (int) (n->get_long()));
 3127   match(ConL);
 3128 
 3129   op_cost(15);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 operand immL_Pow2()
 3135 %{
 3136   predicate(is_power_of_2((julong)n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(15);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 operand immL_NotPow2()
 3145 %{
 3146   predicate(is_power_of_2((julong)~n->get_long()));
 3147   match(ConL);
 3148 
 3149   op_cost(15);
 3150   format %{ %}
 3151   interface(CONST_INTER);
 3152 %}
 3153 
 3154 // Long Immediate zero
 3155 operand immL0()
 3156 %{
 3157   predicate(n->get_long() == 0L);
 3158   match(ConL);
 3159 
 3160   op_cost(10);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 // Constant for increment
 3166 operand immL1()
 3167 %{
 3168   predicate(n->get_long() == 1);
 3169   match(ConL);
 3170 
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Constant for decrement
 3176 operand immL_M1()
 3177 %{
 3178   predicate(n->get_long() == -1);
 3179   match(ConL);
 3180 
 3181   format %{ %}
 3182   interface(CONST_INTER);
 3183 %}
 3184 
 3185 // Long Immediate: the value 10
 3186 operand immL10()
 3187 %{
 3188   predicate(n->get_long() == 10);
 3189   match(ConL);
 3190 
 3191   format %{ %}
 3192   interface(CONST_INTER);
 3193 %}
 3194 
 3195 // Long immediate from 0 to 127.
 3196 // Used for a shorter form of long mul by 10.
 3197 operand immL_127()
 3198 %{
 3199   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3200   match(ConL);
 3201 
 3202   op_cost(10);
 3203   format %{ %}
 3204   interface(CONST_INTER);
 3205 %}
 3206 
 3207 // Long Immediate: low 32-bit mask
 3208 operand immL_32bits()
 3209 %{
 3210   predicate(n->get_long() == 0xFFFFFFFFL);
 3211   match(ConL);
 3212   op_cost(20);
 3213 
 3214   format %{ %}
 3215   interface(CONST_INTER);
 3216 %}
 3217 
 3218 // Int Immediate: 2^n-1, positive
 3219 operand immI_Pow2M1()
 3220 %{
 3221   predicate((n->get_int() > 0)
 3222             && is_power_of_2(n->get_int() + 1));
 3223   match(ConI);
 3224 
 3225   op_cost(20);
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Float Immediate zero
 3231 operand immF0()
 3232 %{
 3233   predicate(jint_cast(n->getf()) == 0);
 3234   match(ConF);
 3235 
 3236   op_cost(5);
 3237   format %{ %}
 3238   interface(CONST_INTER);
 3239 %}
 3240 
 3241 // Float Immediate
 3242 operand immF()
 3243 %{
 3244   match(ConF);
 3245 
 3246   op_cost(15);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Double Immediate zero
 3252 operand immD0()
 3253 %{
 3254   predicate(jlong_cast(n->getd()) == 0);
 3255   match(ConD);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Double Immediate
 3263 operand immD()
 3264 %{
 3265   match(ConD);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Immediates for special shifts (sign extend)
 3273 
 3274 // Constants for increment
 3275 operand immI_16()
 3276 %{
 3277   predicate(n->get_int() == 16);
 3278   match(ConI);
 3279 
 3280   format %{ %}
 3281   interface(CONST_INTER);
 3282 %}
 3283 
 3284 operand immI_24()
 3285 %{
 3286   predicate(n->get_int() == 24);
 3287   match(ConI);
 3288 
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Constant for byte-wide masking
 3294 operand immI_255()
 3295 %{
 3296   predicate(n->get_int() == 255);
 3297   match(ConI);
 3298 
 3299   format %{ %}
 3300   interface(CONST_INTER);
 3301 %}
 3302 
 3303 // Constant for short-wide masking
 3304 operand immI_65535()
 3305 %{
 3306   predicate(n->get_int() == 65535);
 3307   match(ConI);
 3308 
 3309   format %{ %}
 3310   interface(CONST_INTER);
 3311 %}
 3312 
 3313 // Constant for byte-wide masking
 3314 operand immL_255()
 3315 %{
 3316   predicate(n->get_long() == 255);
 3317   match(ConL);
 3318 
 3319   format %{ %}
 3320   interface(CONST_INTER);
 3321 %}
 3322 
 3323 // Constant for short-wide masking
 3324 operand immL_65535()
 3325 %{
 3326   predicate(n->get_long() == 65535);
 3327   match(ConL);
 3328 
 3329   format %{ %}
 3330   interface(CONST_INTER);
 3331 %}
 3332 
 3333 operand kReg()
 3334 %{
 3335   constraint(ALLOC_IN_RC(vectmask_reg));
 3336   match(RegVectMask);
 3337   format %{%}
 3338   interface(REG_INTER);
 3339 %}
 3340 
 3341 operand kReg_K1()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K2()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 // Special Registers
 3358 operand kReg_K3()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K4()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K5()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K6()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 // Special Registers
 3391 operand kReg_K7()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 // Register Operands
 3400 // Integer Register
 3401 operand rRegI()
 3402 %{
 3403   constraint(ALLOC_IN_RC(int_reg));
 3404   match(RegI);
 3405 
 3406   match(rax_RegI);
 3407   match(rbx_RegI);
 3408   match(rcx_RegI);
 3409   match(rdx_RegI);
 3410   match(rdi_RegI);
 3411 
 3412   format %{ %}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Special Registers
 3417 operand rax_RegI()
 3418 %{
 3419   constraint(ALLOC_IN_RC(int_rax_reg));
 3420   match(RegI);
 3421   match(rRegI);
 3422 
 3423   format %{ "RAX" %}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand rbx_RegI()
 3429 %{
 3430   constraint(ALLOC_IN_RC(int_rbx_reg));
 3431   match(RegI);
 3432   match(rRegI);
 3433 
 3434   format %{ "RBX" %}
 3435   interface(REG_INTER);
 3436 %}
 3437 
 3438 operand rcx_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rcx_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RCX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 operand rdx_RegI()
 3449 %{
 3450   constraint(ALLOC_IN_RC(int_rdx_reg));
 3451   match(RegI);
 3452   match(rRegI);
 3453 
 3454   format %{ "RDX" %}
 3455   interface(REG_INTER);
 3456 %}
 3457 
 3458 operand rdi_RegI()
 3459 %{
 3460   constraint(ALLOC_IN_RC(int_rdi_reg));
 3461   match(RegI);
 3462   match(rRegI);
 3463 
 3464   format %{ "RDI" %}
 3465   interface(REG_INTER);
 3466 %}
 3467 
 3468 operand no_rax_rdx_RegI()
 3469 %{
 3470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3471   match(RegI);
 3472   match(rbx_RegI);
 3473   match(rcx_RegI);
 3474   match(rdi_RegI);
 3475 
 3476   format %{ %}
 3477   interface(REG_INTER);
 3478 %}
 3479 
 3480 operand no_rbp_r13_RegI()
 3481 %{
 3482   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3483   match(RegI);
 3484   match(rRegI);
 3485   match(rax_RegI);
 3486   match(rbx_RegI);
 3487   match(rcx_RegI);
 3488   match(rdx_RegI);
 3489   match(rdi_RegI);
 3490 
 3491   format %{ %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 // Pointer Register
 3496 operand any_RegP()
 3497 %{
 3498   constraint(ALLOC_IN_RC(any_reg));
 3499   match(RegP);
 3500   match(rax_RegP);
 3501   match(rbx_RegP);
 3502   match(rdi_RegP);
 3503   match(rsi_RegP);
 3504   match(rbp_RegP);
 3505   match(r15_RegP);
 3506   match(rRegP);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 operand rRegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_reg));
 3515   match(RegP);
 3516   match(rax_RegP);
 3517   match(rbx_RegP);
 3518   match(rdi_RegP);
 3519   match(rsi_RegP);
 3520   match(rbp_RegP);  // See Q&A below about
 3521   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3522 
 3523   format %{ %}
 3524   interface(REG_INTER);
 3525 %}
 3526 
 3527 operand rRegN() %{
 3528   constraint(ALLOC_IN_RC(int_reg));
 3529   match(RegN);
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3536 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3537 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3538 // The output of an instruction is controlled by the allocator, which respects
 3539 // register class masks, not match rules.  Unless an instruction mentions
 3540 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3541 // by the allocator as an input.
 3542 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3543 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3544 // result, RBP is not included in the output of the instruction either.
 3545 
 3546 operand no_rax_RegP()
 3547 %{
 3548   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3549   match(RegP);
 3550   match(rbx_RegP);
 3551   match(rsi_RegP);
 3552   match(rdi_RegP);
 3553 
 3554   format %{ %}
 3555   interface(REG_INTER);
 3556 %}
 3557 
 3558 // This operand is not allowed to use RBP even if
 3559 // RBP is not used to hold the frame pointer.
 3560 operand no_rbp_RegP()
 3561 %{
 3562   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3563   match(RegP);
 3564   match(rbx_RegP);
 3565   match(rsi_RegP);
 3566   match(rdi_RegP);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 operand no_rax_rbx_RegP()
 3573 %{
 3574   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3575   match(RegP);
 3576   match(rsi_RegP);
 3577   match(rdi_RegP);
 3578 
 3579   format %{ %}
 3580   interface(REG_INTER);
 3581 %}
 3582 
 3583 // Special Registers
 3584 // Return a pointer value
 3585 operand rax_RegP()
 3586 %{
 3587   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3588   match(RegP);
 3589   match(rRegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // Special Registers
 3596 // Return a compressed pointer value
 3597 operand rax_RegN()
 3598 %{
 3599   constraint(ALLOC_IN_RC(int_rax_reg));
 3600   match(RegN);
 3601   match(rRegN);
 3602 
 3603   format %{ %}
 3604   interface(REG_INTER);
 3605 %}
 3606 
 3607 // Used in AtomicAdd
 3608 operand rbx_RegP()
 3609 %{
 3610   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3611   match(RegP);
 3612   match(rRegP);
 3613 
 3614   format %{ %}
 3615   interface(REG_INTER);
 3616 %}
 3617 
 3618 operand rsi_RegP()
 3619 %{
 3620   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3621   match(RegP);
 3622   match(rRegP);
 3623 
 3624   format %{ %}
 3625   interface(REG_INTER);
 3626 %}
 3627 
 3628 operand rbp_RegP()
 3629 %{
 3630   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3631   match(RegP);
 3632   match(rRegP);
 3633 
 3634   format %{ %}
 3635   interface(REG_INTER);
 3636 %}
 3637 
 3638 // Used in rep stosq
 3639 operand rdi_RegP()
 3640 %{
 3641   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3642   match(RegP);
 3643   match(rRegP);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand r15_RegP()
 3650 %{
 3651   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3652   match(RegP);
 3653   match(rRegP);
 3654 
 3655   format %{ %}
 3656   interface(REG_INTER);
 3657 %}
 3658 
 3659 operand rRegL()
 3660 %{
 3661   constraint(ALLOC_IN_RC(long_reg));
 3662   match(RegL);
 3663   match(rax_RegL);
 3664   match(rdx_RegL);
 3665 
 3666   format %{ %}
 3667   interface(REG_INTER);
 3668 %}
 3669 
 3670 // Special Registers
 3671 operand no_rax_rdx_RegL()
 3672 %{
 3673   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3674   match(RegL);
 3675   match(rRegL);
 3676 
 3677   format %{ %}
 3678   interface(REG_INTER);
 3679 %}
 3680 
 3681 operand no_rax_RegL()
 3682 %{
 3683   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3684   match(RegL);
 3685   match(rRegL);
 3686   match(rdx_RegL);
 3687 
 3688   format %{ %}
 3689   interface(REG_INTER);
 3690 %}
 3691 
 3692 operand rax_RegL()
 3693 %{
 3694   constraint(ALLOC_IN_RC(long_rax_reg));
 3695   match(RegL);
 3696   match(rRegL);
 3697 
 3698   format %{ "RAX" %}
 3699   interface(REG_INTER);
 3700 %}
 3701 
 3702 operand rcx_RegL()
 3703 %{
 3704   constraint(ALLOC_IN_RC(long_rcx_reg));
 3705   match(RegL);
 3706   match(rRegL);
 3707 
 3708   format %{ %}
 3709   interface(REG_INTER);
 3710 %}
 3711 
 3712 operand rdx_RegL()
 3713 %{
 3714   constraint(ALLOC_IN_RC(long_rdx_reg));
 3715   match(RegL);
 3716   match(rRegL);
 3717 
 3718   format %{ %}
 3719   interface(REG_INTER);
 3720 %}
 3721 
 3722 operand no_rbp_r13_RegL()
 3723 %{
 3724   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3725   match(RegL);
 3726   match(rRegL);
 3727   match(rax_RegL);
 3728   match(rcx_RegL);
 3729   match(rdx_RegL);
 3730 
 3731   format %{ %}
 3732   interface(REG_INTER);
 3733 %}
 3734 
 3735 // Flags register, used as output of compare instructions
 3736 operand rFlagsReg()
 3737 %{
 3738   constraint(ALLOC_IN_RC(int_flags));
 3739   match(RegFlags);
 3740 
 3741   format %{ "RFLAGS" %}
 3742   interface(REG_INTER);
 3743 %}
 3744 
 3745 // Flags register, used as output of FLOATING POINT compare instructions
 3746 operand rFlagsRegU()
 3747 %{
 3748   constraint(ALLOC_IN_RC(int_flags));
 3749   match(RegFlags);
 3750 
 3751   format %{ "RFLAGS_U" %}
 3752   interface(REG_INTER);
 3753 %}
 3754 
 3755 operand rFlagsRegUCF() %{
 3756   constraint(ALLOC_IN_RC(int_flags));
 3757   match(RegFlags);
 3758   predicate(false);
 3759 
 3760   format %{ "RFLAGS_U_CF" %}
 3761   interface(REG_INTER);
 3762 %}
 3763 
 3764 // Float register operands
 3765 operand regF() %{
 3766    constraint(ALLOC_IN_RC(float_reg));
 3767    match(RegF);
 3768 
 3769    format %{ %}
 3770    interface(REG_INTER);
 3771 %}
 3772 
 3773 // Float register operands
 3774 operand legRegF() %{
 3775    constraint(ALLOC_IN_RC(float_reg_legacy));
 3776    match(RegF);
 3777 
 3778    format %{ %}
 3779    interface(REG_INTER);
 3780 %}
 3781 
 3782 // Float register operands
 3783 operand vlRegF() %{
 3784    constraint(ALLOC_IN_RC(float_reg_vl));
 3785    match(RegF);
 3786 
 3787    format %{ %}
 3788    interface(REG_INTER);
 3789 %}
 3790 
 3791 // Double register operands
 3792 operand regD() %{
 3793    constraint(ALLOC_IN_RC(double_reg));
 3794    match(RegD);
 3795 
 3796    format %{ %}
 3797    interface(REG_INTER);
 3798 %}
 3799 
 3800 // Double register operands
 3801 operand legRegD() %{
 3802    constraint(ALLOC_IN_RC(double_reg_legacy));
 3803    match(RegD);
 3804 
 3805    format %{ %}
 3806    interface(REG_INTER);
 3807 %}
 3808 
 3809 // Double register operands
 3810 operand vlRegD() %{
 3811    constraint(ALLOC_IN_RC(double_reg_vl));
 3812    match(RegD);
 3813 
 3814    format %{ %}
 3815    interface(REG_INTER);
 3816 %}
 3817 
 3818 //----------Memory Operands----------------------------------------------------
 3819 // Direct Memory Operand
 3820 // operand direct(immP addr)
 3821 // %{
 3822 //   match(addr);
 3823 
 3824 //   format %{ "[$addr]" %}
 3825 //   interface(MEMORY_INTER) %{
 3826 //     base(0xFFFFFFFF);
 3827 //     index(0x4);
 3828 //     scale(0x0);
 3829 //     disp($addr);
 3830 //   %}
 3831 // %}
 3832 
 3833 // Indirect Memory Operand
 3834 operand indirect(any_RegP reg)
 3835 %{
 3836   constraint(ALLOC_IN_RC(ptr_reg));
 3837   match(reg);
 3838 
 3839   format %{ "[$reg]" %}
 3840   interface(MEMORY_INTER) %{
 3841     base($reg);
 3842     index(0x4);
 3843     scale(0x0);
 3844     disp(0x0);
 3845   %}
 3846 %}
 3847 
 3848 // Indirect Memory Plus Short Offset Operand
 3849 operand indOffset8(any_RegP reg, immL8 off)
 3850 %{
 3851   constraint(ALLOC_IN_RC(ptr_reg));
 3852   match(AddP reg off);
 3853 
 3854   format %{ "[$reg + $off (8-bit)]" %}
 3855   interface(MEMORY_INTER) %{
 3856     base($reg);
 3857     index(0x4);
 3858     scale(0x0);
 3859     disp($off);
 3860   %}
 3861 %}
 3862 
 3863 // Indirect Memory Plus Long Offset Operand
 3864 operand indOffset32(any_RegP reg, immL32 off)
 3865 %{
 3866   constraint(ALLOC_IN_RC(ptr_reg));
 3867   match(AddP reg off);
 3868 
 3869   format %{ "[$reg + $off (32-bit)]" %}
 3870   interface(MEMORY_INTER) %{
 3871     base($reg);
 3872     index(0x4);
 3873     scale(0x0);
 3874     disp($off);
 3875   %}
 3876 %}
 3877 
 3878 // Indirect Memory Plus Index Register Plus Offset Operand
 3879 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3880 %{
 3881   constraint(ALLOC_IN_RC(ptr_reg));
 3882   match(AddP (AddP reg lreg) off);
 3883 
 3884   op_cost(10);
 3885   format %{"[$reg + $off + $lreg]" %}
 3886   interface(MEMORY_INTER) %{
 3887     base($reg);
 3888     index($lreg);
 3889     scale(0x0);
 3890     disp($off);
 3891   %}
 3892 %}
 3893 
 3894 // Indirect Memory Plus Index Register Plus Offset Operand
 3895 operand indIndex(any_RegP reg, rRegL lreg)
 3896 %{
 3897   constraint(ALLOC_IN_RC(ptr_reg));
 3898   match(AddP reg lreg);
 3899 
 3900   op_cost(10);
 3901   format %{"[$reg + $lreg]" %}
 3902   interface(MEMORY_INTER) %{
 3903     base($reg);
 3904     index($lreg);
 3905     scale(0x0);
 3906     disp(0x0);
 3907   %}
 3908 %}
 3909 
 3910 // Indirect Memory Times Scale Plus Index Register
 3911 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3912 %{
 3913   constraint(ALLOC_IN_RC(ptr_reg));
 3914   match(AddP reg (LShiftL lreg scale));
 3915 
 3916   op_cost(10);
 3917   format %{"[$reg + $lreg << $scale]" %}
 3918   interface(MEMORY_INTER) %{
 3919     base($reg);
 3920     index($lreg);
 3921     scale($scale);
 3922     disp(0x0);
 3923   %}
 3924 %}
 3925 
 3926 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3927 %{
 3928   constraint(ALLOC_IN_RC(ptr_reg));
 3929   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3930   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3931 
 3932   op_cost(10);
 3933   format %{"[$reg + pos $idx << $scale]" %}
 3934   interface(MEMORY_INTER) %{
 3935     base($reg);
 3936     index($idx);
 3937     scale($scale);
 3938     disp(0x0);
 3939   %}
 3940 %}
 3941 
 3942 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3943 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3944 %{
 3945   constraint(ALLOC_IN_RC(ptr_reg));
 3946   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3947 
 3948   op_cost(10);
 3949   format %{"[$reg + $off + $lreg << $scale]" %}
 3950   interface(MEMORY_INTER) %{
 3951     base($reg);
 3952     index($lreg);
 3953     scale($scale);
 3954     disp($off);
 3955   %}
 3956 %}
 3957 
 3958 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3959 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3960 %{
 3961   constraint(ALLOC_IN_RC(ptr_reg));
 3962   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3963   match(AddP (AddP reg (ConvI2L idx)) off);
 3964 
 3965   op_cost(10);
 3966   format %{"[$reg + $off + $idx]" %}
 3967   interface(MEMORY_INTER) %{
 3968     base($reg);
 3969     index($idx);
 3970     scale(0x0);
 3971     disp($off);
 3972   %}
 3973 %}
 3974 
 3975 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3976 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3977 %{
 3978   constraint(ALLOC_IN_RC(ptr_reg));
 3979   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3980   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3981 
 3982   op_cost(10);
 3983   format %{"[$reg + $off + $idx << $scale]" %}
 3984   interface(MEMORY_INTER) %{
 3985     base($reg);
 3986     index($idx);
 3987     scale($scale);
 3988     disp($off);
 3989   %}
 3990 %}
 3991 
 3992 // Indirect Narrow Oop Plus Offset Operand
 3993 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3994 // we can't free r12 even with CompressedOops::base() == NULL.
 3995 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3996   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3997   constraint(ALLOC_IN_RC(ptr_reg));
 3998   match(AddP (DecodeN reg) off);
 3999 
 4000   op_cost(10);
 4001   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 4002   interface(MEMORY_INTER) %{
 4003     base(0xc); // R12
 4004     index($reg);
 4005     scale(0x3);
 4006     disp($off);
 4007   %}
 4008 %}
 4009 
 4010 // Indirect Memory Operand
 4011 operand indirectNarrow(rRegN reg)
 4012 %{
 4013   predicate(CompressedOops::shift() == 0);
 4014   constraint(ALLOC_IN_RC(ptr_reg));
 4015   match(DecodeN reg);
 4016 
 4017   format %{ "[$reg]" %}
 4018   interface(MEMORY_INTER) %{
 4019     base($reg);
 4020     index(0x4);
 4021     scale(0x0);
 4022     disp(0x0);
 4023   %}
 4024 %}
 4025 
 4026 // Indirect Memory Plus Short Offset Operand
 4027 operand indOffset8Narrow(rRegN reg, immL8 off)
 4028 %{
 4029   predicate(CompressedOops::shift() == 0);
 4030   constraint(ALLOC_IN_RC(ptr_reg));
 4031   match(AddP (DecodeN reg) off);
 4032 
 4033   format %{ "[$reg + $off (8-bit)]" %}
 4034   interface(MEMORY_INTER) %{
 4035     base($reg);
 4036     index(0x4);
 4037     scale(0x0);
 4038     disp($off);
 4039   %}
 4040 %}
 4041 
 4042 // Indirect Memory Plus Long Offset Operand
 4043 operand indOffset32Narrow(rRegN reg, immL32 off)
 4044 %{
 4045   predicate(CompressedOops::shift() == 0);
 4046   constraint(ALLOC_IN_RC(ptr_reg));
 4047   match(AddP (DecodeN reg) off);
 4048 
 4049   format %{ "[$reg + $off (32-bit)]" %}
 4050   interface(MEMORY_INTER) %{
 4051     base($reg);
 4052     index(0x4);
 4053     scale(0x0);
 4054     disp($off);
 4055   %}
 4056 %}
 4057 
 4058 // Indirect Memory Plus Index Register Plus Offset Operand
 4059 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4060 %{
 4061   predicate(CompressedOops::shift() == 0);
 4062   constraint(ALLOC_IN_RC(ptr_reg));
 4063   match(AddP (AddP (DecodeN reg) lreg) off);
 4064 
 4065   op_cost(10);
 4066   format %{"[$reg + $off + $lreg]" %}
 4067   interface(MEMORY_INTER) %{
 4068     base($reg);
 4069     index($lreg);
 4070     scale(0x0);
 4071     disp($off);
 4072   %}
 4073 %}
 4074 
 4075 // Indirect Memory Plus Index Register Plus Offset Operand
 4076 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4077 %{
 4078   predicate(CompressedOops::shift() == 0);
 4079   constraint(ALLOC_IN_RC(ptr_reg));
 4080   match(AddP (DecodeN reg) lreg);
 4081 
 4082   op_cost(10);
 4083   format %{"[$reg + $lreg]" %}
 4084   interface(MEMORY_INTER) %{
 4085     base($reg);
 4086     index($lreg);
 4087     scale(0x0);
 4088     disp(0x0);
 4089   %}
 4090 %}
 4091 
 4092 // Indirect Memory Times Scale Plus Index Register
 4093 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4094 %{
 4095   predicate(CompressedOops::shift() == 0);
 4096   constraint(ALLOC_IN_RC(ptr_reg));
 4097   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4098 
 4099   op_cost(10);
 4100   format %{"[$reg + $lreg << $scale]" %}
 4101   interface(MEMORY_INTER) %{
 4102     base($reg);
 4103     index($lreg);
 4104     scale($scale);
 4105     disp(0x0);
 4106   %}
 4107 %}
 4108 
 4109 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4110 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4111 %{
 4112   predicate(CompressedOops::shift() == 0);
 4113   constraint(ALLOC_IN_RC(ptr_reg));
 4114   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4115 
 4116   op_cost(10);
 4117   format %{"[$reg + $off + $lreg << $scale]" %}
 4118   interface(MEMORY_INTER) %{
 4119     base($reg);
 4120     index($lreg);
 4121     scale($scale);
 4122     disp($off);
 4123   %}
 4124 %}
 4125 
 4126 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4127 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4128 %{
 4129   constraint(ALLOC_IN_RC(ptr_reg));
 4130   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4131   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4132 
 4133   op_cost(10);
 4134   format %{"[$reg + $off + $idx]" %}
 4135   interface(MEMORY_INTER) %{
 4136     base($reg);
 4137     index($idx);
 4138     scale(0x0);
 4139     disp($off);
 4140   %}
 4141 %}
 4142 
 4143 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4144 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4145 %{
 4146   constraint(ALLOC_IN_RC(ptr_reg));
 4147   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4148   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4149 
 4150   op_cost(10);
 4151   format %{"[$reg + $off + $idx << $scale]" %}
 4152   interface(MEMORY_INTER) %{
 4153     base($reg);
 4154     index($idx);
 4155     scale($scale);
 4156     disp($off);
 4157   %}
 4158 %}
 4159 
 4160 //----------Special Memory Operands--------------------------------------------
 4161 // Stack Slot Operand - This operand is used for loading and storing temporary
 4162 //                      values on the stack where a match requires a value to
 4163 //                      flow through memory.
 4164 operand stackSlotP(sRegP reg)
 4165 %{
 4166   constraint(ALLOC_IN_RC(stack_slots));
 4167   // No match rule because this operand is only generated in matching
 4168 
 4169   format %{ "[$reg]" %}
 4170   interface(MEMORY_INTER) %{
 4171     base(0x4);   // RSP
 4172     index(0x4);  // No Index
 4173     scale(0x0);  // No Scale
 4174     disp($reg);  // Stack Offset
 4175   %}
 4176 %}
 4177 
 4178 operand stackSlotI(sRegI reg)
 4179 %{
 4180   constraint(ALLOC_IN_RC(stack_slots));
 4181   // No match rule because this operand is only generated in matching
 4182 
 4183   format %{ "[$reg]" %}
 4184   interface(MEMORY_INTER) %{
 4185     base(0x4);   // RSP
 4186     index(0x4);  // No Index
 4187     scale(0x0);  // No Scale
 4188     disp($reg);  // Stack Offset
 4189   %}
 4190 %}
 4191 
 4192 operand stackSlotF(sRegF reg)
 4193 %{
 4194   constraint(ALLOC_IN_RC(stack_slots));
 4195   // No match rule because this operand is only generated in matching
 4196 
 4197   format %{ "[$reg]" %}
 4198   interface(MEMORY_INTER) %{
 4199     base(0x4);   // RSP
 4200     index(0x4);  // No Index
 4201     scale(0x0);  // No Scale
 4202     disp($reg);  // Stack Offset
 4203   %}
 4204 %}
 4205 
 4206 operand stackSlotD(sRegD reg)
 4207 %{
 4208   constraint(ALLOC_IN_RC(stack_slots));
 4209   // No match rule because this operand is only generated in matching
 4210 
 4211   format %{ "[$reg]" %}
 4212   interface(MEMORY_INTER) %{
 4213     base(0x4);   // RSP
 4214     index(0x4);  // No Index
 4215     scale(0x0);  // No Scale
 4216     disp($reg);  // Stack Offset
 4217   %}
 4218 %}
 4219 operand stackSlotL(sRegL reg)
 4220 %{
 4221   constraint(ALLOC_IN_RC(stack_slots));
 4222   // No match rule because this operand is only generated in matching
 4223 
 4224   format %{ "[$reg]" %}
 4225   interface(MEMORY_INTER) %{
 4226     base(0x4);   // RSP
 4227     index(0x4);  // No Index
 4228     scale(0x0);  // No Scale
 4229     disp($reg);  // Stack Offset
 4230   %}
 4231 %}
 4232 
 4233 //----------Conditional Branch Operands----------------------------------------
 4234 // Comparison Op  - This is the operation of the comparison, and is limited to
 4235 //                  the following set of codes:
 4236 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4237 //
 4238 // Other attributes of the comparison, such as unsignedness, are specified
 4239 // by the comparison instruction that sets a condition code flags register.
 4240 // That result is represented by a flags operand whose subtype is appropriate
 4241 // to the unsignedness (etc.) of the comparison.
 4242 //
 4243 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4244 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4245 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4246 
 4247 // Comparison Code
 4248 operand cmpOp()
 4249 %{
 4250   match(Bool);
 4251 
 4252   format %{ "" %}
 4253   interface(COND_INTER) %{
 4254     equal(0x4, "e");
 4255     not_equal(0x5, "ne");
 4256     less(0xC, "l");
 4257     greater_equal(0xD, "ge");
 4258     less_equal(0xE, "le");
 4259     greater(0xF, "g");
 4260     overflow(0x0, "o");
 4261     no_overflow(0x1, "no");
 4262   %}
 4263 %}
 4264 
 4265 // Comparison Code, unsigned compare.  Used by FP also, with
 4266 // C2 (unordered) turned into GT or LT already.  The other bits
 4267 // C0 and C3 are turned into Carry & Zero flags.
 4268 operand cmpOpU()
 4269 %{
 4270   match(Bool);
 4271 
 4272   format %{ "" %}
 4273   interface(COND_INTER) %{
 4274     equal(0x4, "e");
 4275     not_equal(0x5, "ne");
 4276     less(0x2, "b");
 4277     greater_equal(0x3, "ae");
 4278     less_equal(0x6, "be");
 4279     greater(0x7, "a");
 4280     overflow(0x0, "o");
 4281     no_overflow(0x1, "no");
 4282   %}
 4283 %}
 4284 
 4285 
 4286 // Floating comparisons that don't require any fixup for the unordered case,
 4287 // If both inputs of the comparison are the same, ZF is always set so we
 4288 // don't need to use cmpOpUCF2 for eq/ne
 4289 operand cmpOpUCF() %{
 4290   match(Bool);
 4291   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4292             n->as_Bool()->_test._test == BoolTest::ge ||
 4293             n->as_Bool()->_test._test == BoolTest::le ||
 4294             n->as_Bool()->_test._test == BoolTest::gt ||
 4295             n->in(1)->in(1) == n->in(1)->in(2));
 4296   format %{ "" %}
 4297   interface(COND_INTER) %{
 4298     equal(0xb, "np");
 4299     not_equal(0xa, "p");
 4300     less(0x2, "b");
 4301     greater_equal(0x3, "ae");
 4302     less_equal(0x6, "be");
 4303     greater(0x7, "a");
 4304     overflow(0x0, "o");
 4305     no_overflow(0x1, "no");
 4306   %}
 4307 %}
 4308 
 4309 
 4310 // Floating comparisons that can be fixed up with extra conditional jumps
 4311 operand cmpOpUCF2() %{
 4312   match(Bool);
 4313   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4314              n->as_Bool()->_test._test == BoolTest::eq) &&
 4315             n->in(1)->in(1) != n->in(1)->in(2));
 4316   format %{ "" %}
 4317   interface(COND_INTER) %{
 4318     equal(0x4, "e");
 4319     not_equal(0x5, "ne");
 4320     less(0x2, "b");
 4321     greater_equal(0x3, "ae");
 4322     less_equal(0x6, "be");
 4323     greater(0x7, "a");
 4324     overflow(0x0, "o");
 4325     no_overflow(0x1, "no");
 4326   %}
 4327 %}
 4328 
 4329 //----------OPERAND CLASSES----------------------------------------------------
 4330 // Operand Classes are groups of operands that are used as to simplify
 4331 // instruction definitions by not requiring the AD writer to specify separate
 4332 // instructions for every form of operand when the instruction accepts
 4333 // multiple operand types with the same basic encoding and format.  The classic
 4334 // case of this is memory operands.
 4335 
 4336 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4337                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4338                indCompressedOopOffset,
 4339                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4340                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4341                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4342 
 4343 //----------PIPELINE-----------------------------------------------------------
 4344 // Rules which define the behavior of the target architectures pipeline.
 4345 pipeline %{
 4346 
 4347 //----------ATTRIBUTES---------------------------------------------------------
 4348 attributes %{
 4349   variable_size_instructions;        // Fixed size instructions
 4350   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4351   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4352   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4353   instruction_fetch_units = 1;       // of 16 bytes
 4354 
 4355   // List of nop instructions
 4356   nops( MachNop );
 4357 %}
 4358 
 4359 //----------RESOURCES----------------------------------------------------------
 4360 // Resources are the functional units available to the machine
 4361 
 4362 // Generic P2/P3 pipeline
 4363 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4364 // 3 instructions decoded per cycle.
 4365 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4366 // 3 ALU op, only ALU0 handles mul instructions.
 4367 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4368            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4369            BR, FPU,
 4370            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4371 
 4372 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4373 // Pipeline Description specifies the stages in the machine's pipeline
 4374 
 4375 // Generic P2/P3 pipeline
 4376 pipe_desc(S0, S1, S2, S3, S4, S5);
 4377 
 4378 //----------PIPELINE CLASSES---------------------------------------------------
 4379 // Pipeline Classes describe the stages in which input and output are
 4380 // referenced by the hardware pipeline.
 4381 
 4382 // Naming convention: ialu or fpu
 4383 // Then: _reg
 4384 // Then: _reg if there is a 2nd register
 4385 // Then: _long if it's a pair of instructions implementing a long
 4386 // Then: _fat if it requires the big decoder
 4387 //   Or: _mem if it requires the big decoder and a memory unit.
 4388 
 4389 // Integer ALU reg operation
 4390 pipe_class ialu_reg(rRegI dst)
 4391 %{
 4392     single_instruction;
 4393     dst    : S4(write);
 4394     dst    : S3(read);
 4395     DECODE : S0;        // any decoder
 4396     ALU    : S3;        // any alu
 4397 %}
 4398 
 4399 // Long ALU reg operation
 4400 pipe_class ialu_reg_long(rRegL dst)
 4401 %{
 4402     instruction_count(2);
 4403     dst    : S4(write);
 4404     dst    : S3(read);
 4405     DECODE : S0(2);     // any 2 decoders
 4406     ALU    : S3(2);     // both alus
 4407 %}
 4408 
 4409 // Integer ALU reg operation using big decoder
 4410 pipe_class ialu_reg_fat(rRegI dst)
 4411 %{
 4412     single_instruction;
 4413     dst    : S4(write);
 4414     dst    : S3(read);
 4415     D0     : S0;        // big decoder only
 4416     ALU    : S3;        // any alu
 4417 %}
 4418 
 4419 // Integer ALU reg-reg operation
 4420 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4421 %{
 4422     single_instruction;
 4423     dst    : S4(write);
 4424     src    : S3(read);
 4425     DECODE : S0;        // any decoder
 4426     ALU    : S3;        // any alu
 4427 %}
 4428 
 4429 // Integer ALU reg-reg operation
 4430 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4431 %{
 4432     single_instruction;
 4433     dst    : S4(write);
 4434     src    : S3(read);
 4435     D0     : S0;        // big decoder only
 4436     ALU    : S3;        // any alu
 4437 %}
 4438 
 4439 // Integer ALU reg-mem operation
 4440 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4441 %{
 4442     single_instruction;
 4443     dst    : S5(write);
 4444     mem    : S3(read);
 4445     D0     : S0;        // big decoder only
 4446     ALU    : S4;        // any alu
 4447     MEM    : S3;        // any mem
 4448 %}
 4449 
 4450 // Integer mem operation (prefetch)
 4451 pipe_class ialu_mem(memory mem)
 4452 %{
 4453     single_instruction;
 4454     mem    : S3(read);
 4455     D0     : S0;        // big decoder only
 4456     MEM    : S3;        // any mem
 4457 %}
 4458 
 4459 // Integer Store to Memory
 4460 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4461 %{
 4462     single_instruction;
 4463     mem    : S3(read);
 4464     src    : S5(read);
 4465     D0     : S0;        // big decoder only
 4466     ALU    : S4;        // any alu
 4467     MEM    : S3;
 4468 %}
 4469 
 4470 // // Long Store to Memory
 4471 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4472 // %{
 4473 //     instruction_count(2);
 4474 //     mem    : S3(read);
 4475 //     src    : S5(read);
 4476 //     D0     : S0(2);          // big decoder only; twice
 4477 //     ALU    : S4(2);     // any 2 alus
 4478 //     MEM    : S3(2);  // Both mems
 4479 // %}
 4480 
 4481 // Integer Store to Memory
 4482 pipe_class ialu_mem_imm(memory mem)
 4483 %{
 4484     single_instruction;
 4485     mem    : S3(read);
 4486     D0     : S0;        // big decoder only
 4487     ALU    : S4;        // any alu
 4488     MEM    : S3;
 4489 %}
 4490 
 4491 // Integer ALU0 reg-reg operation
 4492 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4493 %{
 4494     single_instruction;
 4495     dst    : S4(write);
 4496     src    : S3(read);
 4497     D0     : S0;        // Big decoder only
 4498     ALU0   : S3;        // only alu0
 4499 %}
 4500 
 4501 // Integer ALU0 reg-mem operation
 4502 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4503 %{
 4504     single_instruction;
 4505     dst    : S5(write);
 4506     mem    : S3(read);
 4507     D0     : S0;        // big decoder only
 4508     ALU0   : S4;        // ALU0 only
 4509     MEM    : S3;        // any mem
 4510 %}
 4511 
 4512 // Integer ALU reg-reg operation
 4513 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     src2   : S3(read);
 4519     DECODE : S0;        // any decoder
 4520     ALU    : S3;        // any alu
 4521 %}
 4522 
 4523 // Integer ALU reg-imm operation
 4524 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4525 %{
 4526     single_instruction;
 4527     cr     : S4(write);
 4528     src1   : S3(read);
 4529     DECODE : S0;        // any decoder
 4530     ALU    : S3;        // any alu
 4531 %}
 4532 
 4533 // Integer ALU reg-mem operation
 4534 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4535 %{
 4536     single_instruction;
 4537     cr     : S4(write);
 4538     src1   : S3(read);
 4539     src2   : S3(read);
 4540     D0     : S0;        // big decoder only
 4541     ALU    : S4;        // any alu
 4542     MEM    : S3;
 4543 %}
 4544 
 4545 // Conditional move reg-reg
 4546 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4547 %{
 4548     instruction_count(4);
 4549     y      : S4(read);
 4550     q      : S3(read);
 4551     p      : S3(read);
 4552     DECODE : S0(4);     // any decoder
 4553 %}
 4554 
 4555 // Conditional move reg-reg
 4556 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4557 %{
 4558     single_instruction;
 4559     dst    : S4(write);
 4560     src    : S3(read);
 4561     cr     : S3(read);
 4562     DECODE : S0;        // any decoder
 4563 %}
 4564 
 4565 // Conditional move reg-mem
 4566 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0;        // any decoder
 4573     MEM    : S3;
 4574 %}
 4575 
 4576 // Conditional move reg-reg long
 4577 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4578 %{
 4579     single_instruction;
 4580     dst    : S4(write);
 4581     src    : S3(read);
 4582     cr     : S3(read);
 4583     DECODE : S0(2);     // any 2 decoders
 4584 %}
 4585 
 4586 // XXX
 4587 // // Conditional move double reg-reg
 4588 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4589 // %{
 4590 //     single_instruction;
 4591 //     dst    : S4(write);
 4592 //     src    : S3(read);
 4593 //     cr     : S3(read);
 4594 //     DECODE : S0;     // any decoder
 4595 // %}
 4596 
 4597 // Float reg-reg operation
 4598 pipe_class fpu_reg(regD dst)
 4599 %{
 4600     instruction_count(2);
 4601     dst    : S3(read);
 4602     DECODE : S0(2);     // any 2 decoders
 4603     FPU    : S3;
 4604 %}
 4605 
 4606 // Float reg-reg operation
 4607 pipe_class fpu_reg_reg(regD dst, regD src)
 4608 %{
 4609     instruction_count(2);
 4610     dst    : S4(write);
 4611     src    : S3(read);
 4612     DECODE : S0(2);     // any 2 decoders
 4613     FPU    : S3;
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4618 %{
 4619     instruction_count(3);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     DECODE : S0(3);     // any 3 decoders
 4624     FPU    : S3(2);
 4625 %}
 4626 
 4627 // Float reg-reg operation
 4628 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4629 %{
 4630     instruction_count(4);
 4631     dst    : S4(write);
 4632     src1   : S3(read);
 4633     src2   : S3(read);
 4634     src3   : S3(read);
 4635     DECODE : S0(4);     // any 3 decoders
 4636     FPU    : S3(2);
 4637 %}
 4638 
 4639 // Float reg-reg operation
 4640 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4641 %{
 4642     instruction_count(4);
 4643     dst    : S4(write);
 4644     src1   : S3(read);
 4645     src2   : S3(read);
 4646     src3   : S3(read);
 4647     DECODE : S1(3);     // any 3 decoders
 4648     D0     : S0;        // Big decoder only
 4649     FPU    : S3(2);
 4650     MEM    : S3;
 4651 %}
 4652 
 4653 // Float reg-mem operation
 4654 pipe_class fpu_reg_mem(regD dst, memory mem)
 4655 %{
 4656     instruction_count(2);
 4657     dst    : S5(write);
 4658     mem    : S3(read);
 4659     D0     : S0;        // big decoder only
 4660     DECODE : S1;        // any decoder for FPU POP
 4661     FPU    : S4;
 4662     MEM    : S3;        // any mem
 4663 %}
 4664 
 4665 // Float reg-mem operation
 4666 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4667 %{
 4668     instruction_count(3);
 4669     dst    : S5(write);
 4670     src1   : S3(read);
 4671     mem    : S3(read);
 4672     D0     : S0;        // big decoder only
 4673     DECODE : S1(2);     // any decoder for FPU POP
 4674     FPU    : S4;
 4675     MEM    : S3;        // any mem
 4676 %}
 4677 
 4678 // Float mem-reg operation
 4679 pipe_class fpu_mem_reg(memory mem, regD src)
 4680 %{
 4681     instruction_count(2);
 4682     src    : S5(read);
 4683     mem    : S3(read);
 4684     DECODE : S0;        // any decoder for FPU PUSH
 4685     D0     : S1;        // big decoder only
 4686     FPU    : S4;
 4687     MEM    : S3;        // any mem
 4688 %}
 4689 
 4690 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4691 %{
 4692     instruction_count(3);
 4693     src1   : S3(read);
 4694     src2   : S3(read);
 4695     mem    : S3(read);
 4696     DECODE : S0(2);     // any decoder for FPU PUSH
 4697     D0     : S1;        // big decoder only
 4698     FPU    : S4;
 4699     MEM    : S3;        // any mem
 4700 %}
 4701 
 4702 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4703 %{
 4704     instruction_count(3);
 4705     src1   : S3(read);
 4706     src2   : S3(read);
 4707     mem    : S4(read);
 4708     DECODE : S0;        // any decoder for FPU PUSH
 4709     D0     : S0(2);     // big decoder only
 4710     FPU    : S4;
 4711     MEM    : S3(2);     // any mem
 4712 %}
 4713 
 4714 pipe_class fpu_mem_mem(memory dst, memory src1)
 4715 %{
 4716     instruction_count(2);
 4717     src1   : S3(read);
 4718     dst    : S4(read);
 4719     D0     : S0(2);     // big decoder only
 4720     MEM    : S3(2);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S3(read);
 4727     src2   : S3(read);
 4728     dst    : S4(read);
 4729     D0     : S0(3);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(3);     // any mem
 4732 %}
 4733 
 4734 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4735 %{
 4736     instruction_count(3);
 4737     src1   : S4(read);
 4738     mem    : S4(read);
 4739     DECODE : S0;        // any decoder for FPU PUSH
 4740     D0     : S0(2);     // big decoder only
 4741     FPU    : S4;
 4742     MEM    : S3(2);     // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_con(regD dst)
 4747 %{
 4748     instruction_count(2);
 4749     dst    : S5(write);
 4750     D0     : S0;        // big decoder only for the load
 4751     DECODE : S1;        // any decoder for FPU POP
 4752     FPU    : S4;
 4753     MEM    : S3;        // any mem
 4754 %}
 4755 
 4756 // Float load constant
 4757 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4758 %{
 4759     instruction_count(3);
 4760     dst    : S5(write);
 4761     src    : S3(read);
 4762     D0     : S0;        // big decoder only for the load
 4763     DECODE : S1(2);     // any decoder for FPU POP
 4764     FPU    : S4;
 4765     MEM    : S3;        // any mem
 4766 %}
 4767 
 4768 // UnConditional branch
 4769 pipe_class pipe_jmp(label labl)
 4770 %{
 4771     single_instruction;
 4772     BR   : S3;
 4773 %}
 4774 
 4775 // Conditional branch
 4776 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4777 %{
 4778     single_instruction;
 4779     cr    : S1(read);
 4780     BR    : S3;
 4781 %}
 4782 
 4783 // Allocation idiom
 4784 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4785 %{
 4786     instruction_count(1); force_serialization;
 4787     fixed_latency(6);
 4788     heap_ptr : S3(read);
 4789     DECODE   : S0(3);
 4790     D0       : S2;
 4791     MEM      : S3;
 4792     ALU      : S3(2);
 4793     dst      : S5(write);
 4794     BR       : S5;
 4795 %}
 4796 
 4797 // Generic big/slow expanded idiom
 4798 pipe_class pipe_slow()
 4799 %{
 4800     instruction_count(10); multiple_bundles; force_serialization;
 4801     fixed_latency(100);
 4802     D0  : S0(2);
 4803     MEM : S3(2);
 4804 %}
 4805 
 4806 // The real do-nothing guy
 4807 pipe_class empty()
 4808 %{
 4809     instruction_count(0);
 4810 %}
 4811 
 4812 // Define the class for the Nop node
 4813 define
 4814 %{
 4815    MachNop = empty;
 4816 %}
 4817 
 4818 %}
 4819 
 4820 //----------INSTRUCTIONS-------------------------------------------------------
 4821 //
 4822 // match      -- States which machine-independent subtree may be replaced
 4823 //               by this instruction.
 4824 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4825 //               selection to identify a minimum cost tree of machine
 4826 //               instructions that matches a tree of machine-independent
 4827 //               instructions.
 4828 // format     -- A string providing the disassembly for this instruction.
 4829 //               The value of an instruction's operand may be inserted
 4830 //               by referring to it with a '$' prefix.
 4831 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4832 //               to within an encode class as $primary, $secondary, and $tertiary
 4833 //               rrspectively.  The primary opcode is commonly used to
 4834 //               indicate the type of machine instruction, while secondary
 4835 //               and tertiary are often used for prefix options or addressing
 4836 //               modes.
 4837 // ins_encode -- A list of encode classes with parameters. The encode class
 4838 //               name must have been defined in an 'enc_class' specification
 4839 //               in the encode section of the architecture description.
 4840 
 4841 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4842 // Load Float
 4843 instruct MoveF2VL(vlRegF dst, regF src) %{
 4844   match(Set dst src);
 4845   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4846   ins_encode %{
 4847     ShouldNotReachHere();
 4848   %}
 4849   ins_pipe( fpu_reg_reg );
 4850 %}
 4851 
 4852 // Load Float
 4853 instruct MoveF2LEG(legRegF dst, regF src) %{
 4854   match(Set dst src);
 4855   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4856   ins_encode %{
 4857     ShouldNotReachHere();
 4858   %}
 4859   ins_pipe( fpu_reg_reg );
 4860 %}
 4861 
 4862 // Load Float
 4863 instruct MoveVL2F(regF dst, vlRegF src) %{
 4864   match(Set dst src);
 4865   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4866   ins_encode %{
 4867     ShouldNotReachHere();
 4868   %}
 4869   ins_pipe( fpu_reg_reg );
 4870 %}
 4871 
 4872 // Load Float
 4873 instruct MoveLEG2F(regF dst, legRegF src) %{
 4874   match(Set dst src);
 4875   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4876   ins_encode %{
 4877     ShouldNotReachHere();
 4878   %}
 4879   ins_pipe( fpu_reg_reg );
 4880 %}
 4881 
 4882 // Load Double
 4883 instruct MoveD2VL(vlRegD dst, regD src) %{
 4884   match(Set dst src);
 4885   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4886   ins_encode %{
 4887     ShouldNotReachHere();
 4888   %}
 4889   ins_pipe( fpu_reg_reg );
 4890 %}
 4891 
 4892 // Load Double
 4893 instruct MoveD2LEG(legRegD dst, regD src) %{
 4894   match(Set dst src);
 4895   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4896   ins_encode %{
 4897     ShouldNotReachHere();
 4898   %}
 4899   ins_pipe( fpu_reg_reg );
 4900 %}
 4901 
 4902 // Load Double
 4903 instruct MoveVL2D(regD dst, vlRegD src) %{
 4904   match(Set dst src);
 4905   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4906   ins_encode %{
 4907     ShouldNotReachHere();
 4908   %}
 4909   ins_pipe( fpu_reg_reg );
 4910 %}
 4911 
 4912 // Load Double
 4913 instruct MoveLEG2D(regD dst, legRegD src) %{
 4914   match(Set dst src);
 4915   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4916   ins_encode %{
 4917     ShouldNotReachHere();
 4918   %}
 4919   ins_pipe( fpu_reg_reg );
 4920 %}
 4921 
 4922 //----------Load/Store/Move Instructions---------------------------------------
 4923 //----------Load Instructions--------------------------------------------------
 4924 
 4925 // Load Byte (8 bit signed)
 4926 instruct loadB(rRegI dst, memory mem)
 4927 %{
 4928   match(Set dst (LoadB mem));
 4929 
 4930   ins_cost(125);
 4931   format %{ "movsbl  $dst, $mem\t# byte" %}
 4932 
 4933   ins_encode %{
 4934     __ movsbl($dst$$Register, $mem$$Address);
 4935   %}
 4936 
 4937   ins_pipe(ialu_reg_mem);
 4938 %}
 4939 
 4940 // Load Byte (8 bit signed) into Long Register
 4941 instruct loadB2L(rRegL dst, memory mem)
 4942 %{
 4943   match(Set dst (ConvI2L (LoadB mem)));
 4944 
 4945   ins_cost(125);
 4946   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4947 
 4948   ins_encode %{
 4949     __ movsbq($dst$$Register, $mem$$Address);
 4950   %}
 4951 
 4952   ins_pipe(ialu_reg_mem);
 4953 %}
 4954 
 4955 // Load Unsigned Byte (8 bit UNsigned)
 4956 instruct loadUB(rRegI dst, memory mem)
 4957 %{
 4958   match(Set dst (LoadUB mem));
 4959 
 4960   ins_cost(125);
 4961   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4962 
 4963   ins_encode %{
 4964     __ movzbl($dst$$Register, $mem$$Address);
 4965   %}
 4966 
 4967   ins_pipe(ialu_reg_mem);
 4968 %}
 4969 
 4970 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4971 instruct loadUB2L(rRegL dst, memory mem)
 4972 %{
 4973   match(Set dst (ConvI2L (LoadUB mem)));
 4974 
 4975   ins_cost(125);
 4976   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4977 
 4978   ins_encode %{
 4979     __ movzbq($dst$$Register, $mem$$Address);
 4980   %}
 4981 
 4982   ins_pipe(ialu_reg_mem);
 4983 %}
 4984 
 4985 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4986 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4987   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4988   effect(KILL cr);
 4989 
 4990   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4991             "andl    $dst, right_n_bits($mask, 8)" %}
 4992   ins_encode %{
 4993     Register Rdst = $dst$$Register;
 4994     __ movzbq(Rdst, $mem$$Address);
 4995     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4996   %}
 4997   ins_pipe(ialu_reg_mem);
 4998 %}
 4999 
 5000 // Load Short (16 bit signed)
 5001 instruct loadS(rRegI dst, memory mem)
 5002 %{
 5003   match(Set dst (LoadS mem));
 5004 
 5005   ins_cost(125);
 5006   format %{ "movswl $dst, $mem\t# short" %}
 5007 
 5008   ins_encode %{
 5009     __ movswl($dst$$Register, $mem$$Address);
 5010   %}
 5011 
 5012   ins_pipe(ialu_reg_mem);
 5013 %}
 5014 
 5015 // Load Short (16 bit signed) to Byte (8 bit signed)
 5016 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5017   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5018 
 5019   ins_cost(125);
 5020   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5021   ins_encode %{
 5022     __ movsbl($dst$$Register, $mem$$Address);
 5023   %}
 5024   ins_pipe(ialu_reg_mem);
 5025 %}
 5026 
 5027 // Load Short (16 bit signed) into Long Register
 5028 instruct loadS2L(rRegL dst, memory mem)
 5029 %{
 5030   match(Set dst (ConvI2L (LoadS mem)));
 5031 
 5032   ins_cost(125);
 5033   format %{ "movswq $dst, $mem\t# short -> long" %}
 5034 
 5035   ins_encode %{
 5036     __ movswq($dst$$Register, $mem$$Address);
 5037   %}
 5038 
 5039   ins_pipe(ialu_reg_mem);
 5040 %}
 5041 
 5042 // Load Unsigned Short/Char (16 bit UNsigned)
 5043 instruct loadUS(rRegI dst, memory mem)
 5044 %{
 5045   match(Set dst (LoadUS mem));
 5046 
 5047   ins_cost(125);
 5048   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5049 
 5050   ins_encode %{
 5051     __ movzwl($dst$$Register, $mem$$Address);
 5052   %}
 5053 
 5054   ins_pipe(ialu_reg_mem);
 5055 %}
 5056 
 5057 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5058 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5059   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5060 
 5061   ins_cost(125);
 5062   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5063   ins_encode %{
 5064     __ movsbl($dst$$Register, $mem$$Address);
 5065   %}
 5066   ins_pipe(ialu_reg_mem);
 5067 %}
 5068 
 5069 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5070 instruct loadUS2L(rRegL dst, memory mem)
 5071 %{
 5072   match(Set dst (ConvI2L (LoadUS mem)));
 5073 
 5074   ins_cost(125);
 5075   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5076 
 5077   ins_encode %{
 5078     __ movzwq($dst$$Register, $mem$$Address);
 5079   %}
 5080 
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5085 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087 
 5088   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5089   ins_encode %{
 5090     __ movzbq($dst$$Register, $mem$$Address);
 5091   %}
 5092   ins_pipe(ialu_reg_mem);
 5093 %}
 5094 
 5095 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5096 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5097   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5098   effect(KILL cr);
 5099 
 5100   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5101             "andl    $dst, right_n_bits($mask, 16)" %}
 5102   ins_encode %{
 5103     Register Rdst = $dst$$Register;
 5104     __ movzwq(Rdst, $mem$$Address);
 5105     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5106   %}
 5107   ins_pipe(ialu_reg_mem);
 5108 %}
 5109 
 5110 // Load Integer
 5111 instruct loadI(rRegI dst, memory mem)
 5112 %{
 5113   match(Set dst (LoadI mem));
 5114 
 5115   ins_cost(125);
 5116   format %{ "movl    $dst, $mem\t# int" %}
 5117 
 5118   ins_encode %{
 5119     __ movl($dst$$Register, $mem$$Address);
 5120   %}
 5121 
 5122   ins_pipe(ialu_reg_mem);
 5123 %}
 5124 
 5125 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5126 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5127   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5128 
 5129   ins_cost(125);
 5130   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5131   ins_encode %{
 5132     __ movsbl($dst$$Register, $mem$$Address);
 5133   %}
 5134   ins_pipe(ialu_reg_mem);
 5135 %}
 5136 
 5137 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5138 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5139   match(Set dst (AndI (LoadI mem) mask));
 5140 
 5141   ins_cost(125);
 5142   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5143   ins_encode %{
 5144     __ movzbl($dst$$Register, $mem$$Address);
 5145   %}
 5146   ins_pipe(ialu_reg_mem);
 5147 %}
 5148 
 5149 // Load Integer (32 bit signed) to Short (16 bit signed)
 5150 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5151   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5152 
 5153   ins_cost(125);
 5154   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5155   ins_encode %{
 5156     __ movswl($dst$$Register, $mem$$Address);
 5157   %}
 5158   ins_pipe(ialu_reg_mem);
 5159 %}
 5160 
 5161 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5162 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5163   match(Set dst (AndI (LoadI mem) mask));
 5164 
 5165   ins_cost(125);
 5166   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5167   ins_encode %{
 5168     __ movzwl($dst$$Register, $mem$$Address);
 5169   %}
 5170   ins_pipe(ialu_reg_mem);
 5171 %}
 5172 
 5173 // Load Integer into Long Register
 5174 instruct loadI2L(rRegL dst, memory mem)
 5175 %{
 5176   match(Set dst (ConvI2L (LoadI mem)));
 5177 
 5178   ins_cost(125);
 5179   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5180 
 5181   ins_encode %{
 5182     __ movslq($dst$$Register, $mem$$Address);
 5183   %}
 5184 
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFF into Long Register
 5189 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5193   ins_encode %{
 5194     __ movzbq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with mask 0xFFFF into Long Register
 5200 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202 
 5203   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5204   ins_encode %{
 5205     __ movzwq($dst$$Register, $mem$$Address);
 5206   %}
 5207   ins_pipe(ialu_reg_mem);
 5208 %}
 5209 
 5210 // Load Integer with a 31-bit mask into Long Register
 5211 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5212   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5213   effect(KILL cr);
 5214 
 5215   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5216             "andl    $dst, $mask" %}
 5217   ins_encode %{
 5218     Register Rdst = $dst$$Register;
 5219     __ movl(Rdst, $mem$$Address);
 5220     __ andl(Rdst, $mask$$constant);
 5221   %}
 5222   ins_pipe(ialu_reg_mem);
 5223 %}
 5224 
 5225 // Load Unsigned Integer into Long Register
 5226 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5227 %{
 5228   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5229 
 5230   ins_cost(125);
 5231   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5232 
 5233   ins_encode %{
 5234     __ movl($dst$$Register, $mem$$Address);
 5235   %}
 5236 
 5237   ins_pipe(ialu_reg_mem);
 5238 %}
 5239 
 5240 // Load Long
 5241 instruct loadL(rRegL dst, memory mem)
 5242 %{
 5243   match(Set dst (LoadL mem));
 5244 
 5245   ins_cost(125);
 5246   format %{ "movq    $dst, $mem\t# long" %}
 5247 
 5248   ins_encode %{
 5249     __ movq($dst$$Register, $mem$$Address);
 5250   %}
 5251 
 5252   ins_pipe(ialu_reg_mem); // XXX
 5253 %}
 5254 
 5255 // Load Range
 5256 instruct loadRange(rRegI dst, memory mem)
 5257 %{
 5258   match(Set dst (LoadRange mem));
 5259 
 5260   ins_cost(125); // XXX
 5261   format %{ "movl    $dst, $mem\t# range" %}
 5262   ins_encode %{
 5263     __ movl($dst$$Register, $mem$$Address);
 5264   %}
 5265   ins_pipe(ialu_reg_mem);
 5266 %}
 5267 
 5268 // Load Pointer
 5269 instruct loadP(rRegP dst, memory mem)
 5270 %{
 5271   match(Set dst (LoadP mem));
 5272   predicate(n->as_Load()->barrier_data() == 0);
 5273 
 5274   ins_cost(125); // XXX
 5275   format %{ "movq    $dst, $mem\t# ptr" %}
 5276   ins_encode %{
 5277     __ movq($dst$$Register, $mem$$Address);
 5278   %}
 5279   ins_pipe(ialu_reg_mem); // XXX
 5280 %}
 5281 
 5282 // Load Compressed Pointer
 5283 instruct loadN(rRegN dst, memory mem)
 5284 %{
 5285    match(Set dst (LoadN mem));
 5286 
 5287    ins_cost(125); // XXX
 5288    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5289    ins_encode %{
 5290      __ movl($dst$$Register, $mem$$Address);
 5291    %}
 5292    ins_pipe(ialu_reg_mem); // XXX
 5293 %}
 5294 
 5295 
 5296 // Load Klass Pointer
 5297 instruct loadKlass(rRegP dst, memory mem)
 5298 %{
 5299   match(Set dst (LoadKlass mem));
 5300 
 5301   ins_cost(125); // XXX
 5302   format %{ "movq    $dst, $mem\t# class" %}
 5303   ins_encode %{
 5304     __ movq($dst$$Register, $mem$$Address);
 5305   %}
 5306   ins_pipe(ialu_reg_mem); // XXX
 5307 %}
 5308 
 5309 // Load narrow Klass Pointer
 5310 instruct loadNKlass(rRegN dst, indOffset8 mem, rFlagsReg cr)
 5311 %{
 5312   match(Set dst (LoadNKlass mem));
 5313   effect(TEMP_DEF dst, KILL cr);
 5314   ins_cost(125); // XXX
 5315   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5316   ins_encode %{
 5317     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset 4, but got: %d", $mem$$disp);
 5318     assert($mem$$index == 4, "expect no index register: %d", $mem$$index);
 5319     __ load_nklass($dst$$Register, $mem$$base$$Register);
 5320   %}
 5321   ins_pipe(pipe_slow); // XXX
 5322 %}
 5323 
 5324 // Load Float
 5325 instruct loadF(regF dst, memory mem)
 5326 %{
 5327   match(Set dst (LoadF mem));
 5328 
 5329   ins_cost(145); // XXX
 5330   format %{ "movss   $dst, $mem\t# float" %}
 5331   ins_encode %{
 5332     __ movflt($dst$$XMMRegister, $mem$$Address);
 5333   %}
 5334   ins_pipe(pipe_slow); // XXX
 5335 %}
 5336 
 5337 // Load Double
 5338 instruct loadD_partial(regD dst, memory mem)
 5339 %{
 5340   predicate(!UseXmmLoadAndClearUpper);
 5341   match(Set dst (LoadD mem));
 5342 
 5343   ins_cost(145); // XXX
 5344   format %{ "movlpd  $dst, $mem\t# double" %}
 5345   ins_encode %{
 5346     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5347   %}
 5348   ins_pipe(pipe_slow); // XXX
 5349 %}
 5350 
 5351 instruct loadD(regD dst, memory mem)
 5352 %{
 5353   predicate(UseXmmLoadAndClearUpper);
 5354   match(Set dst (LoadD mem));
 5355 
 5356   ins_cost(145); // XXX
 5357   format %{ "movsd   $dst, $mem\t# double" %}
 5358   ins_encode %{
 5359     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5360   %}
 5361   ins_pipe(pipe_slow); // XXX
 5362 %}
 5363 
 5364 
 5365 // Following pseudo code describes the algorithm for max[FD]:
 5366 // Min algorithm is on similar lines
 5367 //  btmp = (b < +0.0) ? a : b
 5368 //  atmp = (b < +0.0) ? b : a
 5369 //  Tmp  = Max_Float(atmp , btmp)
 5370 //  Res  = (atmp == NaN) ? atmp : Tmp
 5371 
 5372 // max = java.lang.Math.max(float a, float b)
 5373 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5374   predicate(UseAVX > 0 && !n->is_reduction());
 5375   match(Set dst (MaxF a b));
 5376   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5377   format %{
 5378      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5379      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5380      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5381      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5382      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5383   %}
 5384   ins_encode %{
 5385     int vector_len = Assembler::AVX_128bit;
 5386     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5387     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5388     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5389     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5390     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5391  %}
 5392   ins_pipe( pipe_slow );
 5393 %}
 5394 
 5395 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5396   predicate(UseAVX > 0 && n->is_reduction());
 5397   match(Set dst (MaxF a b));
 5398   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5399 
 5400   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5401   ins_encode %{
 5402     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5403                     false /*min*/, true /*single*/);
 5404   %}
 5405   ins_pipe( pipe_slow );
 5406 %}
 5407 
 5408 // max = java.lang.Math.max(double a, double b)
 5409 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5410   predicate(UseAVX > 0 && !n->is_reduction());
 5411   match(Set dst (MaxD a b));
 5412   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5413   format %{
 5414      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5415      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5416      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5417      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5418      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5419   %}
 5420   ins_encode %{
 5421     int vector_len = Assembler::AVX_128bit;
 5422     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5423     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5424     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5425     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5426     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5427   %}
 5428   ins_pipe( pipe_slow );
 5429 %}
 5430 
 5431 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5432   predicate(UseAVX > 0 && n->is_reduction());
 5433   match(Set dst (MaxD a b));
 5434   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5435 
 5436   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5437   ins_encode %{
 5438     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5439                     false /*min*/, false /*single*/);
 5440   %}
 5441   ins_pipe( pipe_slow );
 5442 %}
 5443 
 5444 // min = java.lang.Math.min(float a, float b)
 5445 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5446   predicate(UseAVX > 0 && !n->is_reduction());
 5447   match(Set dst (MinF a b));
 5448   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5449   format %{
 5450      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5451      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5452      "vminss           $tmp,$atmp,$btmp           \n\t"
 5453      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5454      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5455   %}
 5456   ins_encode %{
 5457     int vector_len = Assembler::AVX_128bit;
 5458     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5459     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5460     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5461     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5462     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5463   %}
 5464   ins_pipe( pipe_slow );
 5465 %}
 5466 
 5467 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5468   predicate(UseAVX > 0 && n->is_reduction());
 5469   match(Set dst (MinF a b));
 5470   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5471 
 5472   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5473   ins_encode %{
 5474     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5475                     true /*min*/, true /*single*/);
 5476   %}
 5477   ins_pipe( pipe_slow );
 5478 %}
 5479 
 5480 // min = java.lang.Math.min(double a, double b)
 5481 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5482   predicate(UseAVX > 0 && !n->is_reduction());
 5483   match(Set dst (MinD a b));
 5484   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5485   format %{
 5486      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5487      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5488      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5489      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5490      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5491   %}
 5492   ins_encode %{
 5493     int vector_len = Assembler::AVX_128bit;
 5494     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5495     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5496     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5497     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5498     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5499   %}
 5500   ins_pipe( pipe_slow );
 5501 %}
 5502 
 5503 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5504   predicate(UseAVX > 0 && n->is_reduction());
 5505   match(Set dst (MinD a b));
 5506   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5507 
 5508   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5509   ins_encode %{
 5510     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5511                     true /*min*/, false /*single*/);
 5512   %}
 5513   ins_pipe( pipe_slow );
 5514 %}
 5515 
 5516 // Load Effective Address
 5517 instruct leaP8(rRegP dst, indOffset8 mem)
 5518 %{
 5519   match(Set dst mem);
 5520 
 5521   ins_cost(110); // XXX
 5522   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5523   ins_encode %{
 5524     __ leaq($dst$$Register, $mem$$Address);
 5525   %}
 5526   ins_pipe(ialu_reg_reg_fat);
 5527 %}
 5528 
 5529 instruct leaP32(rRegP dst, indOffset32 mem)
 5530 %{
 5531   match(Set dst mem);
 5532 
 5533   ins_cost(110);
 5534   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5535   ins_encode %{
 5536     __ leaq($dst$$Register, $mem$$Address);
 5537   %}
 5538   ins_pipe(ialu_reg_reg_fat);
 5539 %}
 5540 
 5541 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5542 %{
 5543   match(Set dst mem);
 5544 
 5545   ins_cost(110);
 5546   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5547   ins_encode %{
 5548     __ leaq($dst$$Register, $mem$$Address);
 5549   %}
 5550   ins_pipe(ialu_reg_reg_fat);
 5551 %}
 5552 
 5553 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5554 %{
 5555   match(Set dst mem);
 5556 
 5557   ins_cost(110);
 5558   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5559   ins_encode %{
 5560     __ leaq($dst$$Register, $mem$$Address);
 5561   %}
 5562   ins_pipe(ialu_reg_reg_fat);
 5563 %}
 5564 
 5565 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5566 %{
 5567   match(Set dst mem);
 5568 
 5569   ins_cost(110);
 5570   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5571   ins_encode %{
 5572     __ leaq($dst$$Register, $mem$$Address);
 5573   %}
 5574   ins_pipe(ialu_reg_reg_fat);
 5575 %}
 5576 
 5577 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5578 %{
 5579   match(Set dst mem);
 5580 
 5581   ins_cost(110);
 5582   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5583   ins_encode %{
 5584     __ leaq($dst$$Register, $mem$$Address);
 5585   %}
 5586   ins_pipe(ialu_reg_reg_fat);
 5587 %}
 5588 
 5589 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5590 %{
 5591   match(Set dst mem);
 5592 
 5593   ins_cost(110);
 5594   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5595   ins_encode %{
 5596     __ leaq($dst$$Register, $mem$$Address);
 5597   %}
 5598   ins_pipe(ialu_reg_reg_fat);
 5599 %}
 5600 
 5601 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5602 %{
 5603   match(Set dst mem);
 5604 
 5605   ins_cost(110);
 5606   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5607   ins_encode %{
 5608     __ leaq($dst$$Register, $mem$$Address);
 5609   %}
 5610   ins_pipe(ialu_reg_reg_fat);
 5611 %}
 5612 
 5613 // Load Effective Address which uses Narrow (32-bits) oop
 5614 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5615 %{
 5616   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5617   match(Set dst mem);
 5618 
 5619   ins_cost(110);
 5620   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5621   ins_encode %{
 5622     __ leaq($dst$$Register, $mem$$Address);
 5623   %}
 5624   ins_pipe(ialu_reg_reg_fat);
 5625 %}
 5626 
 5627 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5628 %{
 5629   predicate(CompressedOops::shift() == 0);
 5630   match(Set dst mem);
 5631 
 5632   ins_cost(110); // XXX
 5633   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5634   ins_encode %{
 5635     __ leaq($dst$$Register, $mem$$Address);
 5636   %}
 5637   ins_pipe(ialu_reg_reg_fat);
 5638 %}
 5639 
 5640 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5641 %{
 5642   predicate(CompressedOops::shift() == 0);
 5643   match(Set dst mem);
 5644 
 5645   ins_cost(110);
 5646   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5647   ins_encode %{
 5648     __ leaq($dst$$Register, $mem$$Address);
 5649   %}
 5650   ins_pipe(ialu_reg_reg_fat);
 5651 %}
 5652 
 5653 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5654 %{
 5655   predicate(CompressedOops::shift() == 0);
 5656   match(Set dst mem);
 5657 
 5658   ins_cost(110);
 5659   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5660   ins_encode %{
 5661     __ leaq($dst$$Register, $mem$$Address);
 5662   %}
 5663   ins_pipe(ialu_reg_reg_fat);
 5664 %}
 5665 
 5666 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5667 %{
 5668   predicate(CompressedOops::shift() == 0);
 5669   match(Set dst mem);
 5670 
 5671   ins_cost(110);
 5672   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5673   ins_encode %{
 5674     __ leaq($dst$$Register, $mem$$Address);
 5675   %}
 5676   ins_pipe(ialu_reg_reg_fat);
 5677 %}
 5678 
 5679 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5680 %{
 5681   predicate(CompressedOops::shift() == 0);
 5682   match(Set dst mem);
 5683 
 5684   ins_cost(110);
 5685   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5686   ins_encode %{
 5687     __ leaq($dst$$Register, $mem$$Address);
 5688   %}
 5689   ins_pipe(ialu_reg_reg_fat);
 5690 %}
 5691 
 5692 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5693 %{
 5694   predicate(CompressedOops::shift() == 0);
 5695   match(Set dst mem);
 5696 
 5697   ins_cost(110);
 5698   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5699   ins_encode %{
 5700     __ leaq($dst$$Register, $mem$$Address);
 5701   %}
 5702   ins_pipe(ialu_reg_reg_fat);
 5703 %}
 5704 
 5705 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5706 %{
 5707   predicate(CompressedOops::shift() == 0);
 5708   match(Set dst mem);
 5709 
 5710   ins_cost(110);
 5711   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5712   ins_encode %{
 5713     __ leaq($dst$$Register, $mem$$Address);
 5714   %}
 5715   ins_pipe(ialu_reg_reg_fat);
 5716 %}
 5717 
 5718 instruct loadConI(rRegI dst, immI src)
 5719 %{
 5720   match(Set dst src);
 5721 
 5722   format %{ "movl    $dst, $src\t# int" %}
 5723   ins_encode %{
 5724     __ movl($dst$$Register, $src$$constant);
 5725   %}
 5726   ins_pipe(ialu_reg_fat); // XXX
 5727 %}
 5728 
 5729 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5730 %{
 5731   match(Set dst src);
 5732   effect(KILL cr);
 5733 
 5734   ins_cost(50);
 5735   format %{ "xorl    $dst, $dst\t# int" %}
 5736   ins_encode %{
 5737     __ xorl($dst$$Register, $dst$$Register);
 5738   %}
 5739   ins_pipe(ialu_reg);
 5740 %}
 5741 
 5742 instruct loadConL(rRegL dst, immL src)
 5743 %{
 5744   match(Set dst src);
 5745 
 5746   ins_cost(150);
 5747   format %{ "movq    $dst, $src\t# long" %}
 5748   ins_encode %{
 5749     __ mov64($dst$$Register, $src$$constant);
 5750   %}
 5751   ins_pipe(ialu_reg);
 5752 %}
 5753 
 5754 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5755 %{
 5756   match(Set dst src);
 5757   effect(KILL cr);
 5758 
 5759   ins_cost(50);
 5760   format %{ "xorl    $dst, $dst\t# long" %}
 5761   ins_encode %{
 5762     __ xorl($dst$$Register, $dst$$Register);
 5763   %}
 5764   ins_pipe(ialu_reg); // XXX
 5765 %}
 5766 
 5767 instruct loadConUL32(rRegL dst, immUL32 src)
 5768 %{
 5769   match(Set dst src);
 5770 
 5771   ins_cost(60);
 5772   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5773   ins_encode %{
 5774     __ movl($dst$$Register, $src$$constant);
 5775   %}
 5776   ins_pipe(ialu_reg);
 5777 %}
 5778 
 5779 instruct loadConL32(rRegL dst, immL32 src)
 5780 %{
 5781   match(Set dst src);
 5782 
 5783   ins_cost(70);
 5784   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5785   ins_encode %{
 5786     __ movq($dst$$Register, $src$$constant);
 5787   %}
 5788   ins_pipe(ialu_reg);
 5789 %}
 5790 
 5791 instruct loadConP(rRegP dst, immP con) %{
 5792   match(Set dst con);
 5793 
 5794   format %{ "movq    $dst, $con\t# ptr" %}
 5795   ins_encode %{
 5796     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5797   %}
 5798   ins_pipe(ialu_reg_fat); // XXX
 5799 %}
 5800 
 5801 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5802 %{
 5803   match(Set dst src);
 5804   effect(KILL cr);
 5805 
 5806   ins_cost(50);
 5807   format %{ "xorl    $dst, $dst\t# ptr" %}
 5808   ins_encode %{
 5809     __ xorl($dst$$Register, $dst$$Register);
 5810   %}
 5811   ins_pipe(ialu_reg);
 5812 %}
 5813 
 5814 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5815 %{
 5816   match(Set dst src);
 5817   effect(KILL cr);
 5818 
 5819   ins_cost(60);
 5820   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5821   ins_encode %{
 5822     __ movl($dst$$Register, $src$$constant);
 5823   %}
 5824   ins_pipe(ialu_reg);
 5825 %}
 5826 
 5827 instruct loadConF(regF dst, immF con) %{
 5828   match(Set dst con);
 5829   ins_cost(125);
 5830   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5831   ins_encode %{
 5832     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5833   %}
 5834   ins_pipe(pipe_slow);
 5835 %}
 5836 
 5837 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5838   match(Set dst src);
 5839   effect(KILL cr);
 5840   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5841   ins_encode %{
 5842     __ xorq($dst$$Register, $dst$$Register);
 5843   %}
 5844   ins_pipe(ialu_reg);
 5845 %}
 5846 
 5847 instruct loadConN(rRegN dst, immN src) %{
 5848   match(Set dst src);
 5849 
 5850   ins_cost(125);
 5851   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5852   ins_encode %{
 5853     address con = (address)$src$$constant;
 5854     if (con == NULL) {
 5855       ShouldNotReachHere();
 5856     } else {
 5857       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5858     }
 5859   %}
 5860   ins_pipe(ialu_reg_fat); // XXX
 5861 %}
 5862 
 5863 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5864   match(Set dst src);
 5865 
 5866   ins_cost(125);
 5867   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5868   ins_encode %{
 5869     address con = (address)$src$$constant;
 5870     if (con == NULL) {
 5871       ShouldNotReachHere();
 5872     } else {
 5873       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5874     }
 5875   %}
 5876   ins_pipe(ialu_reg_fat); // XXX
 5877 %}
 5878 
 5879 instruct loadConF0(regF dst, immF0 src)
 5880 %{
 5881   match(Set dst src);
 5882   ins_cost(100);
 5883 
 5884   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5885   ins_encode %{
 5886     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5887   %}
 5888   ins_pipe(pipe_slow);
 5889 %}
 5890 
 5891 // Use the same format since predicate() can not be used here.
 5892 instruct loadConD(regD dst, immD con) %{
 5893   match(Set dst con);
 5894   ins_cost(125);
 5895   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5896   ins_encode %{
 5897     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5898   %}
 5899   ins_pipe(pipe_slow);
 5900 %}
 5901 
 5902 instruct loadConD0(regD dst, immD0 src)
 5903 %{
 5904   match(Set dst src);
 5905   ins_cost(100);
 5906 
 5907   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5908   ins_encode %{
 5909     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5910   %}
 5911   ins_pipe(pipe_slow);
 5912 %}
 5913 
 5914 instruct loadSSI(rRegI dst, stackSlotI src)
 5915 %{
 5916   match(Set dst src);
 5917 
 5918   ins_cost(125);
 5919   format %{ "movl    $dst, $src\t# int stk" %}
 5920   opcode(0x8B);
 5921   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5922   ins_pipe(ialu_reg_mem);
 5923 %}
 5924 
 5925 instruct loadSSL(rRegL dst, stackSlotL src)
 5926 %{
 5927   match(Set dst src);
 5928 
 5929   ins_cost(125);
 5930   format %{ "movq    $dst, $src\t# long stk" %}
 5931   opcode(0x8B);
 5932   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5933   ins_pipe(ialu_reg_mem);
 5934 %}
 5935 
 5936 instruct loadSSP(rRegP dst, stackSlotP src)
 5937 %{
 5938   match(Set dst src);
 5939 
 5940   ins_cost(125);
 5941   format %{ "movq    $dst, $src\t# ptr stk" %}
 5942   opcode(0x8B);
 5943   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5944   ins_pipe(ialu_reg_mem);
 5945 %}
 5946 
 5947 instruct loadSSF(regF dst, stackSlotF src)
 5948 %{
 5949   match(Set dst src);
 5950 
 5951   ins_cost(125);
 5952   format %{ "movss   $dst, $src\t# float stk" %}
 5953   ins_encode %{
 5954     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5955   %}
 5956   ins_pipe(pipe_slow); // XXX
 5957 %}
 5958 
 5959 // Use the same format since predicate() can not be used here.
 5960 instruct loadSSD(regD dst, stackSlotD src)
 5961 %{
 5962   match(Set dst src);
 5963 
 5964   ins_cost(125);
 5965   format %{ "movsd   $dst, $src\t# double stk" %}
 5966   ins_encode  %{
 5967     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5968   %}
 5969   ins_pipe(pipe_slow); // XXX
 5970 %}
 5971 
 5972 // Prefetch instructions for allocation.
 5973 // Must be safe to execute with invalid address (cannot fault).
 5974 
 5975 instruct prefetchAlloc( memory mem ) %{
 5976   predicate(AllocatePrefetchInstr==3);
 5977   match(PrefetchAllocation mem);
 5978   ins_cost(125);
 5979 
 5980   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5981   ins_encode %{
 5982     __ prefetchw($mem$$Address);
 5983   %}
 5984   ins_pipe(ialu_mem);
 5985 %}
 5986 
 5987 instruct prefetchAllocNTA( memory mem ) %{
 5988   predicate(AllocatePrefetchInstr==0);
 5989   match(PrefetchAllocation mem);
 5990   ins_cost(125);
 5991 
 5992   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5993   ins_encode %{
 5994     __ prefetchnta($mem$$Address);
 5995   %}
 5996   ins_pipe(ialu_mem);
 5997 %}
 5998 
 5999 instruct prefetchAllocT0( memory mem ) %{
 6000   predicate(AllocatePrefetchInstr==1);
 6001   match(PrefetchAllocation mem);
 6002   ins_cost(125);
 6003 
 6004   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6005   ins_encode %{
 6006     __ prefetcht0($mem$$Address);
 6007   %}
 6008   ins_pipe(ialu_mem);
 6009 %}
 6010 
 6011 instruct prefetchAllocT2( memory mem ) %{
 6012   predicate(AllocatePrefetchInstr==2);
 6013   match(PrefetchAllocation mem);
 6014   ins_cost(125);
 6015 
 6016   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6017   ins_encode %{
 6018     __ prefetcht2($mem$$Address);
 6019   %}
 6020   ins_pipe(ialu_mem);
 6021 %}
 6022 
 6023 //----------Store Instructions-------------------------------------------------
 6024 
 6025 // Store Byte
 6026 instruct storeB(memory mem, rRegI src)
 6027 %{
 6028   match(Set mem (StoreB mem src));
 6029 
 6030   ins_cost(125); // XXX
 6031   format %{ "movb    $mem, $src\t# byte" %}
 6032   ins_encode %{
 6033     __ movb($mem$$Address, $src$$Register);
 6034   %}
 6035   ins_pipe(ialu_mem_reg);
 6036 %}
 6037 
 6038 // Store Char/Short
 6039 instruct storeC(memory mem, rRegI src)
 6040 %{
 6041   match(Set mem (StoreC mem src));
 6042 
 6043   ins_cost(125); // XXX
 6044   format %{ "movw    $mem, $src\t# char/short" %}
 6045   ins_encode %{
 6046     __ movw($mem$$Address, $src$$Register);
 6047   %}
 6048   ins_pipe(ialu_mem_reg);
 6049 %}
 6050 
 6051 // Store Integer
 6052 instruct storeI(memory mem, rRegI src)
 6053 %{
 6054   match(Set mem (StoreI mem src));
 6055 
 6056   ins_cost(125); // XXX
 6057   format %{ "movl    $mem, $src\t# int" %}
 6058   ins_encode %{
 6059     __ movl($mem$$Address, $src$$Register);
 6060   %}
 6061   ins_pipe(ialu_mem_reg);
 6062 %}
 6063 
 6064 // Store Long
 6065 instruct storeL(memory mem, rRegL src)
 6066 %{
 6067   match(Set mem (StoreL mem src));
 6068 
 6069   ins_cost(125); // XXX
 6070   format %{ "movq    $mem, $src\t# long" %}
 6071   ins_encode %{
 6072     __ movq($mem$$Address, $src$$Register);
 6073   %}
 6074   ins_pipe(ialu_mem_reg); // XXX
 6075 %}
 6076 
 6077 // Store Pointer
 6078 instruct storeP(memory mem, any_RegP src)
 6079 %{
 6080   match(Set mem (StoreP mem src));
 6081 
 6082   ins_cost(125); // XXX
 6083   format %{ "movq    $mem, $src\t# ptr" %}
 6084   ins_encode %{
 6085     __ movq($mem$$Address, $src$$Register);
 6086   %}
 6087   ins_pipe(ialu_mem_reg);
 6088 %}
 6089 
 6090 instruct storeImmP0(memory mem, immP0 zero)
 6091 %{
 6092   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6093   match(Set mem (StoreP mem zero));
 6094 
 6095   ins_cost(125); // XXX
 6096   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6097   ins_encode %{
 6098     __ movq($mem$$Address, r12);
 6099   %}
 6100   ins_pipe(ialu_mem_reg);
 6101 %}
 6102 
 6103 // Store NULL Pointer, mark word, or other simple pointer constant.
 6104 instruct storeImmP(memory mem, immP31 src)
 6105 %{
 6106   match(Set mem (StoreP mem src));
 6107 
 6108   ins_cost(150); // XXX
 6109   format %{ "movq    $mem, $src\t# ptr" %}
 6110   ins_encode %{
 6111     __ movq($mem$$Address, $src$$constant);
 6112   %}
 6113   ins_pipe(ialu_mem_imm);
 6114 %}
 6115 
 6116 // Store Compressed Pointer
 6117 instruct storeN(memory mem, rRegN src)
 6118 %{
 6119   match(Set mem (StoreN mem src));
 6120 
 6121   ins_cost(125); // XXX
 6122   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6123   ins_encode %{
 6124     __ movl($mem$$Address, $src$$Register);
 6125   %}
 6126   ins_pipe(ialu_mem_reg);
 6127 %}
 6128 
 6129 instruct storeNKlass(memory mem, rRegN src)
 6130 %{
 6131   match(Set mem (StoreNKlass mem src));
 6132 
 6133   ins_cost(125); // XXX
 6134   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6135   ins_encode %{
 6136     __ movl($mem$$Address, $src$$Register);
 6137   %}
 6138   ins_pipe(ialu_mem_reg);
 6139 %}
 6140 
 6141 instruct storeImmN0(memory mem, immN0 zero)
 6142 %{
 6143   predicate(CompressedOops::base() == NULL);
 6144   match(Set mem (StoreN mem zero));
 6145 
 6146   ins_cost(125); // XXX
 6147   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6148   ins_encode %{
 6149     __ movl($mem$$Address, r12);
 6150   %}
 6151   ins_pipe(ialu_mem_reg);
 6152 %}
 6153 
 6154 instruct storeImmN(memory mem, immN src)
 6155 %{
 6156   match(Set mem (StoreN mem src));
 6157 
 6158   ins_cost(150); // XXX
 6159   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6160   ins_encode %{
 6161     address con = (address)$src$$constant;
 6162     if (con == NULL) {
 6163       __ movl($mem$$Address, (int32_t)0);
 6164     } else {
 6165       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6166     }
 6167   %}
 6168   ins_pipe(ialu_mem_imm);
 6169 %}
 6170 
 6171 instruct storeImmNKlass(memory mem, immNKlass src)
 6172 %{
 6173   match(Set mem (StoreNKlass mem src));
 6174 
 6175   ins_cost(150); // XXX
 6176   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6177   ins_encode %{
 6178     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6179   %}
 6180   ins_pipe(ialu_mem_imm);
 6181 %}
 6182 
 6183 // Store Integer Immediate
 6184 instruct storeImmI0(memory mem, immI_0 zero)
 6185 %{
 6186   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6187   match(Set mem (StoreI mem zero));
 6188 
 6189   ins_cost(125); // XXX
 6190   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6191   ins_encode %{
 6192     __ movl($mem$$Address, r12);
 6193   %}
 6194   ins_pipe(ialu_mem_reg);
 6195 %}
 6196 
 6197 instruct storeImmI(memory mem, immI src)
 6198 %{
 6199   match(Set mem (StoreI mem src));
 6200 
 6201   ins_cost(150);
 6202   format %{ "movl    $mem, $src\t# int" %}
 6203   ins_encode %{
 6204     __ movl($mem$$Address, $src$$constant);
 6205   %}
 6206   ins_pipe(ialu_mem_imm);
 6207 %}
 6208 
 6209 // Store Long Immediate
 6210 instruct storeImmL0(memory mem, immL0 zero)
 6211 %{
 6212   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6213   match(Set mem (StoreL mem zero));
 6214 
 6215   ins_cost(125); // XXX
 6216   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6217   ins_encode %{
 6218     __ movq($mem$$Address, r12);
 6219   %}
 6220   ins_pipe(ialu_mem_reg);
 6221 %}
 6222 
 6223 instruct storeImmL(memory mem, immL32 src)
 6224 %{
 6225   match(Set mem (StoreL mem src));
 6226 
 6227   ins_cost(150);
 6228   format %{ "movq    $mem, $src\t# long" %}
 6229   ins_encode %{
 6230     __ movq($mem$$Address, $src$$constant);
 6231   %}
 6232   ins_pipe(ialu_mem_imm);
 6233 %}
 6234 
 6235 // Store Short/Char Immediate
 6236 instruct storeImmC0(memory mem, immI_0 zero)
 6237 %{
 6238   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6239   match(Set mem (StoreC mem zero));
 6240 
 6241   ins_cost(125); // XXX
 6242   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6243   ins_encode %{
 6244     __ movw($mem$$Address, r12);
 6245   %}
 6246   ins_pipe(ialu_mem_reg);
 6247 %}
 6248 
 6249 instruct storeImmI16(memory mem, immI16 src)
 6250 %{
 6251   predicate(UseStoreImmI16);
 6252   match(Set mem (StoreC mem src));
 6253 
 6254   ins_cost(150);
 6255   format %{ "movw    $mem, $src\t# short/char" %}
 6256   ins_encode %{
 6257     __ movw($mem$$Address, $src$$constant);
 6258   %}
 6259   ins_pipe(ialu_mem_imm);
 6260 %}
 6261 
 6262 // Store Byte Immediate
 6263 instruct storeImmB0(memory mem, immI_0 zero)
 6264 %{
 6265   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6266   match(Set mem (StoreB mem zero));
 6267 
 6268   ins_cost(125); // XXX
 6269   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6270   ins_encode %{
 6271     __ movb($mem$$Address, r12);
 6272   %}
 6273   ins_pipe(ialu_mem_reg);
 6274 %}
 6275 
 6276 instruct storeImmB(memory mem, immI8 src)
 6277 %{
 6278   match(Set mem (StoreB mem src));
 6279 
 6280   ins_cost(150); // XXX
 6281   format %{ "movb    $mem, $src\t# byte" %}
 6282   ins_encode %{
 6283     __ movb($mem$$Address, $src$$constant);
 6284   %}
 6285   ins_pipe(ialu_mem_imm);
 6286 %}
 6287 
 6288 // Store CMS card-mark Immediate
 6289 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6290 %{
 6291   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6292   match(Set mem (StoreCM mem zero));
 6293 
 6294   ins_cost(125); // XXX
 6295   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6296   ins_encode %{
 6297     __ movb($mem$$Address, r12);
 6298   %}
 6299   ins_pipe(ialu_mem_reg);
 6300 %}
 6301 
 6302 instruct storeImmCM0(memory mem, immI_0 src)
 6303 %{
 6304   match(Set mem (StoreCM mem src));
 6305 
 6306   ins_cost(150); // XXX
 6307   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6308   ins_encode %{
 6309     __ movb($mem$$Address, $src$$constant);
 6310   %}
 6311   ins_pipe(ialu_mem_imm);
 6312 %}
 6313 
 6314 // Store Float
 6315 instruct storeF(memory mem, regF src)
 6316 %{
 6317   match(Set mem (StoreF mem src));
 6318 
 6319   ins_cost(95); // XXX
 6320   format %{ "movss   $mem, $src\t# float" %}
 6321   ins_encode %{
 6322     __ movflt($mem$$Address, $src$$XMMRegister);
 6323   %}
 6324   ins_pipe(pipe_slow); // XXX
 6325 %}
 6326 
 6327 // Store immediate Float value (it is faster than store from XMM register)
 6328 instruct storeF0(memory mem, immF0 zero)
 6329 %{
 6330   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6331   match(Set mem (StoreF mem zero));
 6332 
 6333   ins_cost(25); // XXX
 6334   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6335   ins_encode %{
 6336     __ movl($mem$$Address, r12);
 6337   %}
 6338   ins_pipe(ialu_mem_reg);
 6339 %}
 6340 
 6341 instruct storeF_imm(memory mem, immF src)
 6342 %{
 6343   match(Set mem (StoreF mem src));
 6344 
 6345   ins_cost(50);
 6346   format %{ "movl    $mem, $src\t# float" %}
 6347   ins_encode %{
 6348     __ movl($mem$$Address, jint_cast($src$$constant));
 6349   %}
 6350   ins_pipe(ialu_mem_imm);
 6351 %}
 6352 
 6353 // Store Double
 6354 instruct storeD(memory mem, regD src)
 6355 %{
 6356   match(Set mem (StoreD mem src));
 6357 
 6358   ins_cost(95); // XXX
 6359   format %{ "movsd   $mem, $src\t# double" %}
 6360   ins_encode %{
 6361     __ movdbl($mem$$Address, $src$$XMMRegister);
 6362   %}
 6363   ins_pipe(pipe_slow); // XXX
 6364 %}
 6365 
 6366 // Store immediate double 0.0 (it is faster than store from XMM register)
 6367 instruct storeD0_imm(memory mem, immD0 src)
 6368 %{
 6369   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6370   match(Set mem (StoreD mem src));
 6371 
 6372   ins_cost(50);
 6373   format %{ "movq    $mem, $src\t# double 0." %}
 6374   ins_encode %{
 6375     __ movq($mem$$Address, $src$$constant);
 6376   %}
 6377   ins_pipe(ialu_mem_imm);
 6378 %}
 6379 
 6380 instruct storeD0(memory mem, immD0 zero)
 6381 %{
 6382   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6383   match(Set mem (StoreD mem zero));
 6384 
 6385   ins_cost(25); // XXX
 6386   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6387   ins_encode %{
 6388     __ movq($mem$$Address, r12);
 6389   %}
 6390   ins_pipe(ialu_mem_reg);
 6391 %}
 6392 
 6393 instruct storeSSI(stackSlotI dst, rRegI src)
 6394 %{
 6395   match(Set dst src);
 6396 
 6397   ins_cost(100);
 6398   format %{ "movl    $dst, $src\t# int stk" %}
 6399   opcode(0x89);
 6400   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6401   ins_pipe( ialu_mem_reg );
 6402 %}
 6403 
 6404 instruct storeSSL(stackSlotL dst, rRegL src)
 6405 %{
 6406   match(Set dst src);
 6407 
 6408   ins_cost(100);
 6409   format %{ "movq    $dst, $src\t# long stk" %}
 6410   opcode(0x89);
 6411   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6412   ins_pipe(ialu_mem_reg);
 6413 %}
 6414 
 6415 instruct storeSSP(stackSlotP dst, rRegP src)
 6416 %{
 6417   match(Set dst src);
 6418 
 6419   ins_cost(100);
 6420   format %{ "movq    $dst, $src\t# ptr stk" %}
 6421   opcode(0x89);
 6422   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6423   ins_pipe(ialu_mem_reg);
 6424 %}
 6425 
 6426 instruct storeSSF(stackSlotF dst, regF src)
 6427 %{
 6428   match(Set dst src);
 6429 
 6430   ins_cost(95); // XXX
 6431   format %{ "movss   $dst, $src\t# float stk" %}
 6432   ins_encode %{
 6433     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6434   %}
 6435   ins_pipe(pipe_slow); // XXX
 6436 %}
 6437 
 6438 instruct storeSSD(stackSlotD dst, regD src)
 6439 %{
 6440   match(Set dst src);
 6441 
 6442   ins_cost(95); // XXX
 6443   format %{ "movsd   $dst, $src\t# double stk" %}
 6444   ins_encode %{
 6445     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6446   %}
 6447   ins_pipe(pipe_slow); // XXX
 6448 %}
 6449 
 6450 instruct cacheWB(indirect addr)
 6451 %{
 6452   predicate(VM_Version::supports_data_cache_line_flush());
 6453   match(CacheWB addr);
 6454 
 6455   ins_cost(100);
 6456   format %{"cache wb $addr" %}
 6457   ins_encode %{
 6458     assert($addr->index_position() < 0, "should be");
 6459     assert($addr$$disp == 0, "should be");
 6460     __ cache_wb(Address($addr$$base$$Register, 0));
 6461   %}
 6462   ins_pipe(pipe_slow); // XXX
 6463 %}
 6464 
 6465 instruct cacheWBPreSync()
 6466 %{
 6467   predicate(VM_Version::supports_data_cache_line_flush());
 6468   match(CacheWBPreSync);
 6469 
 6470   ins_cost(100);
 6471   format %{"cache wb presync" %}
 6472   ins_encode %{
 6473     __ cache_wbsync(true);
 6474   %}
 6475   ins_pipe(pipe_slow); // XXX
 6476 %}
 6477 
 6478 instruct cacheWBPostSync()
 6479 %{
 6480   predicate(VM_Version::supports_data_cache_line_flush());
 6481   match(CacheWBPostSync);
 6482 
 6483   ins_cost(100);
 6484   format %{"cache wb postsync" %}
 6485   ins_encode %{
 6486     __ cache_wbsync(false);
 6487   %}
 6488   ins_pipe(pipe_slow); // XXX
 6489 %}
 6490 
 6491 //----------BSWAP Instructions-------------------------------------------------
 6492 instruct bytes_reverse_int(rRegI dst) %{
 6493   match(Set dst (ReverseBytesI dst));
 6494 
 6495   format %{ "bswapl  $dst" %}
 6496   ins_encode %{
 6497     __ bswapl($dst$$Register);
 6498   %}
 6499   ins_pipe( ialu_reg );
 6500 %}
 6501 
 6502 instruct bytes_reverse_long(rRegL dst) %{
 6503   match(Set dst (ReverseBytesL dst));
 6504 
 6505   format %{ "bswapq  $dst" %}
 6506   ins_encode %{
 6507     __ bswapq($dst$$Register);
 6508   %}
 6509   ins_pipe( ialu_reg);
 6510 %}
 6511 
 6512 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6513   match(Set dst (ReverseBytesUS dst));
 6514   effect(KILL cr);
 6515 
 6516   format %{ "bswapl  $dst\n\t"
 6517             "shrl    $dst,16\n\t" %}
 6518   ins_encode %{
 6519     __ bswapl($dst$$Register);
 6520     __ shrl($dst$$Register, 16);
 6521   %}
 6522   ins_pipe( ialu_reg );
 6523 %}
 6524 
 6525 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6526   match(Set dst (ReverseBytesS dst));
 6527   effect(KILL cr);
 6528 
 6529   format %{ "bswapl  $dst\n\t"
 6530             "sar     $dst,16\n\t" %}
 6531   ins_encode %{
 6532     __ bswapl($dst$$Register);
 6533     __ sarl($dst$$Register, 16);
 6534   %}
 6535   ins_pipe( ialu_reg );
 6536 %}
 6537 
 6538 //---------- Zeros Count Instructions ------------------------------------------
 6539 
 6540 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6541   predicate(UseCountLeadingZerosInstruction);
 6542   match(Set dst (CountLeadingZerosI src));
 6543   effect(KILL cr);
 6544 
 6545   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6546   ins_encode %{
 6547     __ lzcntl($dst$$Register, $src$$Register);
 6548   %}
 6549   ins_pipe(ialu_reg);
 6550 %}
 6551 
 6552 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6553   predicate(UseCountLeadingZerosInstruction);
 6554   match(Set dst (CountLeadingZerosI (LoadI src)));
 6555   effect(KILL cr);
 6556   ins_cost(175);
 6557   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6558   ins_encode %{
 6559     __ lzcntl($dst$$Register, $src$$Address);
 6560   %}
 6561   ins_pipe(ialu_reg_mem);
 6562 %}
 6563 
 6564 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6565   predicate(!UseCountLeadingZerosInstruction);
 6566   match(Set dst (CountLeadingZerosI src));
 6567   effect(KILL cr);
 6568 
 6569   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6570             "jnz     skip\n\t"
 6571             "movl    $dst, -1\n"
 6572       "skip:\n\t"
 6573             "negl    $dst\n\t"
 6574             "addl    $dst, 31" %}
 6575   ins_encode %{
 6576     Register Rdst = $dst$$Register;
 6577     Register Rsrc = $src$$Register;
 6578     Label skip;
 6579     __ bsrl(Rdst, Rsrc);
 6580     __ jccb(Assembler::notZero, skip);
 6581     __ movl(Rdst, -1);
 6582     __ bind(skip);
 6583     __ negl(Rdst);
 6584     __ addl(Rdst, BitsPerInt - 1);
 6585   %}
 6586   ins_pipe(ialu_reg);
 6587 %}
 6588 
 6589 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6590   predicate(UseCountLeadingZerosInstruction);
 6591   match(Set dst (CountLeadingZerosL src));
 6592   effect(KILL cr);
 6593 
 6594   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6595   ins_encode %{
 6596     __ lzcntq($dst$$Register, $src$$Register);
 6597   %}
 6598   ins_pipe(ialu_reg);
 6599 %}
 6600 
 6601 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6602   predicate(UseCountLeadingZerosInstruction);
 6603   match(Set dst (CountLeadingZerosL (LoadL src)));
 6604   effect(KILL cr);
 6605   ins_cost(175);
 6606   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6607   ins_encode %{
 6608     __ lzcntq($dst$$Register, $src$$Address);
 6609   %}
 6610   ins_pipe(ialu_reg_mem);
 6611 %}
 6612 
 6613 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6614   predicate(!UseCountLeadingZerosInstruction);
 6615   match(Set dst (CountLeadingZerosL src));
 6616   effect(KILL cr);
 6617 
 6618   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6619             "jnz     skip\n\t"
 6620             "movl    $dst, -1\n"
 6621       "skip:\n\t"
 6622             "negl    $dst\n\t"
 6623             "addl    $dst, 63" %}
 6624   ins_encode %{
 6625     Register Rdst = $dst$$Register;
 6626     Register Rsrc = $src$$Register;
 6627     Label skip;
 6628     __ bsrq(Rdst, Rsrc);
 6629     __ jccb(Assembler::notZero, skip);
 6630     __ movl(Rdst, -1);
 6631     __ bind(skip);
 6632     __ negl(Rdst);
 6633     __ addl(Rdst, BitsPerLong - 1);
 6634   %}
 6635   ins_pipe(ialu_reg);
 6636 %}
 6637 
 6638 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6639   predicate(UseCountTrailingZerosInstruction);
 6640   match(Set dst (CountTrailingZerosI src));
 6641   effect(KILL cr);
 6642 
 6643   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6644   ins_encode %{
 6645     __ tzcntl($dst$$Register, $src$$Register);
 6646   %}
 6647   ins_pipe(ialu_reg);
 6648 %}
 6649 
 6650 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6651   predicate(UseCountTrailingZerosInstruction);
 6652   match(Set dst (CountTrailingZerosI (LoadI src)));
 6653   effect(KILL cr);
 6654   ins_cost(175);
 6655   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6656   ins_encode %{
 6657     __ tzcntl($dst$$Register, $src$$Address);
 6658   %}
 6659   ins_pipe(ialu_reg_mem);
 6660 %}
 6661 
 6662 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6663   predicate(!UseCountTrailingZerosInstruction);
 6664   match(Set dst (CountTrailingZerosI src));
 6665   effect(KILL cr);
 6666 
 6667   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6668             "jnz     done\n\t"
 6669             "movl    $dst, 32\n"
 6670       "done:" %}
 6671   ins_encode %{
 6672     Register Rdst = $dst$$Register;
 6673     Label done;
 6674     __ bsfl(Rdst, $src$$Register);
 6675     __ jccb(Assembler::notZero, done);
 6676     __ movl(Rdst, BitsPerInt);
 6677     __ bind(done);
 6678   %}
 6679   ins_pipe(ialu_reg);
 6680 %}
 6681 
 6682 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6683   predicate(UseCountTrailingZerosInstruction);
 6684   match(Set dst (CountTrailingZerosL src));
 6685   effect(KILL cr);
 6686 
 6687   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6688   ins_encode %{
 6689     __ tzcntq($dst$$Register, $src$$Register);
 6690   %}
 6691   ins_pipe(ialu_reg);
 6692 %}
 6693 
 6694 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6695   predicate(UseCountTrailingZerosInstruction);
 6696   match(Set dst (CountTrailingZerosL (LoadL src)));
 6697   effect(KILL cr);
 6698   ins_cost(175);
 6699   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6700   ins_encode %{
 6701     __ tzcntq($dst$$Register, $src$$Address);
 6702   %}
 6703   ins_pipe(ialu_reg_mem);
 6704 %}
 6705 
 6706 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6707   predicate(!UseCountTrailingZerosInstruction);
 6708   match(Set dst (CountTrailingZerosL src));
 6709   effect(KILL cr);
 6710 
 6711   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6712             "jnz     done\n\t"
 6713             "movl    $dst, 64\n"
 6714       "done:" %}
 6715   ins_encode %{
 6716     Register Rdst = $dst$$Register;
 6717     Label done;
 6718     __ bsfq(Rdst, $src$$Register);
 6719     __ jccb(Assembler::notZero, done);
 6720     __ movl(Rdst, BitsPerLong);
 6721     __ bind(done);
 6722   %}
 6723   ins_pipe(ialu_reg);
 6724 %}
 6725 
 6726 
 6727 //---------- Population Count Instructions -------------------------------------
 6728 
 6729 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6730   predicate(UsePopCountInstruction);
 6731   match(Set dst (PopCountI src));
 6732   effect(KILL cr);
 6733 
 6734   format %{ "popcnt  $dst, $src" %}
 6735   ins_encode %{
 6736     __ popcntl($dst$$Register, $src$$Register);
 6737   %}
 6738   ins_pipe(ialu_reg);
 6739 %}
 6740 
 6741 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6742   predicate(UsePopCountInstruction);
 6743   match(Set dst (PopCountI (LoadI mem)));
 6744   effect(KILL cr);
 6745 
 6746   format %{ "popcnt  $dst, $mem" %}
 6747   ins_encode %{
 6748     __ popcntl($dst$$Register, $mem$$Address);
 6749   %}
 6750   ins_pipe(ialu_reg);
 6751 %}
 6752 
 6753 // Note: Long.bitCount(long) returns an int.
 6754 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6755   predicate(UsePopCountInstruction);
 6756   match(Set dst (PopCountL src));
 6757   effect(KILL cr);
 6758 
 6759   format %{ "popcnt  $dst, $src" %}
 6760   ins_encode %{
 6761     __ popcntq($dst$$Register, $src$$Register);
 6762   %}
 6763   ins_pipe(ialu_reg);
 6764 %}
 6765 
 6766 // Note: Long.bitCount(long) returns an int.
 6767 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6768   predicate(UsePopCountInstruction);
 6769   match(Set dst (PopCountL (LoadL mem)));
 6770   effect(KILL cr);
 6771 
 6772   format %{ "popcnt  $dst, $mem" %}
 6773   ins_encode %{
 6774     __ popcntq($dst$$Register, $mem$$Address);
 6775   %}
 6776   ins_pipe(ialu_reg);
 6777 %}
 6778 
 6779 
 6780 //----------MemBar Instructions-----------------------------------------------
 6781 // Memory barrier flavors
 6782 
 6783 instruct membar_acquire()
 6784 %{
 6785   match(MemBarAcquire);
 6786   match(LoadFence);
 6787   ins_cost(0);
 6788 
 6789   size(0);
 6790   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6791   ins_encode();
 6792   ins_pipe(empty);
 6793 %}
 6794 
 6795 instruct membar_acquire_lock()
 6796 %{
 6797   match(MemBarAcquireLock);
 6798   ins_cost(0);
 6799 
 6800   size(0);
 6801   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6802   ins_encode();
 6803   ins_pipe(empty);
 6804 %}
 6805 
 6806 instruct membar_release()
 6807 %{
 6808   match(MemBarRelease);
 6809   match(StoreFence);
 6810   ins_cost(0);
 6811 
 6812   size(0);
 6813   format %{ "MEMBAR-release ! (empty encoding)" %}
 6814   ins_encode();
 6815   ins_pipe(empty);
 6816 %}
 6817 
 6818 instruct membar_release_lock()
 6819 %{
 6820   match(MemBarReleaseLock);
 6821   ins_cost(0);
 6822 
 6823   size(0);
 6824   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6825   ins_encode();
 6826   ins_pipe(empty);
 6827 %}
 6828 
 6829 instruct membar_volatile(rFlagsReg cr) %{
 6830   match(MemBarVolatile);
 6831   effect(KILL cr);
 6832   ins_cost(400);
 6833 
 6834   format %{
 6835     $$template
 6836     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6837   %}
 6838   ins_encode %{
 6839     __ membar(Assembler::StoreLoad);
 6840   %}
 6841   ins_pipe(pipe_slow);
 6842 %}
 6843 
 6844 instruct unnecessary_membar_volatile()
 6845 %{
 6846   match(MemBarVolatile);
 6847   predicate(Matcher::post_store_load_barrier(n));
 6848   ins_cost(0);
 6849 
 6850   size(0);
 6851   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6852   ins_encode();
 6853   ins_pipe(empty);
 6854 %}
 6855 
 6856 instruct membar_storestore() %{
 6857   match(MemBarStoreStore);
 6858   match(StoreStoreFence);
 6859   ins_cost(0);
 6860 
 6861   size(0);
 6862   format %{ "MEMBAR-storestore (empty encoding)" %}
 6863   ins_encode( );
 6864   ins_pipe(empty);
 6865 %}
 6866 
 6867 //----------Move Instructions--------------------------------------------------
 6868 
 6869 instruct castX2P(rRegP dst, rRegL src)
 6870 %{
 6871   match(Set dst (CastX2P src));
 6872 
 6873   format %{ "movq    $dst, $src\t# long->ptr" %}
 6874   ins_encode %{
 6875     if ($dst$$reg != $src$$reg) {
 6876       __ movptr($dst$$Register, $src$$Register);
 6877     }
 6878   %}
 6879   ins_pipe(ialu_reg_reg); // XXX
 6880 %}
 6881 
 6882 instruct castP2X(rRegL dst, rRegP src)
 6883 %{
 6884   match(Set dst (CastP2X src));
 6885 
 6886   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6887   ins_encode %{
 6888     if ($dst$$reg != $src$$reg) {
 6889       __ movptr($dst$$Register, $src$$Register);
 6890     }
 6891   %}
 6892   ins_pipe(ialu_reg_reg); // XXX
 6893 %}
 6894 
 6895 // Convert oop into int for vectors alignment masking
 6896 instruct convP2I(rRegI dst, rRegP src)
 6897 %{
 6898   match(Set dst (ConvL2I (CastP2X src)));
 6899 
 6900   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6901   ins_encode %{
 6902     __ movl($dst$$Register, $src$$Register);
 6903   %}
 6904   ins_pipe(ialu_reg_reg); // XXX
 6905 %}
 6906 
 6907 // Convert compressed oop into int for vectors alignment masking
 6908 // in case of 32bit oops (heap < 4Gb).
 6909 instruct convN2I(rRegI dst, rRegN src)
 6910 %{
 6911   predicate(CompressedOops::shift() == 0);
 6912   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6913 
 6914   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6915   ins_encode %{
 6916     __ movl($dst$$Register, $src$$Register);
 6917   %}
 6918   ins_pipe(ialu_reg_reg); // XXX
 6919 %}
 6920 
 6921 // Convert oop pointer into compressed form
 6922 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6923   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6924   match(Set dst (EncodeP src));
 6925   effect(KILL cr);
 6926   format %{ "encode_heap_oop $dst,$src" %}
 6927   ins_encode %{
 6928     Register s = $src$$Register;
 6929     Register d = $dst$$Register;
 6930     if (s != d) {
 6931       __ movq(d, s);
 6932     }
 6933     __ encode_heap_oop(d);
 6934   %}
 6935   ins_pipe(ialu_reg_long);
 6936 %}
 6937 
 6938 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6939   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6940   match(Set dst (EncodeP src));
 6941   effect(KILL cr);
 6942   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6943   ins_encode %{
 6944     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6945   %}
 6946   ins_pipe(ialu_reg_long);
 6947 %}
 6948 
 6949 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6950   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6951             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6952   match(Set dst (DecodeN src));
 6953   effect(KILL cr);
 6954   format %{ "decode_heap_oop $dst,$src" %}
 6955   ins_encode %{
 6956     Register s = $src$$Register;
 6957     Register d = $dst$$Register;
 6958     if (s != d) {
 6959       __ movq(d, s);
 6960     }
 6961     __ decode_heap_oop(d);
 6962   %}
 6963   ins_pipe(ialu_reg_long);
 6964 %}
 6965 
 6966 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6967   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6968             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6969   match(Set dst (DecodeN src));
 6970   effect(KILL cr);
 6971   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6972   ins_encode %{
 6973     Register s = $src$$Register;
 6974     Register d = $dst$$Register;
 6975     if (s != d) {
 6976       __ decode_heap_oop_not_null(d, s);
 6977     } else {
 6978       __ decode_heap_oop_not_null(d);
 6979     }
 6980   %}
 6981   ins_pipe(ialu_reg_long);
 6982 %}
 6983 
 6984 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6985   match(Set dst (EncodePKlass src));
 6986   effect(TEMP dst, KILL cr);
 6987   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6988   ins_encode %{
 6989     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6990   %}
 6991   ins_pipe(ialu_reg_long);
 6992 %}
 6993 
 6994 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6995   match(Set dst (DecodeNKlass src));
 6996   effect(TEMP dst, KILL cr);
 6997   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6998   ins_encode %{
 6999     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7000   %}
 7001   ins_pipe(ialu_reg_long);
 7002 %}
 7003 
 7004 //----------Conditional Move---------------------------------------------------
 7005 // Jump
 7006 // dummy instruction for generating temp registers
 7007 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7008   match(Jump (LShiftL switch_val shift));
 7009   ins_cost(350);
 7010   predicate(false);
 7011   effect(TEMP dest);
 7012 
 7013   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7014             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7015   ins_encode %{
 7016     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7017     // to do that and the compiler is using that register as one it can allocate.
 7018     // So we build it all by hand.
 7019     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7020     // ArrayAddress dispatch(table, index);
 7021     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7022     __ lea($dest$$Register, $constantaddress);
 7023     __ jmp(dispatch);
 7024   %}
 7025   ins_pipe(pipe_jmp);
 7026 %}
 7027 
 7028 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7029   match(Jump (AddL (LShiftL switch_val shift) offset));
 7030   ins_cost(350);
 7031   effect(TEMP dest);
 7032 
 7033   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7034             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7035   ins_encode %{
 7036     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7037     // to do that and the compiler is using that register as one it can allocate.
 7038     // So we build it all by hand.
 7039     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7040     // ArrayAddress dispatch(table, index);
 7041     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7042     __ lea($dest$$Register, $constantaddress);
 7043     __ jmp(dispatch);
 7044   %}
 7045   ins_pipe(pipe_jmp);
 7046 %}
 7047 
 7048 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7049   match(Jump switch_val);
 7050   ins_cost(350);
 7051   effect(TEMP dest);
 7052 
 7053   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7054             "jmp     [$dest + $switch_val]\n\t" %}
 7055   ins_encode %{
 7056     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7057     // to do that and the compiler is using that register as one it can allocate.
 7058     // So we build it all by hand.
 7059     // Address index(noreg, switch_reg, Address::times_1);
 7060     // ArrayAddress dispatch(table, index);
 7061     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7062     __ lea($dest$$Register, $constantaddress);
 7063     __ jmp(dispatch);
 7064   %}
 7065   ins_pipe(pipe_jmp);
 7066 %}
 7067 
 7068 // Conditional move
 7069 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7070 %{
 7071   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7072   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7073 
 7074   ins_cost(100); // XXX
 7075   format %{ "setbn$cop $dst\t# signed, int" %}
 7076   ins_encode %{
 7077     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7078     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7079   %}
 7080   ins_pipe(ialu_reg);
 7081 %}
 7082 
 7083 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7084 %{
 7085   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7086 
 7087   ins_cost(200); // XXX
 7088   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7089   ins_encode %{
 7090     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7091   %}
 7092   ins_pipe(pipe_cmov_reg);
 7093 %}
 7094 
 7095 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7096 %{
 7097   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7098   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7099 
 7100   ins_cost(100); // XXX
 7101   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7102   ins_encode %{
 7103     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7104     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7105   %}
 7106   ins_pipe(ialu_reg);
 7107 %}
 7108 
 7109 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7110   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7111 
 7112   ins_cost(200); // XXX
 7113   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7114   ins_encode %{
 7115     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7116   %}
 7117   ins_pipe(pipe_cmov_reg);
 7118 %}
 7119 
 7120 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7121 %{
 7122   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7123   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7124 
 7125   ins_cost(100); // XXX
 7126   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7127   ins_encode %{
 7128     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7129     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7130   %}
 7131   ins_pipe(ialu_reg);
 7132 %}
 7133 
 7134 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7135   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7136   ins_cost(200);
 7137   expand %{
 7138     cmovI_regU(cop, cr, dst, src);
 7139   %}
 7140 %}
 7141 
 7142 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7143   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7144   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7145 
 7146   ins_cost(200); // XXX
 7147   format %{ "cmovpl  $dst, $src\n\t"
 7148             "cmovnel $dst, $src" %}
 7149   ins_encode %{
 7150     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7151     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7152   %}
 7153   ins_pipe(pipe_cmov_reg);
 7154 %}
 7155 
 7156 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7157 // inputs of the CMove
 7158 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7159   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7160   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7161 
 7162   ins_cost(200); // XXX
 7163   format %{ "cmovpl  $dst, $src\n\t"
 7164             "cmovnel $dst, $src" %}
 7165   ins_encode %{
 7166     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7167     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7168   %}
 7169   ins_pipe(pipe_cmov_reg);
 7170 %}
 7171 
 7172 // Conditional move
 7173 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7174   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7175 
 7176   ins_cost(250); // XXX
 7177   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7178   ins_encode %{
 7179     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7180   %}
 7181   ins_pipe(pipe_cmov_mem);
 7182 %}
 7183 
 7184 // Conditional move
 7185 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7186 %{
 7187   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7188 
 7189   ins_cost(250); // XXX
 7190   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7191   ins_encode %{
 7192     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7193   %}
 7194   ins_pipe(pipe_cmov_mem);
 7195 %}
 7196 
 7197 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7198   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7199   ins_cost(250);
 7200   expand %{
 7201     cmovI_memU(cop, cr, dst, src);
 7202   %}
 7203 %}
 7204 
 7205 // Conditional move
 7206 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7207 %{
 7208   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7209 
 7210   ins_cost(200); // XXX
 7211   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7212   ins_encode %{
 7213     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7214   %}
 7215   ins_pipe(pipe_cmov_reg);
 7216 %}
 7217 
 7218 // Conditional move
 7219 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7220 %{
 7221   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7222 
 7223   ins_cost(200); // XXX
 7224   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7225   ins_encode %{
 7226     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7227   %}
 7228   ins_pipe(pipe_cmov_reg);
 7229 %}
 7230 
 7231 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7232   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7233   ins_cost(200);
 7234   expand %{
 7235     cmovN_regU(cop, cr, dst, src);
 7236   %}
 7237 %}
 7238 
 7239 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7240   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7241   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7242 
 7243   ins_cost(200); // XXX
 7244   format %{ "cmovpl  $dst, $src\n\t"
 7245             "cmovnel $dst, $src" %}
 7246   ins_encode %{
 7247     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7248     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7249   %}
 7250   ins_pipe(pipe_cmov_reg);
 7251 %}
 7252 
 7253 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7254 // inputs of the CMove
 7255 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7256   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7257   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7258 
 7259   ins_cost(200); // XXX
 7260   format %{ "cmovpl  $dst, $src\n\t"
 7261             "cmovnel $dst, $src" %}
 7262   ins_encode %{
 7263     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7264     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7265   %}
 7266   ins_pipe(pipe_cmov_reg);
 7267 %}
 7268 
 7269 // Conditional move
 7270 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7271 %{
 7272   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7273 
 7274   ins_cost(200); // XXX
 7275   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7276   ins_encode %{
 7277     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7278   %}
 7279   ins_pipe(pipe_cmov_reg);  // XXX
 7280 %}
 7281 
 7282 // Conditional move
 7283 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7284 %{
 7285   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7286 
 7287   ins_cost(200); // XXX
 7288   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7289   ins_encode %{
 7290     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7291   %}
 7292   ins_pipe(pipe_cmov_reg); // XXX
 7293 %}
 7294 
 7295 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7296   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7297   ins_cost(200);
 7298   expand %{
 7299     cmovP_regU(cop, cr, dst, src);
 7300   %}
 7301 %}
 7302 
 7303 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7304   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7305   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7306 
 7307   ins_cost(200); // XXX
 7308   format %{ "cmovpq  $dst, $src\n\t"
 7309             "cmovneq $dst, $src" %}
 7310   ins_encode %{
 7311     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7312     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7313   %}
 7314   ins_pipe(pipe_cmov_reg);
 7315 %}
 7316 
 7317 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7318 // inputs of the CMove
 7319 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7320   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7321   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7322 
 7323   ins_cost(200); // XXX
 7324   format %{ "cmovpq  $dst, $src\n\t"
 7325             "cmovneq $dst, $src" %}
 7326   ins_encode %{
 7327     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7328     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7329   %}
 7330   ins_pipe(pipe_cmov_reg);
 7331 %}
 7332 
 7333 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7334 // correctly meets the two pointer arguments; one is an incoming
 7335 // register but the other is a memory operand.  ALSO appears to
 7336 // be buggy with implicit null checks.
 7337 //
 7338 //// Conditional move
 7339 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7340 //%{
 7341 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7342 //  ins_cost(250);
 7343 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7344 //  opcode(0x0F,0x40);
 7345 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7346 //  ins_pipe( pipe_cmov_mem );
 7347 //%}
 7348 //
 7349 //// Conditional move
 7350 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7351 //%{
 7352 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7353 //  ins_cost(250);
 7354 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7355 //  opcode(0x0F,0x40);
 7356 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7357 //  ins_pipe( pipe_cmov_mem );
 7358 //%}
 7359 
 7360 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7361 %{
 7362   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7363   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7364 
 7365   ins_cost(100); // XXX
 7366   format %{ "setbn$cop $dst\t# signed, long" %}
 7367   ins_encode %{
 7368     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7369     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7370   %}
 7371   ins_pipe(ialu_reg);
 7372 %}
 7373 
 7374 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7375 %{
 7376   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7377 
 7378   ins_cost(200); // XXX
 7379   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7380   ins_encode %{
 7381     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7382   %}
 7383   ins_pipe(pipe_cmov_reg);  // XXX
 7384 %}
 7385 
 7386 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7387 %{
 7388   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7389 
 7390   ins_cost(200); // XXX
 7391   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7392   ins_encode %{
 7393     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7394   %}
 7395   ins_pipe(pipe_cmov_mem);  // XXX
 7396 %}
 7397 
 7398 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7399 %{
 7400   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7401   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7402 
 7403   ins_cost(100); // XXX
 7404   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7405   ins_encode %{
 7406     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7407     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7408   %}
 7409   ins_pipe(ialu_reg);
 7410 %}
 7411 
 7412 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7413 %{
 7414   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7415 
 7416   ins_cost(200); // XXX
 7417   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7418   ins_encode %{
 7419     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7420   %}
 7421   ins_pipe(pipe_cmov_reg); // XXX
 7422 %}
 7423 
 7424 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7425 %{
 7426   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7427   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7428 
 7429   ins_cost(100); // XXX
 7430   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7431   ins_encode %{
 7432     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7433     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7434   %}
 7435   ins_pipe(ialu_reg);
 7436 %}
 7437 
 7438 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7439   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7440   ins_cost(200);
 7441   expand %{
 7442     cmovL_regU(cop, cr, dst, src);
 7443   %}
 7444 %}
 7445 
 7446 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7447   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7448   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7449 
 7450   ins_cost(200); // XXX
 7451   format %{ "cmovpq  $dst, $src\n\t"
 7452             "cmovneq $dst, $src" %}
 7453   ins_encode %{
 7454     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7455     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7456   %}
 7457   ins_pipe(pipe_cmov_reg);
 7458 %}
 7459 
 7460 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7461 // inputs of the CMove
 7462 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7463   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7464   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7465 
 7466   ins_cost(200); // XXX
 7467   format %{ "cmovpq  $dst, $src\n\t"
 7468             "cmovneq $dst, $src" %}
 7469   ins_encode %{
 7470     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7471     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7472   %}
 7473   ins_pipe(pipe_cmov_reg);
 7474 %}
 7475 
 7476 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7477 %{
 7478   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7479 
 7480   ins_cost(200); // XXX
 7481   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7482   ins_encode %{
 7483     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7484   %}
 7485   ins_pipe(pipe_cmov_mem); // XXX
 7486 %}
 7487 
 7488 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7489   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7490   ins_cost(200);
 7491   expand %{
 7492     cmovL_memU(cop, cr, dst, src);
 7493   %}
 7494 %}
 7495 
 7496 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7497 %{
 7498   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7499 
 7500   ins_cost(200); // XXX
 7501   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7502             "movss     $dst, $src\n"
 7503     "skip:" %}
 7504   ins_encode %{
 7505     Label Lskip;
 7506     // Invert sense of branch from sense of CMOV
 7507     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7508     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7509     __ bind(Lskip);
 7510   %}
 7511   ins_pipe(pipe_slow);
 7512 %}
 7513 
 7514 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7515 // %{
 7516 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7517 
 7518 //   ins_cost(200); // XXX
 7519 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7520 //             "movss     $dst, $src\n"
 7521 //     "skip:" %}
 7522 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7523 //   ins_pipe(pipe_slow);
 7524 // %}
 7525 
 7526 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7527 %{
 7528   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7529 
 7530   ins_cost(200); // XXX
 7531   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7532             "movss     $dst, $src\n"
 7533     "skip:" %}
 7534   ins_encode %{
 7535     Label Lskip;
 7536     // Invert sense of branch from sense of CMOV
 7537     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7538     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7539     __ bind(Lskip);
 7540   %}
 7541   ins_pipe(pipe_slow);
 7542 %}
 7543 
 7544 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7545   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7546   ins_cost(200);
 7547   expand %{
 7548     cmovF_regU(cop, cr, dst, src);
 7549   %}
 7550 %}
 7551 
 7552 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7553 %{
 7554   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7555 
 7556   ins_cost(200); // XXX
 7557   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7558             "movsd     $dst, $src\n"
 7559     "skip:" %}
 7560   ins_encode %{
 7561     Label Lskip;
 7562     // Invert sense of branch from sense of CMOV
 7563     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7564     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7565     __ bind(Lskip);
 7566   %}
 7567   ins_pipe(pipe_slow);
 7568 %}
 7569 
 7570 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7571 %{
 7572   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7573 
 7574   ins_cost(200); // XXX
 7575   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7576             "movsd     $dst, $src\n"
 7577     "skip:" %}
 7578   ins_encode %{
 7579     Label Lskip;
 7580     // Invert sense of branch from sense of CMOV
 7581     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7582     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7583     __ bind(Lskip);
 7584   %}
 7585   ins_pipe(pipe_slow);
 7586 %}
 7587 
 7588 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7589   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7590   ins_cost(200);
 7591   expand %{
 7592     cmovD_regU(cop, cr, dst, src);
 7593   %}
 7594 %}
 7595 
 7596 //----------Arithmetic Instructions--------------------------------------------
 7597 //----------Addition Instructions----------------------------------------------
 7598 
 7599 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7600 %{
 7601   match(Set dst (AddI dst src));
 7602   effect(KILL cr);
 7603 
 7604   format %{ "addl    $dst, $src\t# int" %}
 7605   ins_encode %{
 7606     __ addl($dst$$Register, $src$$Register);
 7607   %}
 7608   ins_pipe(ialu_reg_reg);
 7609 %}
 7610 
 7611 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7612 %{
 7613   match(Set dst (AddI dst src));
 7614   effect(KILL cr);
 7615 
 7616   format %{ "addl    $dst, $src\t# int" %}
 7617   ins_encode %{
 7618     __ addl($dst$$Register, $src$$constant);
 7619   %}
 7620   ins_pipe( ialu_reg );
 7621 %}
 7622 
 7623 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7624 %{
 7625   match(Set dst (AddI dst (LoadI src)));
 7626   effect(KILL cr);
 7627 
 7628   ins_cost(125); // XXX
 7629   format %{ "addl    $dst, $src\t# int" %}
 7630   ins_encode %{
 7631     __ addl($dst$$Register, $src$$Address);
 7632   %}
 7633   ins_pipe(ialu_reg_mem);
 7634 %}
 7635 
 7636 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7637 %{
 7638   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7639   effect(KILL cr);
 7640 
 7641   ins_cost(150); // XXX
 7642   format %{ "addl    $dst, $src\t# int" %}
 7643   ins_encode %{
 7644     __ addl($dst$$Address, $src$$Register);
 7645   %}
 7646   ins_pipe(ialu_mem_reg);
 7647 %}
 7648 
 7649 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7650 %{
 7651   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7652   effect(KILL cr);
 7653 
 7654   ins_cost(125); // XXX
 7655   format %{ "addl    $dst, $src\t# int" %}
 7656   ins_encode %{
 7657     __ addl($dst$$Address, $src$$constant);
 7658   %}
 7659   ins_pipe(ialu_mem_imm);
 7660 %}
 7661 
 7662 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7663 %{
 7664   predicate(UseIncDec);
 7665   match(Set dst (AddI dst src));
 7666   effect(KILL cr);
 7667 
 7668   format %{ "incl    $dst\t# int" %}
 7669   ins_encode %{
 7670     __ incrementl($dst$$Register);
 7671   %}
 7672   ins_pipe(ialu_reg);
 7673 %}
 7674 
 7675 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7676 %{
 7677   predicate(UseIncDec);
 7678   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7679   effect(KILL cr);
 7680 
 7681   ins_cost(125); // XXX
 7682   format %{ "incl    $dst\t# int" %}
 7683   ins_encode %{
 7684     __ incrementl($dst$$Address);
 7685   %}
 7686   ins_pipe(ialu_mem_imm);
 7687 %}
 7688 
 7689 // XXX why does that use AddI
 7690 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7691 %{
 7692   predicate(UseIncDec);
 7693   match(Set dst (AddI dst src));
 7694   effect(KILL cr);
 7695 
 7696   format %{ "decl    $dst\t# int" %}
 7697   ins_encode %{
 7698     __ decrementl($dst$$Register);
 7699   %}
 7700   ins_pipe(ialu_reg);
 7701 %}
 7702 
 7703 // XXX why does that use AddI
 7704 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7705 %{
 7706   predicate(UseIncDec);
 7707   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7708   effect(KILL cr);
 7709 
 7710   ins_cost(125); // XXX
 7711   format %{ "decl    $dst\t# int" %}
 7712   ins_encode %{
 7713     __ decrementl($dst$$Address);
 7714   %}
 7715   ins_pipe(ialu_mem_imm);
 7716 %}
 7717 
 7718 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7719 %{
 7720   predicate(VM_Version::supports_fast_2op_lea());
 7721   match(Set dst (AddI (LShiftI index scale) disp));
 7722 
 7723   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7724   ins_encode %{
 7725     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7726     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7727   %}
 7728   ins_pipe(ialu_reg_reg);
 7729 %}
 7730 
 7731 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7732 %{
 7733   predicate(VM_Version::supports_fast_3op_lea());
 7734   match(Set dst (AddI (AddI base index) disp));
 7735 
 7736   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7737   ins_encode %{
 7738     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7739   %}
 7740   ins_pipe(ialu_reg_reg);
 7741 %}
 7742 
 7743 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7744 %{
 7745   predicate(VM_Version::supports_fast_2op_lea());
 7746   match(Set dst (AddI base (LShiftI index scale)));
 7747 
 7748   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7749   ins_encode %{
 7750     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7751     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7752   %}
 7753   ins_pipe(ialu_reg_reg);
 7754 %}
 7755 
 7756 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7757 %{
 7758   predicate(VM_Version::supports_fast_3op_lea());
 7759   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7760 
 7761   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7762   ins_encode %{
 7763     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7764     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7765   %}
 7766   ins_pipe(ialu_reg_reg);
 7767 %}
 7768 
 7769 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7770 %{
 7771   match(Set dst (AddL dst src));
 7772   effect(KILL cr);
 7773 
 7774   format %{ "addq    $dst, $src\t# long" %}
 7775   ins_encode %{
 7776     __ addq($dst$$Register, $src$$Register);
 7777   %}
 7778   ins_pipe(ialu_reg_reg);
 7779 %}
 7780 
 7781 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7782 %{
 7783   match(Set dst (AddL dst src));
 7784   effect(KILL cr);
 7785 
 7786   format %{ "addq    $dst, $src\t# long" %}
 7787   ins_encode %{
 7788     __ addq($dst$$Register, $src$$constant);
 7789   %}
 7790   ins_pipe( ialu_reg );
 7791 %}
 7792 
 7793 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7794 %{
 7795   match(Set dst (AddL dst (LoadL src)));
 7796   effect(KILL cr);
 7797 
 7798   ins_cost(125); // XXX
 7799   format %{ "addq    $dst, $src\t# long" %}
 7800   ins_encode %{
 7801     __ addq($dst$$Register, $src$$Address);
 7802   %}
 7803   ins_pipe(ialu_reg_mem);
 7804 %}
 7805 
 7806 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7807 %{
 7808   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7809   effect(KILL cr);
 7810 
 7811   ins_cost(150); // XXX
 7812   format %{ "addq    $dst, $src\t# long" %}
 7813   ins_encode %{
 7814     __ addq($dst$$Address, $src$$Register);
 7815   %}
 7816   ins_pipe(ialu_mem_reg);
 7817 %}
 7818 
 7819 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7820 %{
 7821   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7822   effect(KILL cr);
 7823 
 7824   ins_cost(125); // XXX
 7825   format %{ "addq    $dst, $src\t# long" %}
 7826   ins_encode %{
 7827     __ addq($dst$$Address, $src$$constant);
 7828   %}
 7829   ins_pipe(ialu_mem_imm);
 7830 %}
 7831 
 7832 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7833 %{
 7834   predicate(UseIncDec);
 7835   match(Set dst (AddL dst src));
 7836   effect(KILL cr);
 7837 
 7838   format %{ "incq    $dst\t# long" %}
 7839   ins_encode %{
 7840     __ incrementq($dst$$Register);
 7841   %}
 7842   ins_pipe(ialu_reg);
 7843 %}
 7844 
 7845 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7846 %{
 7847   predicate(UseIncDec);
 7848   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7849   effect(KILL cr);
 7850 
 7851   ins_cost(125); // XXX
 7852   format %{ "incq    $dst\t# long" %}
 7853   ins_encode %{
 7854     __ incrementq($dst$$Address);
 7855   %}
 7856   ins_pipe(ialu_mem_imm);
 7857 %}
 7858 
 7859 // XXX why does that use AddL
 7860 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7861 %{
 7862   predicate(UseIncDec);
 7863   match(Set dst (AddL dst src));
 7864   effect(KILL cr);
 7865 
 7866   format %{ "decq    $dst\t# long" %}
 7867   ins_encode %{
 7868     __ decrementq($dst$$Register);
 7869   %}
 7870   ins_pipe(ialu_reg);
 7871 %}
 7872 
 7873 // XXX why does that use AddL
 7874 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7875 %{
 7876   predicate(UseIncDec);
 7877   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7878   effect(KILL cr);
 7879 
 7880   ins_cost(125); // XXX
 7881   format %{ "decq    $dst\t# long" %}
 7882   ins_encode %{
 7883     __ decrementq($dst$$Address);
 7884   %}
 7885   ins_pipe(ialu_mem_imm);
 7886 %}
 7887 
 7888 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7889 %{
 7890   predicate(VM_Version::supports_fast_2op_lea());
 7891   match(Set dst (AddL (LShiftL index scale) disp));
 7892 
 7893   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7894   ins_encode %{
 7895     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7896     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7897   %}
 7898   ins_pipe(ialu_reg_reg);
 7899 %}
 7900 
 7901 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7902 %{
 7903   predicate(VM_Version::supports_fast_3op_lea());
 7904   match(Set dst (AddL (AddL base index) disp));
 7905 
 7906   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7907   ins_encode %{
 7908     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7909   %}
 7910   ins_pipe(ialu_reg_reg);
 7911 %}
 7912 
 7913 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7914 %{
 7915   predicate(VM_Version::supports_fast_2op_lea());
 7916   match(Set dst (AddL base (LShiftL index scale)));
 7917 
 7918   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7919   ins_encode %{
 7920     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7921     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7922   %}
 7923   ins_pipe(ialu_reg_reg);
 7924 %}
 7925 
 7926 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7927 %{
 7928   predicate(VM_Version::supports_fast_3op_lea());
 7929   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7930 
 7931   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7932   ins_encode %{
 7933     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7934     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7935   %}
 7936   ins_pipe(ialu_reg_reg);
 7937 %}
 7938 
 7939 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7940 %{
 7941   match(Set dst (AddP dst src));
 7942   effect(KILL cr);
 7943 
 7944   format %{ "addq    $dst, $src\t# ptr" %}
 7945   ins_encode %{
 7946     __ addq($dst$$Register, $src$$Register);
 7947   %}
 7948   ins_pipe(ialu_reg_reg);
 7949 %}
 7950 
 7951 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7952 %{
 7953   match(Set dst (AddP dst src));
 7954   effect(KILL cr);
 7955 
 7956   format %{ "addq    $dst, $src\t# ptr" %}
 7957   ins_encode %{
 7958     __ addq($dst$$Register, $src$$constant);
 7959   %}
 7960   ins_pipe( ialu_reg );
 7961 %}
 7962 
 7963 // XXX addP mem ops ????
 7964 
 7965 instruct checkCastPP(rRegP dst)
 7966 %{
 7967   match(Set dst (CheckCastPP dst));
 7968 
 7969   size(0);
 7970   format %{ "# checkcastPP of $dst" %}
 7971   ins_encode(/* empty encoding */);
 7972   ins_pipe(empty);
 7973 %}
 7974 
 7975 instruct castPP(rRegP dst)
 7976 %{
 7977   match(Set dst (CastPP dst));
 7978 
 7979   size(0);
 7980   format %{ "# castPP of $dst" %}
 7981   ins_encode(/* empty encoding */);
 7982   ins_pipe(empty);
 7983 %}
 7984 
 7985 instruct castII(rRegI dst)
 7986 %{
 7987   match(Set dst (CastII dst));
 7988 
 7989   size(0);
 7990   format %{ "# castII of $dst" %}
 7991   ins_encode(/* empty encoding */);
 7992   ins_cost(0);
 7993   ins_pipe(empty);
 7994 %}
 7995 
 7996 instruct castLL(rRegL dst)
 7997 %{
 7998   match(Set dst (CastLL dst));
 7999 
 8000   size(0);
 8001   format %{ "# castLL of $dst" %}
 8002   ins_encode(/* empty encoding */);
 8003   ins_cost(0);
 8004   ins_pipe(empty);
 8005 %}
 8006 
 8007 instruct castFF(regF dst)
 8008 %{
 8009   match(Set dst (CastFF dst));
 8010 
 8011   size(0);
 8012   format %{ "# castFF of $dst" %}
 8013   ins_encode(/* empty encoding */);
 8014   ins_cost(0);
 8015   ins_pipe(empty);
 8016 %}
 8017 
 8018 instruct castDD(regD dst)
 8019 %{
 8020   match(Set dst (CastDD dst));
 8021 
 8022   size(0);
 8023   format %{ "# castDD of $dst" %}
 8024   ins_encode(/* empty encoding */);
 8025   ins_cost(0);
 8026   ins_pipe(empty);
 8027 %}
 8028 
 8029 // LoadP-locked same as a regular LoadP when used with compare-swap
 8030 instruct loadPLocked(rRegP dst, memory mem)
 8031 %{
 8032   match(Set dst (LoadPLocked mem));
 8033 
 8034   ins_cost(125); // XXX
 8035   format %{ "movq    $dst, $mem\t# ptr locked" %}
 8036   ins_encode %{
 8037     __ movq($dst$$Register, $mem$$Address);
 8038   %}
 8039   ins_pipe(ialu_reg_mem); // XXX
 8040 %}
 8041 
 8042 // Conditional-store of the updated heap-top.
 8043 // Used during allocation of the shared heap.
 8044 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 8045 
 8046 instruct storePConditional(memory heap_top_ptr,
 8047                            rax_RegP oldval, rRegP newval,
 8048                            rFlagsReg cr)
 8049 %{
 8050   predicate(n->as_LoadStore()->barrier_data() == 0);
 8051   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 8052 
 8053   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 8054             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 8055   ins_encode %{
 8056     __ lock();
 8057     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 8058   %}
 8059   ins_pipe(pipe_cmpxchg);
 8060 %}
 8061 
 8062 // Conditional-store of an int value.
 8063 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 8064 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 8065 %{
 8066   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 8067   effect(KILL oldval);
 8068 
 8069   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 8070   opcode(0x0F, 0xB1);
 8071   ins_encode(lock_prefix,
 8072              REX_reg_mem(newval, mem),
 8073              OpcP, OpcS,
 8074              reg_mem(newval, mem));
 8075   ins_pipe(pipe_cmpxchg);
 8076 %}
 8077 
 8078 // Conditional-store of a long value.
 8079 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 8080 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 8081 %{
 8082   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 8083   effect(KILL oldval);
 8084 
 8085   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 8086   ins_encode %{
 8087     __ lock();
 8088     __ cmpxchgq($newval$$Register, $mem$$Address);
 8089   %}
 8090   ins_pipe(pipe_cmpxchg);
 8091 %}
 8092 
 8093 
 8094 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8095 instruct compareAndSwapP(rRegI res,
 8096                          memory mem_ptr,
 8097                          rax_RegP oldval, rRegP newval,
 8098                          rFlagsReg cr)
 8099 %{
 8100   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8101   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8102   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8103   effect(KILL cr, KILL oldval);
 8104 
 8105   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8106             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8107             "sete    $res\n\t"
 8108             "movzbl  $res, $res" %}
 8109   ins_encode %{
 8110     __ lock();
 8111     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8112     __ sete($res$$Register);
 8113     __ movzbl($res$$Register, $res$$Register);
 8114   %}
 8115   ins_pipe( pipe_cmpxchg );
 8116 %}
 8117 
 8118 instruct compareAndSwapL(rRegI res,
 8119                          memory mem_ptr,
 8120                          rax_RegL oldval, rRegL newval,
 8121                          rFlagsReg cr)
 8122 %{
 8123   predicate(VM_Version::supports_cx8());
 8124   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8125   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8126   effect(KILL cr, KILL oldval);
 8127 
 8128   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8129             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8130             "sete    $res\n\t"
 8131             "movzbl  $res, $res" %}
 8132   ins_encode %{
 8133     __ lock();
 8134     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8135     __ sete($res$$Register);
 8136     __ movzbl($res$$Register, $res$$Register);
 8137   %}
 8138   ins_pipe( pipe_cmpxchg );
 8139 %}
 8140 
 8141 instruct compareAndSwapI(rRegI res,
 8142                          memory mem_ptr,
 8143                          rax_RegI oldval, rRegI newval,
 8144                          rFlagsReg cr)
 8145 %{
 8146   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8147   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8148   effect(KILL cr, KILL oldval);
 8149 
 8150   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8151             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8152             "sete    $res\n\t"
 8153             "movzbl  $res, $res" %}
 8154   ins_encode %{
 8155     __ lock();
 8156     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8157     __ sete($res$$Register);
 8158     __ movzbl($res$$Register, $res$$Register);
 8159   %}
 8160   ins_pipe( pipe_cmpxchg );
 8161 %}
 8162 
 8163 instruct compareAndSwapB(rRegI res,
 8164                          memory mem_ptr,
 8165                          rax_RegI oldval, rRegI newval,
 8166                          rFlagsReg cr)
 8167 %{
 8168   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8169   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8170   effect(KILL cr, KILL oldval);
 8171 
 8172   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8173             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8174             "sete    $res\n\t"
 8175             "movzbl  $res, $res" %}
 8176   ins_encode %{
 8177     __ lock();
 8178     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8179     __ sete($res$$Register);
 8180     __ movzbl($res$$Register, $res$$Register);
 8181   %}
 8182   ins_pipe( pipe_cmpxchg );
 8183 %}
 8184 
 8185 instruct compareAndSwapS(rRegI res,
 8186                          memory mem_ptr,
 8187                          rax_RegI oldval, rRegI newval,
 8188                          rFlagsReg cr)
 8189 %{
 8190   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8191   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8192   effect(KILL cr, KILL oldval);
 8193 
 8194   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8195             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8196             "sete    $res\n\t"
 8197             "movzbl  $res, $res" %}
 8198   ins_encode %{
 8199     __ lock();
 8200     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8201     __ sete($res$$Register);
 8202     __ movzbl($res$$Register, $res$$Register);
 8203   %}
 8204   ins_pipe( pipe_cmpxchg );
 8205 %}
 8206 
 8207 instruct compareAndSwapN(rRegI res,
 8208                           memory mem_ptr,
 8209                           rax_RegN oldval, rRegN newval,
 8210                           rFlagsReg cr) %{
 8211   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8212   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8213   effect(KILL cr, KILL oldval);
 8214 
 8215   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8216             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8217             "sete    $res\n\t"
 8218             "movzbl  $res, $res" %}
 8219   ins_encode %{
 8220     __ lock();
 8221     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8222     __ sete($res$$Register);
 8223     __ movzbl($res$$Register, $res$$Register);
 8224   %}
 8225   ins_pipe( pipe_cmpxchg );
 8226 %}
 8227 
 8228 instruct compareAndExchangeB(
 8229                          memory mem_ptr,
 8230                          rax_RegI oldval, rRegI newval,
 8231                          rFlagsReg cr)
 8232 %{
 8233   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8234   effect(KILL cr);
 8235 
 8236   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8237             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8238   ins_encode %{
 8239     __ lock();
 8240     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8241   %}
 8242   ins_pipe( pipe_cmpxchg );
 8243 %}
 8244 
 8245 instruct compareAndExchangeS(
 8246                          memory mem_ptr,
 8247                          rax_RegI oldval, rRegI newval,
 8248                          rFlagsReg cr)
 8249 %{
 8250   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8251   effect(KILL cr);
 8252 
 8253   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8254             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8255   ins_encode %{
 8256     __ lock();
 8257     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8258   %}
 8259   ins_pipe( pipe_cmpxchg );
 8260 %}
 8261 
 8262 instruct compareAndExchangeI(
 8263                          memory mem_ptr,
 8264                          rax_RegI oldval, rRegI newval,
 8265                          rFlagsReg cr)
 8266 %{
 8267   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8268   effect(KILL cr);
 8269 
 8270   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8271             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8272   ins_encode %{
 8273     __ lock();
 8274     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8275   %}
 8276   ins_pipe( pipe_cmpxchg );
 8277 %}
 8278 
 8279 instruct compareAndExchangeL(
 8280                          memory mem_ptr,
 8281                          rax_RegL oldval, rRegL newval,
 8282                          rFlagsReg cr)
 8283 %{
 8284   predicate(VM_Version::supports_cx8());
 8285   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8286   effect(KILL cr);
 8287 
 8288   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8289             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8290   ins_encode %{
 8291     __ lock();
 8292     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8293   %}
 8294   ins_pipe( pipe_cmpxchg );
 8295 %}
 8296 
 8297 instruct compareAndExchangeN(
 8298                           memory mem_ptr,
 8299                           rax_RegN oldval, rRegN newval,
 8300                           rFlagsReg cr) %{
 8301   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8302   effect(KILL cr);
 8303 
 8304   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8305             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8306   ins_encode %{
 8307     __ lock();
 8308     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8309   %}
 8310   ins_pipe( pipe_cmpxchg );
 8311 %}
 8312 
 8313 instruct compareAndExchangeP(
 8314                          memory mem_ptr,
 8315                          rax_RegP oldval, rRegP newval,
 8316                          rFlagsReg cr)
 8317 %{
 8318   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8319   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8320   effect(KILL cr);
 8321 
 8322   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8323             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8324   ins_encode %{
 8325     __ lock();
 8326     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8327   %}
 8328   ins_pipe( pipe_cmpxchg );
 8329 %}
 8330 
 8331 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8332   predicate(n->as_LoadStore()->result_not_used());
 8333   match(Set dummy (GetAndAddB mem add));
 8334   effect(KILL cr);
 8335   format %{ "ADDB  [$mem],$add" %}
 8336   ins_encode %{
 8337     __ lock();
 8338     __ addb($mem$$Address, $add$$constant);
 8339   %}
 8340   ins_pipe( pipe_cmpxchg );
 8341 %}
 8342 
 8343 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8344   match(Set newval (GetAndAddB mem newval));
 8345   effect(KILL cr);
 8346   format %{ "XADDB  [$mem],$newval" %}
 8347   ins_encode %{
 8348     __ lock();
 8349     __ xaddb($mem$$Address, $newval$$Register);
 8350   %}
 8351   ins_pipe( pipe_cmpxchg );
 8352 %}
 8353 
 8354 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8355   predicate(n->as_LoadStore()->result_not_used());
 8356   match(Set dummy (GetAndAddS mem add));
 8357   effect(KILL cr);
 8358   format %{ "ADDW  [$mem],$add" %}
 8359   ins_encode %{
 8360     __ lock();
 8361     __ addw($mem$$Address, $add$$constant);
 8362   %}
 8363   ins_pipe( pipe_cmpxchg );
 8364 %}
 8365 
 8366 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8367   match(Set newval (GetAndAddS mem newval));
 8368   effect(KILL cr);
 8369   format %{ "XADDW  [$mem],$newval" %}
 8370   ins_encode %{
 8371     __ lock();
 8372     __ xaddw($mem$$Address, $newval$$Register);
 8373   %}
 8374   ins_pipe( pipe_cmpxchg );
 8375 %}
 8376 
 8377 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8378   predicate(n->as_LoadStore()->result_not_used());
 8379   match(Set dummy (GetAndAddI mem add));
 8380   effect(KILL cr);
 8381   format %{ "ADDL  [$mem],$add" %}
 8382   ins_encode %{
 8383     __ lock();
 8384     __ addl($mem$$Address, $add$$constant);
 8385   %}
 8386   ins_pipe( pipe_cmpxchg );
 8387 %}
 8388 
 8389 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8390   match(Set newval (GetAndAddI mem newval));
 8391   effect(KILL cr);
 8392   format %{ "XADDL  [$mem],$newval" %}
 8393   ins_encode %{
 8394     __ lock();
 8395     __ xaddl($mem$$Address, $newval$$Register);
 8396   %}
 8397   ins_pipe( pipe_cmpxchg );
 8398 %}
 8399 
 8400 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8401   predicate(n->as_LoadStore()->result_not_used());
 8402   match(Set dummy (GetAndAddL mem add));
 8403   effect(KILL cr);
 8404   format %{ "ADDQ  [$mem],$add" %}
 8405   ins_encode %{
 8406     __ lock();
 8407     __ addq($mem$$Address, $add$$constant);
 8408   %}
 8409   ins_pipe( pipe_cmpxchg );
 8410 %}
 8411 
 8412 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8413   match(Set newval (GetAndAddL mem newval));
 8414   effect(KILL cr);
 8415   format %{ "XADDQ  [$mem],$newval" %}
 8416   ins_encode %{
 8417     __ lock();
 8418     __ xaddq($mem$$Address, $newval$$Register);
 8419   %}
 8420   ins_pipe( pipe_cmpxchg );
 8421 %}
 8422 
 8423 instruct xchgB( memory mem, rRegI newval) %{
 8424   match(Set newval (GetAndSetB mem newval));
 8425   format %{ "XCHGB  $newval,[$mem]" %}
 8426   ins_encode %{
 8427     __ xchgb($newval$$Register, $mem$$Address);
 8428   %}
 8429   ins_pipe( pipe_cmpxchg );
 8430 %}
 8431 
 8432 instruct xchgS( memory mem, rRegI newval) %{
 8433   match(Set newval (GetAndSetS mem newval));
 8434   format %{ "XCHGW  $newval,[$mem]" %}
 8435   ins_encode %{
 8436     __ xchgw($newval$$Register, $mem$$Address);
 8437   %}
 8438   ins_pipe( pipe_cmpxchg );
 8439 %}
 8440 
 8441 instruct xchgI( memory mem, rRegI newval) %{
 8442   match(Set newval (GetAndSetI mem newval));
 8443   format %{ "XCHGL  $newval,[$mem]" %}
 8444   ins_encode %{
 8445     __ xchgl($newval$$Register, $mem$$Address);
 8446   %}
 8447   ins_pipe( pipe_cmpxchg );
 8448 %}
 8449 
 8450 instruct xchgL( memory mem, rRegL newval) %{
 8451   match(Set newval (GetAndSetL mem newval));
 8452   format %{ "XCHGL  $newval,[$mem]" %}
 8453   ins_encode %{
 8454     __ xchgq($newval$$Register, $mem$$Address);
 8455   %}
 8456   ins_pipe( pipe_cmpxchg );
 8457 %}
 8458 
 8459 instruct xchgP( memory mem, rRegP newval) %{
 8460   match(Set newval (GetAndSetP mem newval));
 8461   predicate(n->as_LoadStore()->barrier_data() == 0);
 8462   format %{ "XCHGQ  $newval,[$mem]" %}
 8463   ins_encode %{
 8464     __ xchgq($newval$$Register, $mem$$Address);
 8465   %}
 8466   ins_pipe( pipe_cmpxchg );
 8467 %}
 8468 
 8469 instruct xchgN( memory mem, rRegN newval) %{
 8470   match(Set newval (GetAndSetN mem newval));
 8471   format %{ "XCHGL  $newval,$mem]" %}
 8472   ins_encode %{
 8473     __ xchgl($newval$$Register, $mem$$Address);
 8474   %}
 8475   ins_pipe( pipe_cmpxchg );
 8476 %}
 8477 
 8478 //----------Abs Instructions-------------------------------------------
 8479 
 8480 // Integer Absolute Instructions
 8481 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8482 %{
 8483   match(Set dst (AbsI src));
 8484   effect(TEMP dst, TEMP tmp, KILL cr);
 8485   format %{ "movl $tmp, $src\n\t"
 8486             "sarl $tmp, 31\n\t"
 8487             "movl $dst, $src\n\t"
 8488             "xorl $dst, $tmp\n\t"
 8489             "subl $dst, $tmp\n"
 8490           %}
 8491   ins_encode %{
 8492     __ movl($tmp$$Register, $src$$Register);
 8493     __ sarl($tmp$$Register, 31);
 8494     __ movl($dst$$Register, $src$$Register);
 8495     __ xorl($dst$$Register, $tmp$$Register);
 8496     __ subl($dst$$Register, $tmp$$Register);
 8497   %}
 8498 
 8499   ins_pipe(ialu_reg_reg);
 8500 %}
 8501 
 8502 // Long Absolute Instructions
 8503 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8504 %{
 8505   match(Set dst (AbsL src));
 8506   effect(TEMP dst, TEMP tmp, KILL cr);
 8507   format %{ "movq $tmp, $src\n\t"
 8508             "sarq $tmp, 63\n\t"
 8509             "movq $dst, $src\n\t"
 8510             "xorq $dst, $tmp\n\t"
 8511             "subq $dst, $tmp\n"
 8512           %}
 8513   ins_encode %{
 8514     __ movq($tmp$$Register, $src$$Register);
 8515     __ sarq($tmp$$Register, 63);
 8516     __ movq($dst$$Register, $src$$Register);
 8517     __ xorq($dst$$Register, $tmp$$Register);
 8518     __ subq($dst$$Register, $tmp$$Register);
 8519   %}
 8520 
 8521   ins_pipe(ialu_reg_reg);
 8522 %}
 8523 
 8524 //----------Subtraction Instructions-------------------------------------------
 8525 
 8526 // Integer Subtraction Instructions
 8527 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8528 %{
 8529   match(Set dst (SubI dst src));
 8530   effect(KILL cr);
 8531 
 8532   format %{ "subl    $dst, $src\t# int" %}
 8533   ins_encode %{
 8534     __ subl($dst$$Register, $src$$Register);
 8535   %}
 8536   ins_pipe(ialu_reg_reg);
 8537 %}
 8538 
 8539 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8540 %{
 8541   match(Set dst (SubI dst src));
 8542   effect(KILL cr);
 8543 
 8544   format %{ "subl    $dst, $src\t# int" %}
 8545   ins_encode %{
 8546     __ subl($dst$$Register, $src$$constant);
 8547   %}
 8548   ins_pipe(ialu_reg);
 8549 %}
 8550 
 8551 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8552 %{
 8553   match(Set dst (SubI dst (LoadI src)));
 8554   effect(KILL cr);
 8555 
 8556   ins_cost(125);
 8557   format %{ "subl    $dst, $src\t# int" %}
 8558   ins_encode %{
 8559     __ subl($dst$$Register, $src$$Address);
 8560   %}
 8561   ins_pipe(ialu_reg_mem);
 8562 %}
 8563 
 8564 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8565 %{
 8566   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8567   effect(KILL cr);
 8568 
 8569   ins_cost(150);
 8570   format %{ "subl    $dst, $src\t# int" %}
 8571   ins_encode %{
 8572     __ subl($dst$$Address, $src$$Register);
 8573   %}
 8574   ins_pipe(ialu_mem_reg);
 8575 %}
 8576 
 8577 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8578 %{
 8579   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8580   effect(KILL cr);
 8581 
 8582   ins_cost(125); // XXX
 8583   format %{ "subl    $dst, $src\t# int" %}
 8584   ins_encode %{
 8585     __ subl($dst$$Address, $src$$constant);
 8586   %}
 8587   ins_pipe(ialu_mem_imm);
 8588 %}
 8589 
 8590 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8591 %{
 8592   match(Set dst (SubL dst src));
 8593   effect(KILL cr);
 8594 
 8595   format %{ "subq    $dst, $src\t# long" %}
 8596   ins_encode %{
 8597     __ subq($dst$$Register, $src$$Register);
 8598   %}
 8599   ins_pipe(ialu_reg_reg);
 8600 %}
 8601 
 8602 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8603 %{
 8604   match(Set dst (SubL dst src));
 8605   effect(KILL cr);
 8606 
 8607   format %{ "subq    $dst, $src\t# long" %}
 8608   ins_encode %{
 8609     __ subq($dst$$Register, $src$$constant);
 8610   %}
 8611   ins_pipe(ialu_reg);
 8612 %}
 8613 
 8614 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8615 %{
 8616   match(Set dst (SubL dst (LoadL src)));
 8617   effect(KILL cr);
 8618 
 8619   ins_cost(125);
 8620   format %{ "subq    $dst, $src\t# long" %}
 8621   ins_encode %{
 8622     __ subq($dst$$Register, $src$$Address);
 8623   %}
 8624   ins_pipe(ialu_reg_mem);
 8625 %}
 8626 
 8627 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8628 %{
 8629   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8630   effect(KILL cr);
 8631 
 8632   ins_cost(150);
 8633   format %{ "subq    $dst, $src\t# long" %}
 8634   ins_encode %{
 8635     __ subq($dst$$Address, $src$$Register);
 8636   %}
 8637   ins_pipe(ialu_mem_reg);
 8638 %}
 8639 
 8640 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8641 %{
 8642   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8643   effect(KILL cr);
 8644 
 8645   ins_cost(125); // XXX
 8646   format %{ "subq    $dst, $src\t# long" %}
 8647   ins_encode %{
 8648     __ subq($dst$$Address, $src$$constant);
 8649   %}
 8650   ins_pipe(ialu_mem_imm);
 8651 %}
 8652 
 8653 // Subtract from a pointer
 8654 // XXX hmpf???
 8655 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8656 %{
 8657   match(Set dst (AddP dst (SubI zero src)));
 8658   effect(KILL cr);
 8659 
 8660   format %{ "subq    $dst, $src\t# ptr - int" %}
 8661   opcode(0x2B);
 8662   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8663   ins_pipe(ialu_reg_reg);
 8664 %}
 8665 
 8666 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8667 %{
 8668   match(Set dst (SubI zero dst));
 8669   effect(KILL cr);
 8670 
 8671   format %{ "negl    $dst\t# int" %}
 8672   ins_encode %{
 8673     __ negl($dst$$Register);
 8674   %}
 8675   ins_pipe(ialu_reg);
 8676 %}
 8677 
 8678 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8679 %{
 8680   match(Set dst (NegI dst));
 8681   effect(KILL cr);
 8682 
 8683   format %{ "negl    $dst\t# int" %}
 8684   ins_encode %{
 8685     __ negl($dst$$Register);
 8686   %}
 8687   ins_pipe(ialu_reg);
 8688 %}
 8689 
 8690 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8691 %{
 8692   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8693   effect(KILL cr);
 8694 
 8695   format %{ "negl    $dst\t# int" %}
 8696   ins_encode %{
 8697     __ negl($dst$$Address);
 8698   %}
 8699   ins_pipe(ialu_reg);
 8700 %}
 8701 
 8702 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8703 %{
 8704   match(Set dst (SubL zero dst));
 8705   effect(KILL cr);
 8706 
 8707   format %{ "negq    $dst\t# long" %}
 8708   ins_encode %{
 8709     __ negq($dst$$Register);
 8710   %}
 8711   ins_pipe(ialu_reg);
 8712 %}
 8713 
 8714 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8715 %{
 8716   match(Set dst (NegL dst));
 8717   effect(KILL cr);
 8718 
 8719   format %{ "negq    $dst\t# int" %}
 8720   ins_encode %{
 8721     __ negq($dst$$Register);
 8722   %}
 8723   ins_pipe(ialu_reg);
 8724 %}
 8725 
 8726 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8727 %{
 8728   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8729   effect(KILL cr);
 8730 
 8731   format %{ "negq    $dst\t# long" %}
 8732   ins_encode %{
 8733     __ negq($dst$$Address);
 8734   %}
 8735   ins_pipe(ialu_reg);
 8736 %}
 8737 
 8738 //----------Multiplication/Division Instructions-------------------------------
 8739 // Integer Multiplication Instructions
 8740 // Multiply Register
 8741 
 8742 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8743 %{
 8744   match(Set dst (MulI dst src));
 8745   effect(KILL cr);
 8746 
 8747   ins_cost(300);
 8748   format %{ "imull   $dst, $src\t# int" %}
 8749   ins_encode %{
 8750     __ imull($dst$$Register, $src$$Register);
 8751   %}
 8752   ins_pipe(ialu_reg_reg_alu0);
 8753 %}
 8754 
 8755 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8756 %{
 8757   match(Set dst (MulI src imm));
 8758   effect(KILL cr);
 8759 
 8760   ins_cost(300);
 8761   format %{ "imull   $dst, $src, $imm\t# int" %}
 8762   ins_encode %{
 8763     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8764   %}
 8765   ins_pipe(ialu_reg_reg_alu0);
 8766 %}
 8767 
 8768 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8769 %{
 8770   match(Set dst (MulI dst (LoadI src)));
 8771   effect(KILL cr);
 8772 
 8773   ins_cost(350);
 8774   format %{ "imull   $dst, $src\t# int" %}
 8775   ins_encode %{
 8776     __ imull($dst$$Register, $src$$Address);
 8777   %}
 8778   ins_pipe(ialu_reg_mem_alu0);
 8779 %}
 8780 
 8781 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8782 %{
 8783   match(Set dst (MulI (LoadI src) imm));
 8784   effect(KILL cr);
 8785 
 8786   ins_cost(300);
 8787   format %{ "imull   $dst, $src, $imm\t# int" %}
 8788   ins_encode %{
 8789     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8790   %}
 8791   ins_pipe(ialu_reg_mem_alu0);
 8792 %}
 8793 
 8794 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8795 %{
 8796   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8797   effect(KILL cr, KILL src2);
 8798 
 8799   expand %{ mulI_rReg(dst, src1, cr);
 8800            mulI_rReg(src2, src3, cr);
 8801            addI_rReg(dst, src2, cr); %}
 8802 %}
 8803 
 8804 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8805 %{
 8806   match(Set dst (MulL dst src));
 8807   effect(KILL cr);
 8808 
 8809   ins_cost(300);
 8810   format %{ "imulq   $dst, $src\t# long" %}
 8811   ins_encode %{
 8812     __ imulq($dst$$Register, $src$$Register);
 8813   %}
 8814   ins_pipe(ialu_reg_reg_alu0);
 8815 %}
 8816 
 8817 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8818 %{
 8819   match(Set dst (MulL src imm));
 8820   effect(KILL cr);
 8821 
 8822   ins_cost(300);
 8823   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8824   ins_encode %{
 8825     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8826   %}
 8827   ins_pipe(ialu_reg_reg_alu0);
 8828 %}
 8829 
 8830 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8831 %{
 8832   match(Set dst (MulL dst (LoadL src)));
 8833   effect(KILL cr);
 8834 
 8835   ins_cost(350);
 8836   format %{ "imulq   $dst, $src\t# long" %}
 8837   ins_encode %{
 8838     __ imulq($dst$$Register, $src$$Address);
 8839   %}
 8840   ins_pipe(ialu_reg_mem_alu0);
 8841 %}
 8842 
 8843 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8844 %{
 8845   match(Set dst (MulL (LoadL src) imm));
 8846   effect(KILL cr);
 8847 
 8848   ins_cost(300);
 8849   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8850   ins_encode %{
 8851     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8852   %}
 8853   ins_pipe(ialu_reg_mem_alu0);
 8854 %}
 8855 
 8856 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8857 %{
 8858   match(Set dst (MulHiL src rax));
 8859   effect(USE_KILL rax, KILL cr);
 8860 
 8861   ins_cost(300);
 8862   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8863   ins_encode %{
 8864     __ imulq($src$$Register);
 8865   %}
 8866   ins_pipe(ialu_reg_reg_alu0);
 8867 %}
 8868 
 8869 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8870 %{
 8871   match(Set dst (UMulHiL src rax));
 8872   effect(USE_KILL rax, KILL cr);
 8873 
 8874   ins_cost(300);
 8875   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8876   ins_encode %{
 8877     __ mulq($src$$Register);
 8878   %}
 8879   ins_pipe(ialu_reg_reg_alu0);
 8880 %}
 8881 
 8882 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8883                    rFlagsReg cr)
 8884 %{
 8885   match(Set rax (DivI rax div));
 8886   effect(KILL rdx, KILL cr);
 8887 
 8888   ins_cost(30*100+10*100); // XXX
 8889   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8890             "jne,s   normal\n\t"
 8891             "xorl    rdx, rdx\n\t"
 8892             "cmpl    $div, -1\n\t"
 8893             "je,s    done\n"
 8894     "normal: cdql\n\t"
 8895             "idivl   $div\n"
 8896     "done:"        %}
 8897   ins_encode(cdql_enc(div));
 8898   ins_pipe(ialu_reg_reg_alu0);
 8899 %}
 8900 
 8901 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8902                    rFlagsReg cr)
 8903 %{
 8904   match(Set rax (DivL rax div));
 8905   effect(KILL rdx, KILL cr);
 8906 
 8907   ins_cost(30*100+10*100); // XXX
 8908   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8909             "cmpq    rax, rdx\n\t"
 8910             "jne,s   normal\n\t"
 8911             "xorl    rdx, rdx\n\t"
 8912             "cmpq    $div, -1\n\t"
 8913             "je,s    done\n"
 8914     "normal: cdqq\n\t"
 8915             "idivq   $div\n"
 8916     "done:"        %}
 8917   ins_encode(cdqq_enc(div));
 8918   ins_pipe(ialu_reg_reg_alu0);
 8919 %}
 8920 
 8921 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8922 %{
 8923   match(Set rax (UDivI rax div));
 8924   effect(KILL rdx, KILL cr);
 8925 
 8926   ins_cost(300);
 8927   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8928   ins_encode %{
 8929     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8930   %}
 8931   ins_pipe(ialu_reg_reg_alu0);
 8932 %}
 8933 
 8934 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8935 %{
 8936   match(Set rax (UDivL rax div));
 8937   effect(KILL rdx, KILL cr);
 8938 
 8939   ins_cost(300);
 8940   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8941   ins_encode %{
 8942      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8943   %}
 8944   ins_pipe(ialu_reg_reg_alu0);
 8945 %}
 8946 
 8947 // Integer DIVMOD with Register, both quotient and mod results
 8948 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8949                              rFlagsReg cr)
 8950 %{
 8951   match(DivModI rax div);
 8952   effect(KILL cr);
 8953 
 8954   ins_cost(30*100+10*100); // XXX
 8955   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8956             "jne,s   normal\n\t"
 8957             "xorl    rdx, rdx\n\t"
 8958             "cmpl    $div, -1\n\t"
 8959             "je,s    done\n"
 8960     "normal: cdql\n\t"
 8961             "idivl   $div\n"
 8962     "done:"        %}
 8963   ins_encode(cdql_enc(div));
 8964   ins_pipe(pipe_slow);
 8965 %}
 8966 
 8967 // Long DIVMOD with Register, both quotient and mod results
 8968 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8969                              rFlagsReg cr)
 8970 %{
 8971   match(DivModL rax div);
 8972   effect(KILL cr);
 8973 
 8974   ins_cost(30*100+10*100); // XXX
 8975   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8976             "cmpq    rax, rdx\n\t"
 8977             "jne,s   normal\n\t"
 8978             "xorl    rdx, rdx\n\t"
 8979             "cmpq    $div, -1\n\t"
 8980             "je,s    done\n"
 8981     "normal: cdqq\n\t"
 8982             "idivq   $div\n"
 8983     "done:"        %}
 8984   ins_encode(cdqq_enc(div));
 8985   ins_pipe(pipe_slow);
 8986 %}
 8987 
 8988 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8989 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8990                               no_rax_rdx_RegI div, rFlagsReg cr)
 8991 %{
 8992   match(UDivModI rax div);
 8993   effect(TEMP tmp, KILL cr);
 8994 
 8995   ins_cost(300);
 8996   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8997             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8998           %}
 8999   ins_encode %{
 9000     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 9001   %}
 9002   ins_pipe(pipe_slow);
 9003 %}
 9004 
 9005 // Unsigned long DIVMOD with Register, both quotient and mod results
 9006 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 9007                               no_rax_rdx_RegL div, rFlagsReg cr)
 9008 %{
 9009   match(UDivModL rax div);
 9010   effect(TEMP tmp, KILL cr);
 9011 
 9012   ins_cost(300);
 9013   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 9014             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 9015           %}
 9016   ins_encode %{
 9017     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 9018   %}
 9019   ins_pipe(pipe_slow);
 9020 %}
 9021 
 9022 
 9023 //----------- DivL-By-Constant-Expansions--------------------------------------
 9024 // DivI cases are handled by the compiler
 9025 
 9026 // Magic constant, reciprocal of 10
 9027 instruct loadConL_0x6666666666666667(rRegL dst)
 9028 %{
 9029   effect(DEF dst);
 9030 
 9031   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 9032   ins_encode(load_immL(dst, 0x6666666666666667));
 9033   ins_pipe(ialu_reg);
 9034 %}
 9035 
 9036 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 9037 %{
 9038   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 9039 
 9040   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 9041   ins_encode %{
 9042     __ imulq($src$$Register);
 9043   %}
 9044   ins_pipe(ialu_reg_reg_alu0);
 9045 %}
 9046 
 9047 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 9048 %{
 9049   effect(USE_DEF dst, KILL cr);
 9050 
 9051   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 9052   ins_encode %{
 9053     __ sarq($dst$$Register, 63);
 9054   %}
 9055   ins_pipe(ialu_reg);
 9056 %}
 9057 
 9058 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 9059 %{
 9060   effect(USE_DEF dst, KILL cr);
 9061 
 9062   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 9063   ins_encode %{
 9064     __ sarq($dst$$Register, 2);
 9065   %}
 9066   ins_pipe(ialu_reg);
 9067 %}
 9068 
 9069 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 9070 %{
 9071   match(Set dst (DivL src div));
 9072 
 9073   ins_cost((5+8)*100);
 9074   expand %{
 9075     rax_RegL rax;                     // Killed temp
 9076     rFlagsReg cr;                     // Killed
 9077     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 9078     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 9079     sarL_rReg_63(src, cr);            // sarq  src, 63
 9080     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 9081     subL_rReg(dst, src, cr);          // subl  rdx, src
 9082   %}
 9083 %}
 9084 
 9085 //-----------------------------------------------------------------------------
 9086 
 9087 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 9088                    rFlagsReg cr)
 9089 %{
 9090   match(Set rdx (ModI rax div));
 9091   effect(KILL rax, KILL cr);
 9092 
 9093   ins_cost(300); // XXX
 9094   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 9095             "jne,s   normal\n\t"
 9096             "xorl    rdx, rdx\n\t"
 9097             "cmpl    $div, -1\n\t"
 9098             "je,s    done\n"
 9099     "normal: cdql\n\t"
 9100             "idivl   $div\n"
 9101     "done:"        %}
 9102   ins_encode(cdql_enc(div));
 9103   ins_pipe(ialu_reg_reg_alu0);
 9104 %}
 9105 
 9106 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 9107                    rFlagsReg cr)
 9108 %{
 9109   match(Set rdx (ModL rax div));
 9110   effect(KILL rax, KILL cr);
 9111 
 9112   ins_cost(300); // XXX
 9113   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 9114             "cmpq    rax, rdx\n\t"
 9115             "jne,s   normal\n\t"
 9116             "xorl    rdx, rdx\n\t"
 9117             "cmpq    $div, -1\n\t"
 9118             "je,s    done\n"
 9119     "normal: cdqq\n\t"
 9120             "idivq   $div\n"
 9121     "done:"        %}
 9122   ins_encode(cdqq_enc(div));
 9123   ins_pipe(ialu_reg_reg_alu0);
 9124 %}
 9125 
 9126 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9127 %{
 9128   match(Set rdx (UModI rax div));
 9129   effect(KILL rax, KILL cr);
 9130 
 9131   ins_cost(300);
 9132   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9133   ins_encode %{
 9134     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9135   %}
 9136   ins_pipe(ialu_reg_reg_alu0);
 9137 %}
 9138 
 9139 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9140 %{
 9141   match(Set rdx (UModL rax div));
 9142   effect(KILL rax, KILL cr);
 9143 
 9144   ins_cost(300);
 9145   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9146   ins_encode %{
 9147     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9148   %}
 9149   ins_pipe(ialu_reg_reg_alu0);
 9150 %}
 9151 
 9152 // Integer Shift Instructions
 9153 // Shift Left by one
 9154 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9155 %{
 9156   match(Set dst (LShiftI dst shift));
 9157   effect(KILL cr);
 9158 
 9159   format %{ "sall    $dst, $shift" %}
 9160   ins_encode %{
 9161     __ sall($dst$$Register, $shift$$constant);
 9162   %}
 9163   ins_pipe(ialu_reg);
 9164 %}
 9165 
 9166 // Shift Left by one
 9167 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9168 %{
 9169   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9170   effect(KILL cr);
 9171 
 9172   format %{ "sall    $dst, $shift\t" %}
 9173   ins_encode %{
 9174     __ sall($dst$$Address, $shift$$constant);
 9175   %}
 9176   ins_pipe(ialu_mem_imm);
 9177 %}
 9178 
 9179 // Shift Left by 8-bit immediate
 9180 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9181 %{
 9182   match(Set dst (LShiftI dst shift));
 9183   effect(KILL cr);
 9184 
 9185   format %{ "sall    $dst, $shift" %}
 9186   ins_encode %{
 9187     __ sall($dst$$Register, $shift$$constant);
 9188   %}
 9189   ins_pipe(ialu_reg);
 9190 %}
 9191 
 9192 // Shift Left by 8-bit immediate
 9193 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9194 %{
 9195   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9196   effect(KILL cr);
 9197 
 9198   format %{ "sall    $dst, $shift" %}
 9199   ins_encode %{
 9200     __ sall($dst$$Address, $shift$$constant);
 9201   %}
 9202   ins_pipe(ialu_mem_imm);
 9203 %}
 9204 
 9205 // Shift Left by variable
 9206 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9207 %{
 9208   predicate(!VM_Version::supports_bmi2());
 9209   match(Set dst (LShiftI dst shift));
 9210   effect(KILL cr);
 9211 
 9212   format %{ "sall    $dst, $shift" %}
 9213   ins_encode %{
 9214     __ sall($dst$$Register);
 9215   %}
 9216   ins_pipe(ialu_reg_reg);
 9217 %}
 9218 
 9219 // Shift Left by variable
 9220 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9221 %{
 9222   predicate(!VM_Version::supports_bmi2());
 9223   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9224   effect(KILL cr);
 9225 
 9226   format %{ "sall    $dst, $shift" %}
 9227   ins_encode %{
 9228     __ sall($dst$$Address);
 9229   %}
 9230   ins_pipe(ialu_mem_reg);
 9231 %}
 9232 
 9233 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9234 %{
 9235   predicate(VM_Version::supports_bmi2());
 9236   match(Set dst (LShiftI src shift));
 9237 
 9238   format %{ "shlxl   $dst, $src, $shift" %}
 9239   ins_encode %{
 9240     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9241   %}
 9242   ins_pipe(ialu_reg_reg);
 9243 %}
 9244 
 9245 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9246 %{
 9247   predicate(VM_Version::supports_bmi2());
 9248   match(Set dst (LShiftI (LoadI src) shift));
 9249   ins_cost(175);
 9250   format %{ "shlxl   $dst, $src, $shift" %}
 9251   ins_encode %{
 9252     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9253   %}
 9254   ins_pipe(ialu_reg_mem);
 9255 %}
 9256 
 9257 // Arithmetic shift right by one
 9258 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9259 %{
 9260   match(Set dst (RShiftI dst shift));
 9261   effect(KILL cr);
 9262 
 9263   format %{ "sarl    $dst, $shift" %}
 9264   ins_encode %{
 9265     __ sarl($dst$$Register, $shift$$constant);
 9266   %}
 9267   ins_pipe(ialu_reg);
 9268 %}
 9269 
 9270 // Arithmetic shift right by one
 9271 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9272 %{
 9273   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9274   effect(KILL cr);
 9275 
 9276   format %{ "sarl    $dst, $shift" %}
 9277   ins_encode %{
 9278     __ sarl($dst$$Address, $shift$$constant);
 9279   %}
 9280   ins_pipe(ialu_mem_imm);
 9281 %}
 9282 
 9283 // Arithmetic Shift Right by 8-bit immediate
 9284 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9285 %{
 9286   match(Set dst (RShiftI dst shift));
 9287   effect(KILL cr);
 9288 
 9289   format %{ "sarl    $dst, $shift" %}
 9290   ins_encode %{
 9291     __ sarl($dst$$Register, $shift$$constant);
 9292   %}
 9293   ins_pipe(ialu_mem_imm);
 9294 %}
 9295 
 9296 // Arithmetic Shift Right by 8-bit immediate
 9297 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9298 %{
 9299   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9300   effect(KILL cr);
 9301 
 9302   format %{ "sarl    $dst, $shift" %}
 9303   ins_encode %{
 9304     __ sarl($dst$$Address, $shift$$constant);
 9305   %}
 9306   ins_pipe(ialu_mem_imm);
 9307 %}
 9308 
 9309 // Arithmetic Shift Right by variable
 9310 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9311 %{
 9312   predicate(!VM_Version::supports_bmi2());
 9313   match(Set dst (RShiftI dst shift));
 9314   effect(KILL cr);
 9315   format %{ "sarl    $dst, $shift" %}
 9316   ins_encode %{
 9317     __ sarl($dst$$Register);
 9318   %}
 9319   ins_pipe(ialu_reg_reg);
 9320 %}
 9321 
 9322 // Arithmetic Shift Right by variable
 9323 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9324 %{
 9325   predicate(!VM_Version::supports_bmi2());
 9326   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9327   effect(KILL cr);
 9328 
 9329   format %{ "sarl    $dst, $shift" %}
 9330   ins_encode %{
 9331     __ sarl($dst$$Address);
 9332   %}
 9333   ins_pipe(ialu_mem_reg);
 9334 %}
 9335 
 9336 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9337 %{
 9338   predicate(VM_Version::supports_bmi2());
 9339   match(Set dst (RShiftI src shift));
 9340 
 9341   format %{ "sarxl   $dst, $src, $shift" %}
 9342   ins_encode %{
 9343     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9344   %}
 9345   ins_pipe(ialu_reg_reg);
 9346 %}
 9347 
 9348 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9349 %{
 9350   predicate(VM_Version::supports_bmi2());
 9351   match(Set dst (RShiftI (LoadI src) shift));
 9352   ins_cost(175);
 9353   format %{ "sarxl   $dst, $src, $shift" %}
 9354   ins_encode %{
 9355     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9356   %}
 9357   ins_pipe(ialu_reg_mem);
 9358 %}
 9359 
 9360 // Logical shift right by one
 9361 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9362 %{
 9363   match(Set dst (URShiftI dst shift));
 9364   effect(KILL cr);
 9365 
 9366   format %{ "shrl    $dst, $shift" %}
 9367   ins_encode %{
 9368     __ shrl($dst$$Register, $shift$$constant);
 9369   %}
 9370   ins_pipe(ialu_reg);
 9371 %}
 9372 
 9373 // Logical shift right by one
 9374 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9375 %{
 9376   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9377   effect(KILL cr);
 9378 
 9379   format %{ "shrl    $dst, $shift" %}
 9380   ins_encode %{
 9381     __ shrl($dst$$Address, $shift$$constant);
 9382   %}
 9383   ins_pipe(ialu_mem_imm);
 9384 %}
 9385 
 9386 // Logical Shift Right by 8-bit immediate
 9387 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9388 %{
 9389   match(Set dst (URShiftI dst shift));
 9390   effect(KILL cr);
 9391 
 9392   format %{ "shrl    $dst, $shift" %}
 9393   ins_encode %{
 9394     __ shrl($dst$$Register, $shift$$constant);
 9395   %}
 9396   ins_pipe(ialu_reg);
 9397 %}
 9398 
 9399 // Logical Shift Right by 8-bit immediate
 9400 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9401 %{
 9402   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9403   effect(KILL cr);
 9404 
 9405   format %{ "shrl    $dst, $shift" %}
 9406   ins_encode %{
 9407     __ shrl($dst$$Address, $shift$$constant);
 9408   %}
 9409   ins_pipe(ialu_mem_imm);
 9410 %}
 9411 
 9412 // Logical Shift Right by variable
 9413 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9414 %{
 9415   predicate(!VM_Version::supports_bmi2());
 9416   match(Set dst (URShiftI dst shift));
 9417   effect(KILL cr);
 9418 
 9419   format %{ "shrl    $dst, $shift" %}
 9420   ins_encode %{
 9421     __ shrl($dst$$Register);
 9422   %}
 9423   ins_pipe(ialu_reg_reg);
 9424 %}
 9425 
 9426 // Logical Shift Right by variable
 9427 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9428 %{
 9429   predicate(!VM_Version::supports_bmi2());
 9430   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9431   effect(KILL cr);
 9432 
 9433   format %{ "shrl    $dst, $shift" %}
 9434   ins_encode %{
 9435     __ shrl($dst$$Address);
 9436   %}
 9437   ins_pipe(ialu_mem_reg);
 9438 %}
 9439 
 9440 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9441 %{
 9442   predicate(VM_Version::supports_bmi2());
 9443   match(Set dst (URShiftI src shift));
 9444 
 9445   format %{ "shrxl   $dst, $src, $shift" %}
 9446   ins_encode %{
 9447     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9448   %}
 9449   ins_pipe(ialu_reg_reg);
 9450 %}
 9451 
 9452 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9453 %{
 9454   predicate(VM_Version::supports_bmi2());
 9455   match(Set dst (URShiftI (LoadI src) shift));
 9456   ins_cost(175);
 9457   format %{ "shrxl   $dst, $src, $shift" %}
 9458   ins_encode %{
 9459     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9460   %}
 9461   ins_pipe(ialu_reg_mem);
 9462 %}
 9463 
 9464 // Long Shift Instructions
 9465 // Shift Left by one
 9466 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9467 %{
 9468   match(Set dst (LShiftL dst shift));
 9469   effect(KILL cr);
 9470 
 9471   format %{ "salq    $dst, $shift" %}
 9472   ins_encode %{
 9473     __ salq($dst$$Register, $shift$$constant);
 9474   %}
 9475   ins_pipe(ialu_reg);
 9476 %}
 9477 
 9478 // Shift Left by one
 9479 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9480 %{
 9481   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9482   effect(KILL cr);
 9483 
 9484   format %{ "salq    $dst, $shift" %}
 9485   ins_encode %{
 9486     __ salq($dst$$Address, $shift$$constant);
 9487   %}
 9488   ins_pipe(ialu_mem_imm);
 9489 %}
 9490 
 9491 // Shift Left by 8-bit immediate
 9492 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9493 %{
 9494   match(Set dst (LShiftL dst shift));
 9495   effect(KILL cr);
 9496 
 9497   format %{ "salq    $dst, $shift" %}
 9498   ins_encode %{
 9499     __ salq($dst$$Register, $shift$$constant);
 9500   %}
 9501   ins_pipe(ialu_reg);
 9502 %}
 9503 
 9504 // Shift Left by 8-bit immediate
 9505 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9506 %{
 9507   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9508   effect(KILL cr);
 9509 
 9510   format %{ "salq    $dst, $shift" %}
 9511   ins_encode %{
 9512     __ salq($dst$$Address, $shift$$constant);
 9513   %}
 9514   ins_pipe(ialu_mem_imm);
 9515 %}
 9516 
 9517 // Shift Left by variable
 9518 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9519 %{
 9520   predicate(!VM_Version::supports_bmi2());
 9521   match(Set dst (LShiftL dst shift));
 9522   effect(KILL cr);
 9523 
 9524   format %{ "salq    $dst, $shift" %}
 9525   ins_encode %{
 9526     __ salq($dst$$Register);
 9527   %}
 9528   ins_pipe(ialu_reg_reg);
 9529 %}
 9530 
 9531 // Shift Left by variable
 9532 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9533 %{
 9534   predicate(!VM_Version::supports_bmi2());
 9535   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9536   effect(KILL cr);
 9537 
 9538   format %{ "salq    $dst, $shift" %}
 9539   ins_encode %{
 9540     __ salq($dst$$Address);
 9541   %}
 9542   ins_pipe(ialu_mem_reg);
 9543 %}
 9544 
 9545 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9546 %{
 9547   predicate(VM_Version::supports_bmi2());
 9548   match(Set dst (LShiftL src shift));
 9549 
 9550   format %{ "shlxq   $dst, $src, $shift" %}
 9551   ins_encode %{
 9552     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9553   %}
 9554   ins_pipe(ialu_reg_reg);
 9555 %}
 9556 
 9557 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9558 %{
 9559   predicate(VM_Version::supports_bmi2());
 9560   match(Set dst (LShiftL (LoadL src) shift));
 9561   ins_cost(175);
 9562   format %{ "shlxq   $dst, $src, $shift" %}
 9563   ins_encode %{
 9564     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9565   %}
 9566   ins_pipe(ialu_reg_mem);
 9567 %}
 9568 
 9569 // Arithmetic shift right by one
 9570 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9571 %{
 9572   match(Set dst (RShiftL dst shift));
 9573   effect(KILL cr);
 9574 
 9575   format %{ "sarq    $dst, $shift" %}
 9576   ins_encode %{
 9577     __ sarq($dst$$Register, $shift$$constant);
 9578   %}
 9579   ins_pipe(ialu_reg);
 9580 %}
 9581 
 9582 // Arithmetic shift right by one
 9583 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9584 %{
 9585   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9586   effect(KILL cr);
 9587 
 9588   format %{ "sarq    $dst, $shift" %}
 9589   ins_encode %{
 9590     __ sarq($dst$$Address, $shift$$constant);
 9591   %}
 9592   ins_pipe(ialu_mem_imm);
 9593 %}
 9594 
 9595 // Arithmetic Shift Right by 8-bit immediate
 9596 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9597 %{
 9598   match(Set dst (RShiftL dst shift));
 9599   effect(KILL cr);
 9600 
 9601   format %{ "sarq    $dst, $shift" %}
 9602   ins_encode %{
 9603     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9604   %}
 9605   ins_pipe(ialu_mem_imm);
 9606 %}
 9607 
 9608 // Arithmetic Shift Right by 8-bit immediate
 9609 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9610 %{
 9611   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9612   effect(KILL cr);
 9613 
 9614   format %{ "sarq    $dst, $shift" %}
 9615   ins_encode %{
 9616     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9617   %}
 9618   ins_pipe(ialu_mem_imm);
 9619 %}
 9620 
 9621 // Arithmetic Shift Right by variable
 9622 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9623 %{
 9624   predicate(!VM_Version::supports_bmi2());
 9625   match(Set dst (RShiftL dst shift));
 9626   effect(KILL cr);
 9627 
 9628   format %{ "sarq    $dst, $shift" %}
 9629   ins_encode %{
 9630     __ sarq($dst$$Register);
 9631   %}
 9632   ins_pipe(ialu_reg_reg);
 9633 %}
 9634 
 9635 // Arithmetic Shift Right by variable
 9636 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9637 %{
 9638   predicate(!VM_Version::supports_bmi2());
 9639   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9640   effect(KILL cr);
 9641 
 9642   format %{ "sarq    $dst, $shift" %}
 9643   ins_encode %{
 9644     __ sarq($dst$$Address);
 9645   %}
 9646   ins_pipe(ialu_mem_reg);
 9647 %}
 9648 
 9649 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9650 %{
 9651   predicate(VM_Version::supports_bmi2());
 9652   match(Set dst (RShiftL src shift));
 9653 
 9654   format %{ "sarxq   $dst, $src, $shift" %}
 9655   ins_encode %{
 9656     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9657   %}
 9658   ins_pipe(ialu_reg_reg);
 9659 %}
 9660 
 9661 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9662 %{
 9663   predicate(VM_Version::supports_bmi2());
 9664   match(Set dst (RShiftL (LoadL src) shift));
 9665   ins_cost(175);
 9666   format %{ "sarxq   $dst, $src, $shift" %}
 9667   ins_encode %{
 9668     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9669   %}
 9670   ins_pipe(ialu_reg_mem);
 9671 %}
 9672 
 9673 // Logical shift right by one
 9674 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9675 %{
 9676   match(Set dst (URShiftL dst shift));
 9677   effect(KILL cr);
 9678 
 9679   format %{ "shrq    $dst, $shift" %}
 9680   ins_encode %{
 9681     __ shrq($dst$$Register, $shift$$constant);
 9682   %}
 9683   ins_pipe(ialu_reg);
 9684 %}
 9685 
 9686 // Logical shift right by one
 9687 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9688 %{
 9689   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9690   effect(KILL cr);
 9691 
 9692   format %{ "shrq    $dst, $shift" %}
 9693   ins_encode %{
 9694     __ shrq($dst$$Address, $shift$$constant);
 9695   %}
 9696   ins_pipe(ialu_mem_imm);
 9697 %}
 9698 
 9699 // Logical Shift Right by 8-bit immediate
 9700 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9701 %{
 9702   match(Set dst (URShiftL dst shift));
 9703   effect(KILL cr);
 9704 
 9705   format %{ "shrq    $dst, $shift" %}
 9706   ins_encode %{
 9707     __ shrq($dst$$Register, $shift$$constant);
 9708   %}
 9709   ins_pipe(ialu_reg);
 9710 %}
 9711 
 9712 // Logical Shift Right by 8-bit immediate
 9713 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9714 %{
 9715   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9716   effect(KILL cr);
 9717 
 9718   format %{ "shrq    $dst, $shift" %}
 9719   ins_encode %{
 9720     __ shrq($dst$$Address, $shift$$constant);
 9721   %}
 9722   ins_pipe(ialu_mem_imm);
 9723 %}
 9724 
 9725 // Logical Shift Right by variable
 9726 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9727 %{
 9728   predicate(!VM_Version::supports_bmi2());
 9729   match(Set dst (URShiftL dst shift));
 9730   effect(KILL cr);
 9731 
 9732   format %{ "shrq    $dst, $shift" %}
 9733   ins_encode %{
 9734     __ shrq($dst$$Register);
 9735   %}
 9736   ins_pipe(ialu_reg_reg);
 9737 %}
 9738 
 9739 // Logical Shift Right by variable
 9740 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9741 %{
 9742   predicate(!VM_Version::supports_bmi2());
 9743   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9744   effect(KILL cr);
 9745 
 9746   format %{ "shrq    $dst, $shift" %}
 9747   ins_encode %{
 9748     __ shrq($dst$$Address);
 9749   %}
 9750   ins_pipe(ialu_mem_reg);
 9751 %}
 9752 
 9753 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9754 %{
 9755   predicate(VM_Version::supports_bmi2());
 9756   match(Set dst (URShiftL src shift));
 9757 
 9758   format %{ "shrxq   $dst, $src, $shift" %}
 9759   ins_encode %{
 9760     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9761   %}
 9762   ins_pipe(ialu_reg_reg);
 9763 %}
 9764 
 9765 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9766 %{
 9767   predicate(VM_Version::supports_bmi2());
 9768   match(Set dst (URShiftL (LoadL src) shift));
 9769   ins_cost(175);
 9770   format %{ "shrxq   $dst, $src, $shift" %}
 9771   ins_encode %{
 9772     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9773   %}
 9774   ins_pipe(ialu_reg_mem);
 9775 %}
 9776 
 9777 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9778 // This idiom is used by the compiler for the i2b bytecode.
 9779 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9780 %{
 9781   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9782 
 9783   format %{ "movsbl  $dst, $src\t# i2b" %}
 9784   ins_encode %{
 9785     __ movsbl($dst$$Register, $src$$Register);
 9786   %}
 9787   ins_pipe(ialu_reg_reg);
 9788 %}
 9789 
 9790 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9791 // This idiom is used by the compiler the i2s bytecode.
 9792 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9793 %{
 9794   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9795 
 9796   format %{ "movswl  $dst, $src\t# i2s" %}
 9797   ins_encode %{
 9798     __ movswl($dst$$Register, $src$$Register);
 9799   %}
 9800   ins_pipe(ialu_reg_reg);
 9801 %}
 9802 
 9803 // ROL/ROR instructions
 9804 
 9805 // Rotate left by constant.
 9806 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9807 %{
 9808   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9809   match(Set dst (RotateLeft dst shift));
 9810   effect(KILL cr);
 9811   format %{ "roll    $dst, $shift" %}
 9812   ins_encode %{
 9813     __ roll($dst$$Register, $shift$$constant);
 9814   %}
 9815   ins_pipe(ialu_reg);
 9816 %}
 9817 
 9818 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9819 %{
 9820   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9821   match(Set dst (RotateLeft src shift));
 9822   format %{ "rolxl   $dst, $src, $shift" %}
 9823   ins_encode %{
 9824     int shift = 32 - ($shift$$constant & 31);
 9825     __ rorxl($dst$$Register, $src$$Register, shift);
 9826   %}
 9827   ins_pipe(ialu_reg_reg);
 9828 %}
 9829 
 9830 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9831 %{
 9832   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9833   match(Set dst (RotateLeft (LoadI src) shift));
 9834   ins_cost(175);
 9835   format %{ "rolxl   $dst, $src, $shift" %}
 9836   ins_encode %{
 9837     int shift = 32 - ($shift$$constant & 31);
 9838     __ rorxl($dst$$Register, $src$$Address, shift);
 9839   %}
 9840   ins_pipe(ialu_reg_mem);
 9841 %}
 9842 
 9843 // Rotate Left by variable
 9844 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9845 %{
 9846   predicate(n->bottom_type()->basic_type() == T_INT);
 9847   match(Set dst (RotateLeft dst shift));
 9848   effect(KILL cr);
 9849   format %{ "roll    $dst, $shift" %}
 9850   ins_encode %{
 9851     __ roll($dst$$Register);
 9852   %}
 9853   ins_pipe(ialu_reg_reg);
 9854 %}
 9855 
 9856 // Rotate Right by constant.
 9857 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9858 %{
 9859   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9860   match(Set dst (RotateRight dst shift));
 9861   effect(KILL cr);
 9862   format %{ "rorl    $dst, $shift" %}
 9863   ins_encode %{
 9864     __ rorl($dst$$Register, $shift$$constant);
 9865   %}
 9866   ins_pipe(ialu_reg);
 9867 %}
 9868 
 9869 // Rotate Right by constant.
 9870 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9871 %{
 9872   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9873   match(Set dst (RotateRight src shift));
 9874   format %{ "rorxl   $dst, $src, $shift" %}
 9875   ins_encode %{
 9876     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9877   %}
 9878   ins_pipe(ialu_reg_reg);
 9879 %}
 9880 
 9881 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9882 %{
 9883   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9884   match(Set dst (RotateRight (LoadI src) shift));
 9885   ins_cost(175);
 9886   format %{ "rorxl   $dst, $src, $shift" %}
 9887   ins_encode %{
 9888     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9889   %}
 9890   ins_pipe(ialu_reg_mem);
 9891 %}
 9892 
 9893 // Rotate Right by variable
 9894 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9895 %{
 9896   predicate(n->bottom_type()->basic_type() == T_INT);
 9897   match(Set dst (RotateRight dst shift));
 9898   effect(KILL cr);
 9899   format %{ "rorl    $dst, $shift" %}
 9900   ins_encode %{
 9901     __ rorl($dst$$Register);
 9902   %}
 9903   ins_pipe(ialu_reg_reg);
 9904 %}
 9905 
 9906 // Rotate Left by constant.
 9907 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9908 %{
 9909   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9910   match(Set dst (RotateLeft dst shift));
 9911   effect(KILL cr);
 9912   format %{ "rolq    $dst, $shift" %}
 9913   ins_encode %{
 9914     __ rolq($dst$$Register, $shift$$constant);
 9915   %}
 9916   ins_pipe(ialu_reg);
 9917 %}
 9918 
 9919 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9920 %{
 9921   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9922   match(Set dst (RotateLeft src shift));
 9923   format %{ "rolxq   $dst, $src, $shift" %}
 9924   ins_encode %{
 9925     int shift = 64 - ($shift$$constant & 63);
 9926     __ rorxq($dst$$Register, $src$$Register, shift);
 9927   %}
 9928   ins_pipe(ialu_reg_reg);
 9929 %}
 9930 
 9931 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9932 %{
 9933   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9934   match(Set dst (RotateLeft (LoadL src) shift));
 9935   ins_cost(175);
 9936   format %{ "rolxq   $dst, $src, $shift" %}
 9937   ins_encode %{
 9938     int shift = 64 - ($shift$$constant & 63);
 9939     __ rorxq($dst$$Register, $src$$Address, shift);
 9940   %}
 9941   ins_pipe(ialu_reg_mem);
 9942 %}
 9943 
 9944 // Rotate Left by variable
 9945 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9946 %{
 9947   predicate(n->bottom_type()->basic_type() == T_LONG);
 9948   match(Set dst (RotateLeft dst shift));
 9949   effect(KILL cr);
 9950   format %{ "rolq    $dst, $shift" %}
 9951   ins_encode %{
 9952     __ rolq($dst$$Register);
 9953   %}
 9954   ins_pipe(ialu_reg_reg);
 9955 %}
 9956 
 9957 // Rotate Right by constant.
 9958 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9959 %{
 9960   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9961   match(Set dst (RotateRight dst shift));
 9962   effect(KILL cr);
 9963   format %{ "rorq    $dst, $shift" %}
 9964   ins_encode %{
 9965     __ rorq($dst$$Register, $shift$$constant);
 9966   %}
 9967   ins_pipe(ialu_reg);
 9968 %}
 9969 
 9970 // Rotate Right by constant
 9971 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9972 %{
 9973   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9974   match(Set dst (RotateRight src shift));
 9975   format %{ "rorxq   $dst, $src, $shift" %}
 9976   ins_encode %{
 9977     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9978   %}
 9979   ins_pipe(ialu_reg_reg);
 9980 %}
 9981 
 9982 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9983 %{
 9984   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9985   match(Set dst (RotateRight (LoadL src) shift));
 9986   ins_cost(175);
 9987   format %{ "rorxq   $dst, $src, $shift" %}
 9988   ins_encode %{
 9989     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9990   %}
 9991   ins_pipe(ialu_reg_mem);
 9992 %}
 9993 
 9994 // Rotate Right by variable
 9995 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9996 %{
 9997   predicate(n->bottom_type()->basic_type() == T_LONG);
 9998   match(Set dst (RotateRight dst shift));
 9999   effect(KILL cr);
10000   format %{ "rorq    $dst, $shift" %}
10001   ins_encode %{
10002     __ rorq($dst$$Register);
10003   %}
10004   ins_pipe(ialu_reg_reg);
10005 %}
10006 
10007 //----------------------------- CompressBits/ExpandBits ------------------------
10008 
10009 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10010   predicate(n->bottom_type()->isa_long());
10011   match(Set dst (CompressBits src mask));
10012   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10013   ins_encode %{
10014     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
10015   %}
10016   ins_pipe( pipe_slow );
10017 %}
10018 
10019 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10020   predicate(n->bottom_type()->isa_long());
10021   match(Set dst (ExpandBits src mask));
10022   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10023   ins_encode %{
10024     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10025   %}
10026   ins_pipe( pipe_slow );
10027 %}
10028 
10029 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10030   predicate(n->bottom_type()->isa_long());
10031   match(Set dst (CompressBits src (LoadL mask)));
10032   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10033   ins_encode %{
10034     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10035   %}
10036   ins_pipe( pipe_slow );
10037 %}
10038 
10039 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10040   predicate(n->bottom_type()->isa_long());
10041   match(Set dst (ExpandBits src (LoadL mask)));
10042   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10043   ins_encode %{
10044     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10045   %}
10046   ins_pipe( pipe_slow );
10047 %}
10048 
10049 
10050 // Logical Instructions
10051 
10052 // Integer Logical Instructions
10053 
10054 // And Instructions
10055 // And Register with Register
10056 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10057 %{
10058   match(Set dst (AndI dst src));
10059   effect(KILL cr);
10060 
10061   format %{ "andl    $dst, $src\t# int" %}
10062   ins_encode %{
10063     __ andl($dst$$Register, $src$$Register);
10064   %}
10065   ins_pipe(ialu_reg_reg);
10066 %}
10067 
10068 // And Register with Immediate 255
10069 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
10070 %{
10071   match(Set dst (AndI dst src));
10072 
10073   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
10074   ins_encode %{
10075     __ movzbl($dst$$Register, $dst$$Register);
10076   %}
10077   ins_pipe(ialu_reg);
10078 %}
10079 
10080 // And Register with Immediate 255 and promote to long
10081 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10082 %{
10083   match(Set dst (ConvI2L (AndI src mask)));
10084 
10085   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10086   ins_encode %{
10087     __ movzbl($dst$$Register, $src$$Register);
10088   %}
10089   ins_pipe(ialu_reg);
10090 %}
10091 
10092 // And Register with Immediate 65535
10093 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
10094 %{
10095   match(Set dst (AndI dst src));
10096 
10097   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
10098   ins_encode %{
10099     __ movzwl($dst$$Register, $dst$$Register);
10100   %}
10101   ins_pipe(ialu_reg);
10102 %}
10103 
10104 // And Register with Immediate 65535 and promote to long
10105 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10106 %{
10107   match(Set dst (ConvI2L (AndI src mask)));
10108 
10109   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10110   ins_encode %{
10111     __ movzwl($dst$$Register, $src$$Register);
10112   %}
10113   ins_pipe(ialu_reg);
10114 %}
10115 
10116 // Can skip int2long conversions after AND with small bitmask
10117 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10118 %{
10119   predicate(VM_Version::supports_bmi2());
10120   ins_cost(125);
10121   effect(TEMP tmp, KILL cr);
10122   match(Set dst (ConvI2L (AndI src mask)));
10123   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10124   ins_encode %{
10125     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10126     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10127   %}
10128   ins_pipe(ialu_reg_reg);
10129 %}
10130 
10131 // And Register with Immediate
10132 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10133 %{
10134   match(Set dst (AndI dst src));
10135   effect(KILL cr);
10136 
10137   format %{ "andl    $dst, $src\t# int" %}
10138   ins_encode %{
10139     __ andl($dst$$Register, $src$$constant);
10140   %}
10141   ins_pipe(ialu_reg);
10142 %}
10143 
10144 // And Register with Memory
10145 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10146 %{
10147   match(Set dst (AndI dst (LoadI src)));
10148   effect(KILL cr);
10149 
10150   ins_cost(125);
10151   format %{ "andl    $dst, $src\t# int" %}
10152   ins_encode %{
10153     __ andl($dst$$Register, $src$$Address);
10154   %}
10155   ins_pipe(ialu_reg_mem);
10156 %}
10157 
10158 // And Memory with Register
10159 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10160 %{
10161   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10162   effect(KILL cr);
10163 
10164   ins_cost(150);
10165   format %{ "andb    $dst, $src\t# byte" %}
10166   ins_encode %{
10167     __ andb($dst$$Address, $src$$Register);
10168   %}
10169   ins_pipe(ialu_mem_reg);
10170 %}
10171 
10172 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10173 %{
10174   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10175   effect(KILL cr);
10176 
10177   ins_cost(150);
10178   format %{ "andl    $dst, $src\t# int" %}
10179   ins_encode %{
10180     __ andl($dst$$Address, $src$$Register);
10181   %}
10182   ins_pipe(ialu_mem_reg);
10183 %}
10184 
10185 // And Memory with Immediate
10186 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10187 %{
10188   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10189   effect(KILL cr);
10190 
10191   ins_cost(125);
10192   format %{ "andl    $dst, $src\t# int" %}
10193   ins_encode %{
10194     __ andl($dst$$Address, $src$$constant);
10195   %}
10196   ins_pipe(ialu_mem_imm);
10197 %}
10198 
10199 // BMI1 instructions
10200 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10201   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10202   predicate(UseBMI1Instructions);
10203   effect(KILL cr);
10204 
10205   ins_cost(125);
10206   format %{ "andnl  $dst, $src1, $src2" %}
10207 
10208   ins_encode %{
10209     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10210   %}
10211   ins_pipe(ialu_reg_mem);
10212 %}
10213 
10214 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10215   match(Set dst (AndI (XorI src1 minus_1) src2));
10216   predicate(UseBMI1Instructions);
10217   effect(KILL cr);
10218 
10219   format %{ "andnl  $dst, $src1, $src2" %}
10220 
10221   ins_encode %{
10222     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10223   %}
10224   ins_pipe(ialu_reg);
10225 %}
10226 
10227 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10228   match(Set dst (AndI (SubI imm_zero src) src));
10229   predicate(UseBMI1Instructions);
10230   effect(KILL cr);
10231 
10232   format %{ "blsil  $dst, $src" %}
10233 
10234   ins_encode %{
10235     __ blsil($dst$$Register, $src$$Register);
10236   %}
10237   ins_pipe(ialu_reg);
10238 %}
10239 
10240 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10241   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10242   predicate(UseBMI1Instructions);
10243   effect(KILL cr);
10244 
10245   ins_cost(125);
10246   format %{ "blsil  $dst, $src" %}
10247 
10248   ins_encode %{
10249     __ blsil($dst$$Register, $src$$Address);
10250   %}
10251   ins_pipe(ialu_reg_mem);
10252 %}
10253 
10254 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10255 %{
10256   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10257   predicate(UseBMI1Instructions);
10258   effect(KILL cr);
10259 
10260   ins_cost(125);
10261   format %{ "blsmskl $dst, $src" %}
10262 
10263   ins_encode %{
10264     __ blsmskl($dst$$Register, $src$$Address);
10265   %}
10266   ins_pipe(ialu_reg_mem);
10267 %}
10268 
10269 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10270 %{
10271   match(Set dst (XorI (AddI src minus_1) src));
10272   predicate(UseBMI1Instructions);
10273   effect(KILL cr);
10274 
10275   format %{ "blsmskl $dst, $src" %}
10276 
10277   ins_encode %{
10278     __ blsmskl($dst$$Register, $src$$Register);
10279   %}
10280 
10281   ins_pipe(ialu_reg);
10282 %}
10283 
10284 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10285 %{
10286   match(Set dst (AndI (AddI src minus_1) src) );
10287   predicate(UseBMI1Instructions);
10288   effect(KILL cr);
10289 
10290   format %{ "blsrl  $dst, $src" %}
10291 
10292   ins_encode %{
10293     __ blsrl($dst$$Register, $src$$Register);
10294   %}
10295 
10296   ins_pipe(ialu_reg_mem);
10297 %}
10298 
10299 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10300 %{
10301   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10302   predicate(UseBMI1Instructions);
10303   effect(KILL cr);
10304 
10305   ins_cost(125);
10306   format %{ "blsrl  $dst, $src" %}
10307 
10308   ins_encode %{
10309     __ blsrl($dst$$Register, $src$$Address);
10310   %}
10311 
10312   ins_pipe(ialu_reg);
10313 %}
10314 
10315 // Or Instructions
10316 // Or Register with Register
10317 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10318 %{
10319   match(Set dst (OrI dst src));
10320   effect(KILL cr);
10321 
10322   format %{ "orl     $dst, $src\t# int" %}
10323   ins_encode %{
10324     __ orl($dst$$Register, $src$$Register);
10325   %}
10326   ins_pipe(ialu_reg_reg);
10327 %}
10328 
10329 // Or Register with Immediate
10330 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10331 %{
10332   match(Set dst (OrI dst src));
10333   effect(KILL cr);
10334 
10335   format %{ "orl     $dst, $src\t# int" %}
10336   ins_encode %{
10337     __ orl($dst$$Register, $src$$constant);
10338   %}
10339   ins_pipe(ialu_reg);
10340 %}
10341 
10342 // Or Register with Memory
10343 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10344 %{
10345   match(Set dst (OrI dst (LoadI src)));
10346   effect(KILL cr);
10347 
10348   ins_cost(125);
10349   format %{ "orl     $dst, $src\t# int" %}
10350   ins_encode %{
10351     __ orl($dst$$Register, $src$$Address);
10352   %}
10353   ins_pipe(ialu_reg_mem);
10354 %}
10355 
10356 // Or Memory with Register
10357 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10358 %{
10359   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10360   effect(KILL cr);
10361 
10362   ins_cost(150);
10363   format %{ "orb    $dst, $src\t# byte" %}
10364   ins_encode %{
10365     __ orb($dst$$Address, $src$$Register);
10366   %}
10367   ins_pipe(ialu_mem_reg);
10368 %}
10369 
10370 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10371 %{
10372   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10373   effect(KILL cr);
10374 
10375   ins_cost(150);
10376   format %{ "orl     $dst, $src\t# int" %}
10377   ins_encode %{
10378     __ orl($dst$$Address, $src$$Register);
10379   %}
10380   ins_pipe(ialu_mem_reg);
10381 %}
10382 
10383 // Or Memory with Immediate
10384 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10385 %{
10386   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10387   effect(KILL cr);
10388 
10389   ins_cost(125);
10390   format %{ "orl     $dst, $src\t# int" %}
10391   ins_encode %{
10392     __ orl($dst$$Address, $src$$constant);
10393   %}
10394   ins_pipe(ialu_mem_imm);
10395 %}
10396 
10397 // Xor Instructions
10398 // Xor Register with Register
10399 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10400 %{
10401   match(Set dst (XorI dst src));
10402   effect(KILL cr);
10403 
10404   format %{ "xorl    $dst, $src\t# int" %}
10405   ins_encode %{
10406     __ xorl($dst$$Register, $src$$Register);
10407   %}
10408   ins_pipe(ialu_reg_reg);
10409 %}
10410 
10411 // Xor Register with Immediate -1
10412 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10413   match(Set dst (XorI dst imm));
10414 
10415   format %{ "not    $dst" %}
10416   ins_encode %{
10417      __ notl($dst$$Register);
10418   %}
10419   ins_pipe(ialu_reg);
10420 %}
10421 
10422 // Xor Register with Immediate
10423 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10424 %{
10425   match(Set dst (XorI dst src));
10426   effect(KILL cr);
10427 
10428   format %{ "xorl    $dst, $src\t# int" %}
10429   ins_encode %{
10430     __ xorl($dst$$Register, $src$$constant);
10431   %}
10432   ins_pipe(ialu_reg);
10433 %}
10434 
10435 // Xor Register with Memory
10436 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10437 %{
10438   match(Set dst (XorI dst (LoadI src)));
10439   effect(KILL cr);
10440 
10441   ins_cost(125);
10442   format %{ "xorl    $dst, $src\t# int" %}
10443   ins_encode %{
10444     __ xorl($dst$$Register, $src$$Address);
10445   %}
10446   ins_pipe(ialu_reg_mem);
10447 %}
10448 
10449 // Xor Memory with Register
10450 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10451 %{
10452   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10453   effect(KILL cr);
10454 
10455   ins_cost(150);
10456   format %{ "xorb    $dst, $src\t# byte" %}
10457   ins_encode %{
10458     __ xorb($dst$$Address, $src$$Register);
10459   %}
10460   ins_pipe(ialu_mem_reg);
10461 %}
10462 
10463 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10464 %{
10465   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10466   effect(KILL cr);
10467 
10468   ins_cost(150);
10469   format %{ "xorl    $dst, $src\t# int" %}
10470   ins_encode %{
10471     __ xorl($dst$$Address, $src$$Register);
10472   %}
10473   ins_pipe(ialu_mem_reg);
10474 %}
10475 
10476 // Xor Memory with Immediate
10477 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10478 %{
10479   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10480   effect(KILL cr);
10481 
10482   ins_cost(125);
10483   format %{ "xorl    $dst, $src\t# int" %}
10484   ins_encode %{
10485     __ xorl($dst$$Address, $src$$constant);
10486   %}
10487   ins_pipe(ialu_mem_imm);
10488 %}
10489 
10490 
10491 // Long Logical Instructions
10492 
10493 // And Instructions
10494 // And Register with Register
10495 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10496 %{
10497   match(Set dst (AndL dst src));
10498   effect(KILL cr);
10499 
10500   format %{ "andq    $dst, $src\t# long" %}
10501   ins_encode %{
10502     __ andq($dst$$Register, $src$$Register);
10503   %}
10504   ins_pipe(ialu_reg_reg);
10505 %}
10506 
10507 // And Register with Immediate 255
10508 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
10509 %{
10510   match(Set dst (AndL dst src));
10511 
10512   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
10513   ins_encode %{
10514     __ movzbq($dst$$Register, $dst$$Register);
10515   %}
10516   ins_pipe(ialu_reg);
10517 %}
10518 
10519 // And Register with Immediate 65535
10520 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
10521 %{
10522   match(Set dst (AndL dst src));
10523 
10524   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
10525   ins_encode %{
10526     __ movzwq($dst$$Register, $dst$$Register);
10527   %}
10528   ins_pipe(ialu_reg);
10529 %}
10530 
10531 // And Register with Immediate
10532 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10533 %{
10534   match(Set dst (AndL dst src));
10535   effect(KILL cr);
10536 
10537   format %{ "andq    $dst, $src\t# long" %}
10538   ins_encode %{
10539     __ andq($dst$$Register, $src$$constant);
10540   %}
10541   ins_pipe(ialu_reg);
10542 %}
10543 
10544 // And Register with Memory
10545 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10546 %{
10547   match(Set dst (AndL dst (LoadL src)));
10548   effect(KILL cr);
10549 
10550   ins_cost(125);
10551   format %{ "andq    $dst, $src\t# long" %}
10552   ins_encode %{
10553     __ andq($dst$$Register, $src$$Address);
10554   %}
10555   ins_pipe(ialu_reg_mem);
10556 %}
10557 
10558 // And Memory with Register
10559 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10560 %{
10561   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10562   effect(KILL cr);
10563 
10564   ins_cost(150);
10565   format %{ "andq    $dst, $src\t# long" %}
10566   ins_encode %{
10567     __ andq($dst$$Address, $src$$Register);
10568   %}
10569   ins_pipe(ialu_mem_reg);
10570 %}
10571 
10572 // And Memory with Immediate
10573 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10574 %{
10575   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10576   effect(KILL cr);
10577 
10578   ins_cost(125);
10579   format %{ "andq    $dst, $src\t# long" %}
10580   ins_encode %{
10581     __ andq($dst$$Address, $src$$constant);
10582   %}
10583   ins_pipe(ialu_mem_imm);
10584 %}
10585 
10586 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10587 %{
10588   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10589   // because AND/OR works well enough for 8/32-bit values.
10590   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10591 
10592   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10593   effect(KILL cr);
10594 
10595   ins_cost(125);
10596   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10597   ins_encode %{
10598     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10599   %}
10600   ins_pipe(ialu_mem_imm);
10601 %}
10602 
10603 // BMI1 instructions
10604 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10605   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10606   predicate(UseBMI1Instructions);
10607   effect(KILL cr);
10608 
10609   ins_cost(125);
10610   format %{ "andnq  $dst, $src1, $src2" %}
10611 
10612   ins_encode %{
10613     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10614   %}
10615   ins_pipe(ialu_reg_mem);
10616 %}
10617 
10618 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10619   match(Set dst (AndL (XorL src1 minus_1) src2));
10620   predicate(UseBMI1Instructions);
10621   effect(KILL cr);
10622 
10623   format %{ "andnq  $dst, $src1, $src2" %}
10624 
10625   ins_encode %{
10626   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10627   %}
10628   ins_pipe(ialu_reg_mem);
10629 %}
10630 
10631 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10632   match(Set dst (AndL (SubL imm_zero src) src));
10633   predicate(UseBMI1Instructions);
10634   effect(KILL cr);
10635 
10636   format %{ "blsiq  $dst, $src" %}
10637 
10638   ins_encode %{
10639     __ blsiq($dst$$Register, $src$$Register);
10640   %}
10641   ins_pipe(ialu_reg);
10642 %}
10643 
10644 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10645   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10646   predicate(UseBMI1Instructions);
10647   effect(KILL cr);
10648 
10649   ins_cost(125);
10650   format %{ "blsiq  $dst, $src" %}
10651 
10652   ins_encode %{
10653     __ blsiq($dst$$Register, $src$$Address);
10654   %}
10655   ins_pipe(ialu_reg_mem);
10656 %}
10657 
10658 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10659 %{
10660   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10661   predicate(UseBMI1Instructions);
10662   effect(KILL cr);
10663 
10664   ins_cost(125);
10665   format %{ "blsmskq $dst, $src" %}
10666 
10667   ins_encode %{
10668     __ blsmskq($dst$$Register, $src$$Address);
10669   %}
10670   ins_pipe(ialu_reg_mem);
10671 %}
10672 
10673 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10674 %{
10675   match(Set dst (XorL (AddL src minus_1) src));
10676   predicate(UseBMI1Instructions);
10677   effect(KILL cr);
10678 
10679   format %{ "blsmskq $dst, $src" %}
10680 
10681   ins_encode %{
10682     __ blsmskq($dst$$Register, $src$$Register);
10683   %}
10684 
10685   ins_pipe(ialu_reg);
10686 %}
10687 
10688 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10689 %{
10690   match(Set dst (AndL (AddL src minus_1) src) );
10691   predicate(UseBMI1Instructions);
10692   effect(KILL cr);
10693 
10694   format %{ "blsrq  $dst, $src" %}
10695 
10696   ins_encode %{
10697     __ blsrq($dst$$Register, $src$$Register);
10698   %}
10699 
10700   ins_pipe(ialu_reg);
10701 %}
10702 
10703 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10704 %{
10705   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10706   predicate(UseBMI1Instructions);
10707   effect(KILL cr);
10708 
10709   ins_cost(125);
10710   format %{ "blsrq  $dst, $src" %}
10711 
10712   ins_encode %{
10713     __ blsrq($dst$$Register, $src$$Address);
10714   %}
10715 
10716   ins_pipe(ialu_reg);
10717 %}
10718 
10719 // Or Instructions
10720 // Or Register with Register
10721 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10722 %{
10723   match(Set dst (OrL dst src));
10724   effect(KILL cr);
10725 
10726   format %{ "orq     $dst, $src\t# long" %}
10727   ins_encode %{
10728     __ orq($dst$$Register, $src$$Register);
10729   %}
10730   ins_pipe(ialu_reg_reg);
10731 %}
10732 
10733 // Use any_RegP to match R15 (TLS register) without spilling.
10734 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10735   match(Set dst (OrL dst (CastP2X src)));
10736   effect(KILL cr);
10737 
10738   format %{ "orq     $dst, $src\t# long" %}
10739   ins_encode %{
10740     __ orq($dst$$Register, $src$$Register);
10741   %}
10742   ins_pipe(ialu_reg_reg);
10743 %}
10744 
10745 
10746 // Or Register with Immediate
10747 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10748 %{
10749   match(Set dst (OrL dst src));
10750   effect(KILL cr);
10751 
10752   format %{ "orq     $dst, $src\t# long" %}
10753   ins_encode %{
10754     __ orq($dst$$Register, $src$$constant);
10755   %}
10756   ins_pipe(ialu_reg);
10757 %}
10758 
10759 // Or Register with Memory
10760 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10761 %{
10762   match(Set dst (OrL dst (LoadL src)));
10763   effect(KILL cr);
10764 
10765   ins_cost(125);
10766   format %{ "orq     $dst, $src\t# long" %}
10767   ins_encode %{
10768     __ orq($dst$$Register, $src$$Address);
10769   %}
10770   ins_pipe(ialu_reg_mem);
10771 %}
10772 
10773 // Or Memory with Register
10774 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10775 %{
10776   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10777   effect(KILL cr);
10778 
10779   ins_cost(150);
10780   format %{ "orq     $dst, $src\t# long" %}
10781   ins_encode %{
10782     __ orq($dst$$Address, $src$$Register);
10783   %}
10784   ins_pipe(ialu_mem_reg);
10785 %}
10786 
10787 // Or Memory with Immediate
10788 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10789 %{
10790   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10791   effect(KILL cr);
10792 
10793   ins_cost(125);
10794   format %{ "orq     $dst, $src\t# long" %}
10795   ins_encode %{
10796     __ orq($dst$$Address, $src$$constant);
10797   %}
10798   ins_pipe(ialu_mem_imm);
10799 %}
10800 
10801 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10802 %{
10803   // con should be a pure 64-bit power of 2 immediate
10804   // because AND/OR works well enough for 8/32-bit values.
10805   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10806 
10807   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10808   effect(KILL cr);
10809 
10810   ins_cost(125);
10811   format %{ "btsq    $dst, log2($con)\t# long" %}
10812   ins_encode %{
10813     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10814   %}
10815   ins_pipe(ialu_mem_imm);
10816 %}
10817 
10818 // Xor Instructions
10819 // Xor Register with Register
10820 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10821 %{
10822   match(Set dst (XorL dst src));
10823   effect(KILL cr);
10824 
10825   format %{ "xorq    $dst, $src\t# long" %}
10826   ins_encode %{
10827     __ xorq($dst$$Register, $src$$Register);
10828   %}
10829   ins_pipe(ialu_reg_reg);
10830 %}
10831 
10832 // Xor Register with Immediate -1
10833 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10834   match(Set dst (XorL dst imm));
10835 
10836   format %{ "notq   $dst" %}
10837   ins_encode %{
10838      __ notq($dst$$Register);
10839   %}
10840   ins_pipe(ialu_reg);
10841 %}
10842 
10843 // Xor Register with Immediate
10844 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10845 %{
10846   match(Set dst (XorL dst src));
10847   effect(KILL cr);
10848 
10849   format %{ "xorq    $dst, $src\t# long" %}
10850   ins_encode %{
10851     __ xorq($dst$$Register, $src$$constant);
10852   %}
10853   ins_pipe(ialu_reg);
10854 %}
10855 
10856 // Xor Register with Memory
10857 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10858 %{
10859   match(Set dst (XorL dst (LoadL src)));
10860   effect(KILL cr);
10861 
10862   ins_cost(125);
10863   format %{ "xorq    $dst, $src\t# long" %}
10864   ins_encode %{
10865     __ xorq($dst$$Register, $src$$Address);
10866   %}
10867   ins_pipe(ialu_reg_mem);
10868 %}
10869 
10870 // Xor Memory with Register
10871 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10872 %{
10873   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10874   effect(KILL cr);
10875 
10876   ins_cost(150);
10877   format %{ "xorq    $dst, $src\t# long" %}
10878   ins_encode %{
10879     __ xorq($dst$$Address, $src$$Register);
10880   %}
10881   ins_pipe(ialu_mem_reg);
10882 %}
10883 
10884 // Xor Memory with Immediate
10885 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10886 %{
10887   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10888   effect(KILL cr);
10889 
10890   ins_cost(125);
10891   format %{ "xorq    $dst, $src\t# long" %}
10892   ins_encode %{
10893     __ xorq($dst$$Address, $src$$constant);
10894   %}
10895   ins_pipe(ialu_mem_imm);
10896 %}
10897 
10898 // Convert Int to Boolean
10899 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10900 %{
10901   match(Set dst (Conv2B src));
10902   effect(KILL cr);
10903 
10904   format %{ "testl   $src, $src\t# ci2b\n\t"
10905             "setnz   $dst\n\t"
10906             "movzbl  $dst, $dst" %}
10907   ins_encode %{
10908     __ testl($src$$Register, $src$$Register);
10909     __ set_byte_if_not_zero($dst$$Register);
10910     __ movzbl($dst$$Register, $dst$$Register);
10911   %}
10912   ins_pipe(pipe_slow); // XXX
10913 %}
10914 
10915 // Convert Pointer to Boolean
10916 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10917 %{
10918   match(Set dst (Conv2B src));
10919   effect(KILL cr);
10920 
10921   format %{ "testq   $src, $src\t# cp2b\n\t"
10922             "setnz   $dst\n\t"
10923             "movzbl  $dst, $dst" %}
10924   ins_encode %{
10925     __ testq($src$$Register, $src$$Register);
10926     __ set_byte_if_not_zero($dst$$Register);
10927     __ movzbl($dst$$Register, $dst$$Register);
10928   %}
10929   ins_pipe(pipe_slow); // XXX
10930 %}
10931 
10932 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10933 %{
10934   match(Set dst (CmpLTMask p q));
10935   effect(KILL cr);
10936 
10937   ins_cost(400);
10938   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10939             "setlt   $dst\n\t"
10940             "movzbl  $dst, $dst\n\t"
10941             "negl    $dst" %}
10942   ins_encode %{
10943     __ cmpl($p$$Register, $q$$Register);
10944     __ setl($dst$$Register);
10945     __ movzbl($dst$$Register, $dst$$Register);
10946     __ negl($dst$$Register);
10947   %}
10948   ins_pipe(pipe_slow);
10949 %}
10950 
10951 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10952 %{
10953   match(Set dst (CmpLTMask dst zero));
10954   effect(KILL cr);
10955 
10956   ins_cost(100);
10957   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10958   ins_encode %{
10959     __ sarl($dst$$Register, 31);
10960   %}
10961   ins_pipe(ialu_reg);
10962 %}
10963 
10964 /* Better to save a register than avoid a branch */
10965 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10966 %{
10967   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10968   effect(KILL cr);
10969   ins_cost(300);
10970   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10971             "jge     done\n\t"
10972             "addl    $p,$y\n"
10973             "done:   " %}
10974   ins_encode %{
10975     Register Rp = $p$$Register;
10976     Register Rq = $q$$Register;
10977     Register Ry = $y$$Register;
10978     Label done;
10979     __ subl(Rp, Rq);
10980     __ jccb(Assembler::greaterEqual, done);
10981     __ addl(Rp, Ry);
10982     __ bind(done);
10983   %}
10984   ins_pipe(pipe_cmplt);
10985 %}
10986 
10987 /* Better to save a register than avoid a branch */
10988 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10989 %{
10990   match(Set y (AndI (CmpLTMask p q) y));
10991   effect(KILL cr);
10992 
10993   ins_cost(300);
10994 
10995   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10996             "jlt     done\n\t"
10997             "xorl    $y, $y\n"
10998             "done:   " %}
10999   ins_encode %{
11000     Register Rp = $p$$Register;
11001     Register Rq = $q$$Register;
11002     Register Ry = $y$$Register;
11003     Label done;
11004     __ cmpl(Rp, Rq);
11005     __ jccb(Assembler::less, done);
11006     __ xorl(Ry, Ry);
11007     __ bind(done);
11008   %}
11009   ins_pipe(pipe_cmplt);
11010 %}
11011 
11012 
11013 //---------- FP Instructions------------------------------------------------
11014 
11015 // Really expensive, avoid
11016 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11017 %{
11018   match(Set cr (CmpF src1 src2));
11019 
11020   ins_cost(500);
11021   format %{ "ucomiss $src1, $src2\n\t"
11022             "jnp,s   exit\n\t"
11023             "pushfq\t# saw NaN, set CF\n\t"
11024             "andq    [rsp], #0xffffff2b\n\t"
11025             "popfq\n"
11026     "exit:" %}
11027   ins_encode %{
11028     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11029     emit_cmpfp_fixup(_masm);
11030   %}
11031   ins_pipe(pipe_slow);
11032 %}
11033 
11034 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11035   match(Set cr (CmpF src1 src2));
11036 
11037   ins_cost(100);
11038   format %{ "ucomiss $src1, $src2" %}
11039   ins_encode %{
11040     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11041   %}
11042   ins_pipe(pipe_slow);
11043 %}
11044 
11045 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11046   match(Set cr (CmpF src1 (LoadF src2)));
11047 
11048   ins_cost(100);
11049   format %{ "ucomiss $src1, $src2" %}
11050   ins_encode %{
11051     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11052   %}
11053   ins_pipe(pipe_slow);
11054 %}
11055 
11056 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11057   match(Set cr (CmpF src con));
11058   ins_cost(100);
11059   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11060   ins_encode %{
11061     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11062   %}
11063   ins_pipe(pipe_slow);
11064 %}
11065 
11066 // Really expensive, avoid
11067 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11068 %{
11069   match(Set cr (CmpD src1 src2));
11070 
11071   ins_cost(500);
11072   format %{ "ucomisd $src1, $src2\n\t"
11073             "jnp,s   exit\n\t"
11074             "pushfq\t# saw NaN, set CF\n\t"
11075             "andq    [rsp], #0xffffff2b\n\t"
11076             "popfq\n"
11077     "exit:" %}
11078   ins_encode %{
11079     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11080     emit_cmpfp_fixup(_masm);
11081   %}
11082   ins_pipe(pipe_slow);
11083 %}
11084 
11085 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11086   match(Set cr (CmpD src1 src2));
11087 
11088   ins_cost(100);
11089   format %{ "ucomisd $src1, $src2 test" %}
11090   ins_encode %{
11091     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11092   %}
11093   ins_pipe(pipe_slow);
11094 %}
11095 
11096 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11097   match(Set cr (CmpD src1 (LoadD src2)));
11098 
11099   ins_cost(100);
11100   format %{ "ucomisd $src1, $src2" %}
11101   ins_encode %{
11102     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11103   %}
11104   ins_pipe(pipe_slow);
11105 %}
11106 
11107 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11108   match(Set cr (CmpD src con));
11109   ins_cost(100);
11110   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11111   ins_encode %{
11112     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11113   %}
11114   ins_pipe(pipe_slow);
11115 %}
11116 
11117 // Compare into -1,0,1
11118 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11119 %{
11120   match(Set dst (CmpF3 src1 src2));
11121   effect(KILL cr);
11122 
11123   ins_cost(275);
11124   format %{ "ucomiss $src1, $src2\n\t"
11125             "movl    $dst, #-1\n\t"
11126             "jp,s    done\n\t"
11127             "jb,s    done\n\t"
11128             "setne   $dst\n\t"
11129             "movzbl  $dst, $dst\n"
11130     "done:" %}
11131   ins_encode %{
11132     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11133     emit_cmpfp3(_masm, $dst$$Register);
11134   %}
11135   ins_pipe(pipe_slow);
11136 %}
11137 
11138 // Compare into -1,0,1
11139 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11140 %{
11141   match(Set dst (CmpF3 src1 (LoadF src2)));
11142   effect(KILL cr);
11143 
11144   ins_cost(275);
11145   format %{ "ucomiss $src1, $src2\n\t"
11146             "movl    $dst, #-1\n\t"
11147             "jp,s    done\n\t"
11148             "jb,s    done\n\t"
11149             "setne   $dst\n\t"
11150             "movzbl  $dst, $dst\n"
11151     "done:" %}
11152   ins_encode %{
11153     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11154     emit_cmpfp3(_masm, $dst$$Register);
11155   %}
11156   ins_pipe(pipe_slow);
11157 %}
11158 
11159 // Compare into -1,0,1
11160 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11161   match(Set dst (CmpF3 src con));
11162   effect(KILL cr);
11163 
11164   ins_cost(275);
11165   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11166             "movl    $dst, #-1\n\t"
11167             "jp,s    done\n\t"
11168             "jb,s    done\n\t"
11169             "setne   $dst\n\t"
11170             "movzbl  $dst, $dst\n"
11171     "done:" %}
11172   ins_encode %{
11173     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11174     emit_cmpfp3(_masm, $dst$$Register);
11175   %}
11176   ins_pipe(pipe_slow);
11177 %}
11178 
11179 // Compare into -1,0,1
11180 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11181 %{
11182   match(Set dst (CmpD3 src1 src2));
11183   effect(KILL cr);
11184 
11185   ins_cost(275);
11186   format %{ "ucomisd $src1, $src2\n\t"
11187             "movl    $dst, #-1\n\t"
11188             "jp,s    done\n\t"
11189             "jb,s    done\n\t"
11190             "setne   $dst\n\t"
11191             "movzbl  $dst, $dst\n"
11192     "done:" %}
11193   ins_encode %{
11194     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11195     emit_cmpfp3(_masm, $dst$$Register);
11196   %}
11197   ins_pipe(pipe_slow);
11198 %}
11199 
11200 // Compare into -1,0,1
11201 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11202 %{
11203   match(Set dst (CmpD3 src1 (LoadD src2)));
11204   effect(KILL cr);
11205 
11206   ins_cost(275);
11207   format %{ "ucomisd $src1, $src2\n\t"
11208             "movl    $dst, #-1\n\t"
11209             "jp,s    done\n\t"
11210             "jb,s    done\n\t"
11211             "setne   $dst\n\t"
11212             "movzbl  $dst, $dst\n"
11213     "done:" %}
11214   ins_encode %{
11215     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11216     emit_cmpfp3(_masm, $dst$$Register);
11217   %}
11218   ins_pipe(pipe_slow);
11219 %}
11220 
11221 // Compare into -1,0,1
11222 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11223   match(Set dst (CmpD3 src con));
11224   effect(KILL cr);
11225 
11226   ins_cost(275);
11227   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11228             "movl    $dst, #-1\n\t"
11229             "jp,s    done\n\t"
11230             "jb,s    done\n\t"
11231             "setne   $dst\n\t"
11232             "movzbl  $dst, $dst\n"
11233     "done:" %}
11234   ins_encode %{
11235     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11236     emit_cmpfp3(_masm, $dst$$Register);
11237   %}
11238   ins_pipe(pipe_slow);
11239 %}
11240 
11241 //----------Arithmetic Conversion Instructions---------------------------------
11242 
11243 instruct convF2D_reg_reg(regD dst, regF src)
11244 %{
11245   match(Set dst (ConvF2D src));
11246 
11247   format %{ "cvtss2sd $dst, $src" %}
11248   ins_encode %{
11249     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11250   %}
11251   ins_pipe(pipe_slow); // XXX
11252 %}
11253 
11254 instruct convF2D_reg_mem(regD dst, memory src)
11255 %{
11256   match(Set dst (ConvF2D (LoadF src)));
11257 
11258   format %{ "cvtss2sd $dst, $src" %}
11259   ins_encode %{
11260     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11261   %}
11262   ins_pipe(pipe_slow); // XXX
11263 %}
11264 
11265 instruct convD2F_reg_reg(regF dst, regD src)
11266 %{
11267   match(Set dst (ConvD2F src));
11268 
11269   format %{ "cvtsd2ss $dst, $src" %}
11270   ins_encode %{
11271     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11272   %}
11273   ins_pipe(pipe_slow); // XXX
11274 %}
11275 
11276 instruct convD2F_reg_mem(regF dst, memory src)
11277 %{
11278   match(Set dst (ConvD2F (LoadD src)));
11279 
11280   format %{ "cvtsd2ss $dst, $src" %}
11281   ins_encode %{
11282     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11283   %}
11284   ins_pipe(pipe_slow); // XXX
11285 %}
11286 
11287 // XXX do mem variants
11288 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11289 %{
11290   match(Set dst (ConvF2I src));
11291   effect(KILL cr);
11292   format %{ "convert_f2i $dst,$src" %}
11293   ins_encode %{
11294     __ convert_f2i($dst$$Register, $src$$XMMRegister);
11295   %}
11296   ins_pipe(pipe_slow);
11297 %}
11298 
11299 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11300 %{
11301   match(Set dst (ConvF2L src));
11302   effect(KILL cr);
11303   format %{ "convert_f2l $dst,$src"%}
11304   ins_encode %{
11305     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11306   %}
11307   ins_pipe(pipe_slow);
11308 %}
11309 
11310 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11311 %{
11312   match(Set dst (ConvD2I src));
11313   effect(KILL cr);
11314   format %{ "convert_d2i $dst,$src"%}
11315   ins_encode %{
11316     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11317   %}
11318   ins_pipe(pipe_slow);
11319 %}
11320 
11321 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11322 %{
11323   match(Set dst (ConvD2L src));
11324   effect(KILL cr);
11325   format %{ "convert_d2l $dst,$src"%}
11326   ins_encode %{
11327     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11328   %}
11329   ins_pipe(pipe_slow);
11330 %}
11331 
11332 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11333 %{
11334   match(Set dst (RoundD src));
11335   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11336   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11337   ins_encode %{
11338     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11339   %}
11340   ins_pipe(pipe_slow);
11341 %}
11342 
11343 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11344 %{
11345   match(Set dst (RoundF src));
11346   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11347   format %{ "round_float $dst,$src" %}
11348   ins_encode %{
11349     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11350   %}
11351   ins_pipe(pipe_slow);
11352 %}
11353 
11354 instruct convI2F_reg_reg(regF dst, rRegI src)
11355 %{
11356   predicate(!UseXmmI2F);
11357   match(Set dst (ConvI2F src));
11358 
11359   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11360   ins_encode %{
11361     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11362   %}
11363   ins_pipe(pipe_slow); // XXX
11364 %}
11365 
11366 instruct convI2F_reg_mem(regF dst, memory src)
11367 %{
11368   match(Set dst (ConvI2F (LoadI src)));
11369 
11370   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11371   ins_encode %{
11372     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11373   %}
11374   ins_pipe(pipe_slow); // XXX
11375 %}
11376 
11377 instruct convI2D_reg_reg(regD dst, rRegI src)
11378 %{
11379   predicate(!UseXmmI2D);
11380   match(Set dst (ConvI2D src));
11381 
11382   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11383   ins_encode %{
11384     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11385   %}
11386   ins_pipe(pipe_slow); // XXX
11387 %}
11388 
11389 instruct convI2D_reg_mem(regD dst, memory src)
11390 %{
11391   match(Set dst (ConvI2D (LoadI src)));
11392 
11393   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11394   ins_encode %{
11395     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11396   %}
11397   ins_pipe(pipe_slow); // XXX
11398 %}
11399 
11400 instruct convXI2F_reg(regF dst, rRegI src)
11401 %{
11402   predicate(UseXmmI2F);
11403   match(Set dst (ConvI2F src));
11404 
11405   format %{ "movdl $dst, $src\n\t"
11406             "cvtdq2psl $dst, $dst\t# i2f" %}
11407   ins_encode %{
11408     __ movdl($dst$$XMMRegister, $src$$Register);
11409     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11410   %}
11411   ins_pipe(pipe_slow); // XXX
11412 %}
11413 
11414 instruct convXI2D_reg(regD dst, rRegI src)
11415 %{
11416   predicate(UseXmmI2D);
11417   match(Set dst (ConvI2D src));
11418 
11419   format %{ "movdl $dst, $src\n\t"
11420             "cvtdq2pdl $dst, $dst\t# i2d" %}
11421   ins_encode %{
11422     __ movdl($dst$$XMMRegister, $src$$Register);
11423     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11424   %}
11425   ins_pipe(pipe_slow); // XXX
11426 %}
11427 
11428 instruct convL2F_reg_reg(regF dst, rRegL src)
11429 %{
11430   match(Set dst (ConvL2F src));
11431 
11432   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11433   ins_encode %{
11434     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11435   %}
11436   ins_pipe(pipe_slow); // XXX
11437 %}
11438 
11439 instruct convL2F_reg_mem(regF dst, memory src)
11440 %{
11441   match(Set dst (ConvL2F (LoadL src)));
11442 
11443   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11444   ins_encode %{
11445     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11446   %}
11447   ins_pipe(pipe_slow); // XXX
11448 %}
11449 
11450 instruct convL2D_reg_reg(regD dst, rRegL src)
11451 %{
11452   match(Set dst (ConvL2D src));
11453 
11454   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11455   ins_encode %{
11456     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11457   %}
11458   ins_pipe(pipe_slow); // XXX
11459 %}
11460 
11461 instruct convL2D_reg_mem(regD dst, memory src)
11462 %{
11463   match(Set dst (ConvL2D (LoadL src)));
11464 
11465   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11466   ins_encode %{
11467     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11468   %}
11469   ins_pipe(pipe_slow); // XXX
11470 %}
11471 
11472 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11473 %{
11474   match(Set dst (ConvI2L src));
11475 
11476   ins_cost(125);
11477   format %{ "movslq  $dst, $src\t# i2l" %}
11478   ins_encode %{
11479     __ movslq($dst$$Register, $src$$Register);
11480   %}
11481   ins_pipe(ialu_reg_reg);
11482 %}
11483 
11484 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11485 // %{
11486 //   match(Set dst (ConvI2L src));
11487 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11488 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11489 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11490 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11491 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11492 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11493 
11494 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11495 //   ins_encode(enc_copy(dst, src));
11496 // //   opcode(0x63); // needs REX.W
11497 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11498 //   ins_pipe(ialu_reg_reg);
11499 // %}
11500 
11501 // Zero-extend convert int to long
11502 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11503 %{
11504   match(Set dst (AndL (ConvI2L src) mask));
11505 
11506   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11507   ins_encode %{
11508     if ($dst$$reg != $src$$reg) {
11509       __ movl($dst$$Register, $src$$Register);
11510     }
11511   %}
11512   ins_pipe(ialu_reg_reg);
11513 %}
11514 
11515 // Zero-extend convert int to long
11516 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11517 %{
11518   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11519 
11520   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11521   ins_encode %{
11522     __ movl($dst$$Register, $src$$Address);
11523   %}
11524   ins_pipe(ialu_reg_mem);
11525 %}
11526 
11527 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11528 %{
11529   match(Set dst (AndL src mask));
11530 
11531   format %{ "movl    $dst, $src\t# zero-extend long" %}
11532   ins_encode %{
11533     __ movl($dst$$Register, $src$$Register);
11534   %}
11535   ins_pipe(ialu_reg_reg);
11536 %}
11537 
11538 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11539 %{
11540   match(Set dst (ConvL2I src));
11541 
11542   format %{ "movl    $dst, $src\t# l2i" %}
11543   ins_encode %{
11544     __ movl($dst$$Register, $src$$Register);
11545   %}
11546   ins_pipe(ialu_reg_reg);
11547 %}
11548 
11549 
11550 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11551   match(Set dst (MoveF2I src));
11552   effect(DEF dst, USE src);
11553 
11554   ins_cost(125);
11555   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11556   ins_encode %{
11557     __ movl($dst$$Register, Address(rsp, $src$$disp));
11558   %}
11559   ins_pipe(ialu_reg_mem);
11560 %}
11561 
11562 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11563   match(Set dst (MoveI2F src));
11564   effect(DEF dst, USE src);
11565 
11566   ins_cost(125);
11567   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11568   ins_encode %{
11569     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11570   %}
11571   ins_pipe(pipe_slow);
11572 %}
11573 
11574 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11575   match(Set dst (MoveD2L src));
11576   effect(DEF dst, USE src);
11577 
11578   ins_cost(125);
11579   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11580   ins_encode %{
11581     __ movq($dst$$Register, Address(rsp, $src$$disp));
11582   %}
11583   ins_pipe(ialu_reg_mem);
11584 %}
11585 
11586 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11587   predicate(!UseXmmLoadAndClearUpper);
11588   match(Set dst (MoveL2D src));
11589   effect(DEF dst, USE src);
11590 
11591   ins_cost(125);
11592   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11593   ins_encode %{
11594     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11595   %}
11596   ins_pipe(pipe_slow);
11597 %}
11598 
11599 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11600   predicate(UseXmmLoadAndClearUpper);
11601   match(Set dst (MoveL2D src));
11602   effect(DEF dst, USE src);
11603 
11604   ins_cost(125);
11605   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11606   ins_encode %{
11607     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11608   %}
11609   ins_pipe(pipe_slow);
11610 %}
11611 
11612 
11613 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11614   match(Set dst (MoveF2I src));
11615   effect(DEF dst, USE src);
11616 
11617   ins_cost(95); // XXX
11618   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11619   ins_encode %{
11620     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11621   %}
11622   ins_pipe(pipe_slow);
11623 %}
11624 
11625 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11626   match(Set dst (MoveI2F src));
11627   effect(DEF dst, USE src);
11628 
11629   ins_cost(100);
11630   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11631   ins_encode %{
11632     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11633   %}
11634   ins_pipe( ialu_mem_reg );
11635 %}
11636 
11637 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11638   match(Set dst (MoveD2L src));
11639   effect(DEF dst, USE src);
11640 
11641   ins_cost(95); // XXX
11642   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11643   ins_encode %{
11644     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11645   %}
11646   ins_pipe(pipe_slow);
11647 %}
11648 
11649 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11650   match(Set dst (MoveL2D src));
11651   effect(DEF dst, USE src);
11652 
11653   ins_cost(100);
11654   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11655   ins_encode %{
11656     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11657   %}
11658   ins_pipe(ialu_mem_reg);
11659 %}
11660 
11661 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11662   match(Set dst (MoveF2I src));
11663   effect(DEF dst, USE src);
11664   ins_cost(85);
11665   format %{ "movd    $dst,$src\t# MoveF2I" %}
11666   ins_encode %{
11667     __ movdl($dst$$Register, $src$$XMMRegister);
11668   %}
11669   ins_pipe( pipe_slow );
11670 %}
11671 
11672 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11673   match(Set dst (MoveD2L src));
11674   effect(DEF dst, USE src);
11675   ins_cost(85);
11676   format %{ "movd    $dst,$src\t# MoveD2L" %}
11677   ins_encode %{
11678     __ movdq($dst$$Register, $src$$XMMRegister);
11679   %}
11680   ins_pipe( pipe_slow );
11681 %}
11682 
11683 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11684   match(Set dst (MoveI2F src));
11685   effect(DEF dst, USE src);
11686   ins_cost(100);
11687   format %{ "movd    $dst,$src\t# MoveI2F" %}
11688   ins_encode %{
11689     __ movdl($dst$$XMMRegister, $src$$Register);
11690   %}
11691   ins_pipe( pipe_slow );
11692 %}
11693 
11694 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11695   match(Set dst (MoveL2D src));
11696   effect(DEF dst, USE src);
11697   ins_cost(100);
11698   format %{ "movd    $dst,$src\t# MoveL2D" %}
11699   ins_encode %{
11700      __ movdq($dst$$XMMRegister, $src$$Register);
11701   %}
11702   ins_pipe( pipe_slow );
11703 %}
11704 
11705 // Fast clearing of an array
11706 // Small ClearArray non-AVX512.
11707 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11708                   Universe dummy, rFlagsReg cr)
11709 %{
11710   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11711   match(Set dummy (ClearArray cnt base));
11712   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11713 
11714   format %{ $$template
11715     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11716     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11717     $$emit$$"jg      LARGE\n\t"
11718     $$emit$$"dec     rcx\n\t"
11719     $$emit$$"js      DONE\t# Zero length\n\t"
11720     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11721     $$emit$$"dec     rcx\n\t"
11722     $$emit$$"jge     LOOP\n\t"
11723     $$emit$$"jmp     DONE\n\t"
11724     $$emit$$"# LARGE:\n\t"
11725     if (UseFastStosb) {
11726        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11727        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11728     } else if (UseXMMForObjInit) {
11729        $$emit$$"mov     rdi,rax\n\t"
11730        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11731        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11732        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11733        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11734        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11735        $$emit$$"add     0x40,rax\n\t"
11736        $$emit$$"# L_zero_64_bytes:\n\t"
11737        $$emit$$"sub     0x8,rcx\n\t"
11738        $$emit$$"jge     L_loop\n\t"
11739        $$emit$$"add     0x4,rcx\n\t"
11740        $$emit$$"jl      L_tail\n\t"
11741        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11742        $$emit$$"add     0x20,rax\n\t"
11743        $$emit$$"sub     0x4,rcx\n\t"
11744        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11745        $$emit$$"add     0x4,rcx\n\t"
11746        $$emit$$"jle     L_end\n\t"
11747        $$emit$$"dec     rcx\n\t"
11748        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11749        $$emit$$"vmovq   xmm0,(rax)\n\t"
11750        $$emit$$"add     0x8,rax\n\t"
11751        $$emit$$"dec     rcx\n\t"
11752        $$emit$$"jge     L_sloop\n\t"
11753        $$emit$$"# L_end:\n\t"
11754     } else {
11755        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11756     }
11757     $$emit$$"# DONE"
11758   %}
11759   ins_encode %{
11760     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11761                  $tmp$$XMMRegister, false, knoreg);
11762   %}
11763   ins_pipe(pipe_slow);
11764 %}
11765 
11766 // Small ClearArray AVX512 non-constant length.
11767 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11768                        Universe dummy, rFlagsReg cr)
11769 %{
11770   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11771   match(Set dummy (ClearArray cnt base));
11772   ins_cost(125);
11773   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11774 
11775   format %{ $$template
11776     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11777     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11778     $$emit$$"jg      LARGE\n\t"
11779     $$emit$$"dec     rcx\n\t"
11780     $$emit$$"js      DONE\t# Zero length\n\t"
11781     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11782     $$emit$$"dec     rcx\n\t"
11783     $$emit$$"jge     LOOP\n\t"
11784     $$emit$$"jmp     DONE\n\t"
11785     $$emit$$"# LARGE:\n\t"
11786     if (UseFastStosb) {
11787        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11788        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11789     } else if (UseXMMForObjInit) {
11790        $$emit$$"mov     rdi,rax\n\t"
11791        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11792        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11793        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11794        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11795        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11796        $$emit$$"add     0x40,rax\n\t"
11797        $$emit$$"# L_zero_64_bytes:\n\t"
11798        $$emit$$"sub     0x8,rcx\n\t"
11799        $$emit$$"jge     L_loop\n\t"
11800        $$emit$$"add     0x4,rcx\n\t"
11801        $$emit$$"jl      L_tail\n\t"
11802        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11803        $$emit$$"add     0x20,rax\n\t"
11804        $$emit$$"sub     0x4,rcx\n\t"
11805        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11806        $$emit$$"add     0x4,rcx\n\t"
11807        $$emit$$"jle     L_end\n\t"
11808        $$emit$$"dec     rcx\n\t"
11809        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11810        $$emit$$"vmovq   xmm0,(rax)\n\t"
11811        $$emit$$"add     0x8,rax\n\t"
11812        $$emit$$"dec     rcx\n\t"
11813        $$emit$$"jge     L_sloop\n\t"
11814        $$emit$$"# L_end:\n\t"
11815     } else {
11816        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11817     }
11818     $$emit$$"# DONE"
11819   %}
11820   ins_encode %{
11821     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11822                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11823   %}
11824   ins_pipe(pipe_slow);
11825 %}
11826 
11827 // Large ClearArray non-AVX512.
11828 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11829                         Universe dummy, rFlagsReg cr)
11830 %{
11831   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11832   match(Set dummy (ClearArray cnt base));
11833   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11834 
11835   format %{ $$template
11836     if (UseFastStosb) {
11837        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11838        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11839        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11840     } else if (UseXMMForObjInit) {
11841        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11842        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11843        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11844        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11845        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11846        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11847        $$emit$$"add     0x40,rax\n\t"
11848        $$emit$$"# L_zero_64_bytes:\n\t"
11849        $$emit$$"sub     0x8,rcx\n\t"
11850        $$emit$$"jge     L_loop\n\t"
11851        $$emit$$"add     0x4,rcx\n\t"
11852        $$emit$$"jl      L_tail\n\t"
11853        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11854        $$emit$$"add     0x20,rax\n\t"
11855        $$emit$$"sub     0x4,rcx\n\t"
11856        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11857        $$emit$$"add     0x4,rcx\n\t"
11858        $$emit$$"jle     L_end\n\t"
11859        $$emit$$"dec     rcx\n\t"
11860        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11861        $$emit$$"vmovq   xmm0,(rax)\n\t"
11862        $$emit$$"add     0x8,rax\n\t"
11863        $$emit$$"dec     rcx\n\t"
11864        $$emit$$"jge     L_sloop\n\t"
11865        $$emit$$"# L_end:\n\t"
11866     } else {
11867        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11868        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11869     }
11870   %}
11871   ins_encode %{
11872     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11873                  $tmp$$XMMRegister, true, knoreg);
11874   %}
11875   ins_pipe(pipe_slow);
11876 %}
11877 
11878 // Large ClearArray AVX512.
11879 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11880                              Universe dummy, rFlagsReg cr)
11881 %{
11882   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11883   match(Set dummy (ClearArray cnt base));
11884   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11885 
11886   format %{ $$template
11887     if (UseFastStosb) {
11888        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11889        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11890        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11891     } else if (UseXMMForObjInit) {
11892        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11893        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11894        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11895        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11896        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11897        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11898        $$emit$$"add     0x40,rax\n\t"
11899        $$emit$$"# L_zero_64_bytes:\n\t"
11900        $$emit$$"sub     0x8,rcx\n\t"
11901        $$emit$$"jge     L_loop\n\t"
11902        $$emit$$"add     0x4,rcx\n\t"
11903        $$emit$$"jl      L_tail\n\t"
11904        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11905        $$emit$$"add     0x20,rax\n\t"
11906        $$emit$$"sub     0x4,rcx\n\t"
11907        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11908        $$emit$$"add     0x4,rcx\n\t"
11909        $$emit$$"jle     L_end\n\t"
11910        $$emit$$"dec     rcx\n\t"
11911        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11912        $$emit$$"vmovq   xmm0,(rax)\n\t"
11913        $$emit$$"add     0x8,rax\n\t"
11914        $$emit$$"dec     rcx\n\t"
11915        $$emit$$"jge     L_sloop\n\t"
11916        $$emit$$"# L_end:\n\t"
11917     } else {
11918        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11919        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11920     }
11921   %}
11922   ins_encode %{
11923     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11924                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11925   %}
11926   ins_pipe(pipe_slow);
11927 %}
11928 
11929 // Small ClearArray AVX512 constant length.
11930 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11931 %{
11932   predicate(!((ClearArrayNode*)n)->is_large() &&
11933               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11934   match(Set dummy (ClearArray cnt base));
11935   ins_cost(100);
11936   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11937   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11938   ins_encode %{
11939    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11940   %}
11941   ins_pipe(pipe_slow);
11942 %}
11943 
11944 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11945                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11946 %{
11947   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11948   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11949   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11950 
11951   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11952   ins_encode %{
11953     __ string_compare($str1$$Register, $str2$$Register,
11954                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11955                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11956   %}
11957   ins_pipe( pipe_slow );
11958 %}
11959 
11960 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11961                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11962 %{
11963   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11964   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11965   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11966 
11967   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11968   ins_encode %{
11969     __ string_compare($str1$$Register, $str2$$Register,
11970                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11971                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11972   %}
11973   ins_pipe( pipe_slow );
11974 %}
11975 
11976 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11977                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11978 %{
11979   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11980   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11981   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11982 
11983   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11984   ins_encode %{
11985     __ string_compare($str1$$Register, $str2$$Register,
11986                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11987                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11988   %}
11989   ins_pipe( pipe_slow );
11990 %}
11991 
11992 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11993                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11994 %{
11995   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11996   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11997   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11998 
11999   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12000   ins_encode %{
12001     __ string_compare($str1$$Register, $str2$$Register,
12002                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12003                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12004   %}
12005   ins_pipe( pipe_slow );
12006 %}
12007 
12008 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12009                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12010 %{
12011   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12012   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12013   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12014 
12015   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12016   ins_encode %{
12017     __ string_compare($str1$$Register, $str2$$Register,
12018                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12019                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12020   %}
12021   ins_pipe( pipe_slow );
12022 %}
12023 
12024 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12025                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12026 %{
12027   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12028   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12029   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12030 
12031   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12032   ins_encode %{
12033     __ string_compare($str1$$Register, $str2$$Register,
12034                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12035                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12036   %}
12037   ins_pipe( pipe_slow );
12038 %}
12039 
12040 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12041                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12042 %{
12043   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12044   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12045   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12046 
12047   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12048   ins_encode %{
12049     __ string_compare($str2$$Register, $str1$$Register,
12050                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12051                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12052   %}
12053   ins_pipe( pipe_slow );
12054 %}
12055 
12056 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12057                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12058 %{
12059   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12060   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12061   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12062 
12063   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12064   ins_encode %{
12065     __ string_compare($str2$$Register, $str1$$Register,
12066                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12067                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12068   %}
12069   ins_pipe( pipe_slow );
12070 %}
12071 
12072 // fast search of substring with known size.
12073 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12074                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12075 %{
12076   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12077   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12078   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12079 
12080   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12081   ins_encode %{
12082     int icnt2 = (int)$int_cnt2$$constant;
12083     if (icnt2 >= 16) {
12084       // IndexOf for constant substrings with size >= 16 elements
12085       // which don't need to be loaded through stack.
12086       __ string_indexofC8($str1$$Register, $str2$$Register,
12087                           $cnt1$$Register, $cnt2$$Register,
12088                           icnt2, $result$$Register,
12089                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12090     } else {
12091       // Small strings are loaded through stack if they cross page boundary.
12092       __ string_indexof($str1$$Register, $str2$$Register,
12093                         $cnt1$$Register, $cnt2$$Register,
12094                         icnt2, $result$$Register,
12095                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12096     }
12097   %}
12098   ins_pipe( pipe_slow );
12099 %}
12100 
12101 // fast search of substring with known size.
12102 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12103                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12104 %{
12105   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12106   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12107   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12108 
12109   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12110   ins_encode %{
12111     int icnt2 = (int)$int_cnt2$$constant;
12112     if (icnt2 >= 8) {
12113       // IndexOf for constant substrings with size >= 8 elements
12114       // which don't need to be loaded through stack.
12115       __ string_indexofC8($str1$$Register, $str2$$Register,
12116                           $cnt1$$Register, $cnt2$$Register,
12117                           icnt2, $result$$Register,
12118                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12119     } else {
12120       // Small strings are loaded through stack if they cross page boundary.
12121       __ string_indexof($str1$$Register, $str2$$Register,
12122                         $cnt1$$Register, $cnt2$$Register,
12123                         icnt2, $result$$Register,
12124                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12125     }
12126   %}
12127   ins_pipe( pipe_slow );
12128 %}
12129 
12130 // fast search of substring with known size.
12131 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12132                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12133 %{
12134   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12135   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12136   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12137 
12138   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12139   ins_encode %{
12140     int icnt2 = (int)$int_cnt2$$constant;
12141     if (icnt2 >= 8) {
12142       // IndexOf for constant substrings with size >= 8 elements
12143       // which don't need to be loaded through stack.
12144       __ string_indexofC8($str1$$Register, $str2$$Register,
12145                           $cnt1$$Register, $cnt2$$Register,
12146                           icnt2, $result$$Register,
12147                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12148     } else {
12149       // Small strings are loaded through stack if they cross page boundary.
12150       __ string_indexof($str1$$Register, $str2$$Register,
12151                         $cnt1$$Register, $cnt2$$Register,
12152                         icnt2, $result$$Register,
12153                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12154     }
12155   %}
12156   ins_pipe( pipe_slow );
12157 %}
12158 
12159 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12160                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12161 %{
12162   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12163   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12164   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12165 
12166   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12167   ins_encode %{
12168     __ string_indexof($str1$$Register, $str2$$Register,
12169                       $cnt1$$Register, $cnt2$$Register,
12170                       (-1), $result$$Register,
12171                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12172   %}
12173   ins_pipe( pipe_slow );
12174 %}
12175 
12176 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12177                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12178 %{
12179   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12180   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12181   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12182 
12183   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12184   ins_encode %{
12185     __ string_indexof($str1$$Register, $str2$$Register,
12186                       $cnt1$$Register, $cnt2$$Register,
12187                       (-1), $result$$Register,
12188                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12189   %}
12190   ins_pipe( pipe_slow );
12191 %}
12192 
12193 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12194                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12195 %{
12196   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12197   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12198   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12199 
12200   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12201   ins_encode %{
12202     __ string_indexof($str1$$Register, $str2$$Register,
12203                       $cnt1$$Register, $cnt2$$Register,
12204                       (-1), $result$$Register,
12205                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12206   %}
12207   ins_pipe( pipe_slow );
12208 %}
12209 
12210 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12211                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12212 %{
12213   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12214   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12215   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12216   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12217   ins_encode %{
12218     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12219                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12220   %}
12221   ins_pipe( pipe_slow );
12222 %}
12223 
12224 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12225                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12226 %{
12227   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12228   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12229   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12230   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12231   ins_encode %{
12232     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12233                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12234   %}
12235   ins_pipe( pipe_slow );
12236 %}
12237 
12238 // fast string equals
12239 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12240                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12241 %{
12242   predicate(!VM_Version::supports_avx512vlbw());
12243   match(Set result (StrEquals (Binary str1 str2) cnt));
12244   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12245 
12246   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12247   ins_encode %{
12248     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12249                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12250                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12251   %}
12252   ins_pipe( pipe_slow );
12253 %}
12254 
12255 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12256                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12257 %{
12258   predicate(VM_Version::supports_avx512vlbw());
12259   match(Set result (StrEquals (Binary str1 str2) cnt));
12260   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12261 
12262   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12263   ins_encode %{
12264     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12265                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12266                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12267   %}
12268   ins_pipe( pipe_slow );
12269 %}
12270 
12271 // fast array equals
12272 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12273                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12274 %{
12275   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12276   match(Set result (AryEq ary1 ary2));
12277   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12278 
12279   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12280   ins_encode %{
12281     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12282                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12283                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12284   %}
12285   ins_pipe( pipe_slow );
12286 %}
12287 
12288 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12289                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12290 %{
12291   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12292   match(Set result (AryEq ary1 ary2));
12293   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12294 
12295   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12296   ins_encode %{
12297     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12298                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12299                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12300   %}
12301   ins_pipe( pipe_slow );
12302 %}
12303 
12304 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12305                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12306 %{
12307   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12308   match(Set result (AryEq ary1 ary2));
12309   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12310 
12311   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12312   ins_encode %{
12313     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12314                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12315                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12316   %}
12317   ins_pipe( pipe_slow );
12318 %}
12319 
12320 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12321                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12322 %{
12323   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12324   match(Set result (AryEq ary1 ary2));
12325   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12326 
12327   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12328   ins_encode %{
12329     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12330                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12331                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12332   %}
12333   ins_pipe( pipe_slow );
12334 %}
12335 
12336 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12337                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12338 %{
12339   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12340   match(Set result (CountPositives ary1 len));
12341   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12342 
12343   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12344   ins_encode %{
12345     __ count_positives($ary1$$Register, $len$$Register,
12346                        $result$$Register, $tmp3$$Register,
12347                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12348   %}
12349   ins_pipe( pipe_slow );
12350 %}
12351 
12352 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12353                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12354 %{
12355   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12356   match(Set result (CountPositives ary1 len));
12357   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12358 
12359   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12360   ins_encode %{
12361     __ count_positives($ary1$$Register, $len$$Register,
12362                        $result$$Register, $tmp3$$Register,
12363                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12364   %}
12365   ins_pipe( pipe_slow );
12366 %}
12367 
12368 // fast char[] to byte[] compression
12369 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12370                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12371   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12372   match(Set result (StrCompressedCopy src (Binary dst len)));
12373   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12374          USE_KILL len, KILL tmp5, KILL cr);
12375 
12376   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12377   ins_encode %{
12378     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12379                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12380                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12381                            knoreg, knoreg);
12382   %}
12383   ins_pipe( pipe_slow );
12384 %}
12385 
12386 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12387                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12388   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12389   match(Set result (StrCompressedCopy src (Binary dst len)));
12390   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12391          USE_KILL len, KILL tmp5, KILL cr);
12392 
12393   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12394   ins_encode %{
12395     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12396                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12397                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12398                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12399   %}
12400   ins_pipe( pipe_slow );
12401 %}
12402 // fast byte[] to char[] inflation
12403 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12404                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12405   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12406   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12407   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12408 
12409   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12410   ins_encode %{
12411     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12412                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12413   %}
12414   ins_pipe( pipe_slow );
12415 %}
12416 
12417 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12418                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12419   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12420   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12421   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12422 
12423   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12424   ins_encode %{
12425     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12426                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12427   %}
12428   ins_pipe( pipe_slow );
12429 %}
12430 
12431 // encode char[] to byte[] in ISO_8859_1
12432 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12433                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12434                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12435   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12436   match(Set result (EncodeISOArray src (Binary dst len)));
12437   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12438 
12439   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12440   ins_encode %{
12441     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12442                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12443                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12444   %}
12445   ins_pipe( pipe_slow );
12446 %}
12447 
12448 // encode char[] to byte[] in ASCII
12449 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12450                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12451                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12452   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12453   match(Set result (EncodeISOArray src (Binary dst len)));
12454   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12455 
12456   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12457   ins_encode %{
12458     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12459                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12460                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12461   %}
12462   ins_pipe( pipe_slow );
12463 %}
12464 
12465 //----------Overflow Math Instructions-----------------------------------------
12466 
12467 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12468 %{
12469   match(Set cr (OverflowAddI op1 op2));
12470   effect(DEF cr, USE_KILL op1, USE op2);
12471 
12472   format %{ "addl    $op1, $op2\t# overflow check int" %}
12473 
12474   ins_encode %{
12475     __ addl($op1$$Register, $op2$$Register);
12476   %}
12477   ins_pipe(ialu_reg_reg);
12478 %}
12479 
12480 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12481 %{
12482   match(Set cr (OverflowAddI op1 op2));
12483   effect(DEF cr, USE_KILL op1, USE op2);
12484 
12485   format %{ "addl    $op1, $op2\t# overflow check int" %}
12486 
12487   ins_encode %{
12488     __ addl($op1$$Register, $op2$$constant);
12489   %}
12490   ins_pipe(ialu_reg_reg);
12491 %}
12492 
12493 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12494 %{
12495   match(Set cr (OverflowAddL op1 op2));
12496   effect(DEF cr, USE_KILL op1, USE op2);
12497 
12498   format %{ "addq    $op1, $op2\t# overflow check long" %}
12499   ins_encode %{
12500     __ addq($op1$$Register, $op2$$Register);
12501   %}
12502   ins_pipe(ialu_reg_reg);
12503 %}
12504 
12505 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12506 %{
12507   match(Set cr (OverflowAddL op1 op2));
12508   effect(DEF cr, USE_KILL op1, USE op2);
12509 
12510   format %{ "addq    $op1, $op2\t# overflow check long" %}
12511   ins_encode %{
12512     __ addq($op1$$Register, $op2$$constant);
12513   %}
12514   ins_pipe(ialu_reg_reg);
12515 %}
12516 
12517 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12518 %{
12519   match(Set cr (OverflowSubI op1 op2));
12520 
12521   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12522   ins_encode %{
12523     __ cmpl($op1$$Register, $op2$$Register);
12524   %}
12525   ins_pipe(ialu_reg_reg);
12526 %}
12527 
12528 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12529 %{
12530   match(Set cr (OverflowSubI op1 op2));
12531 
12532   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12533   ins_encode %{
12534     __ cmpl($op1$$Register, $op2$$constant);
12535   %}
12536   ins_pipe(ialu_reg_reg);
12537 %}
12538 
12539 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12540 %{
12541   match(Set cr (OverflowSubL op1 op2));
12542 
12543   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12544   ins_encode %{
12545     __ cmpq($op1$$Register, $op2$$Register);
12546   %}
12547   ins_pipe(ialu_reg_reg);
12548 %}
12549 
12550 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12551 %{
12552   match(Set cr (OverflowSubL op1 op2));
12553 
12554   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12555   ins_encode %{
12556     __ cmpq($op1$$Register, $op2$$constant);
12557   %}
12558   ins_pipe(ialu_reg_reg);
12559 %}
12560 
12561 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12562 %{
12563   match(Set cr (OverflowSubI zero op2));
12564   effect(DEF cr, USE_KILL op2);
12565 
12566   format %{ "negl    $op2\t# overflow check int" %}
12567   ins_encode %{
12568     __ negl($op2$$Register);
12569   %}
12570   ins_pipe(ialu_reg_reg);
12571 %}
12572 
12573 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12574 %{
12575   match(Set cr (OverflowSubL zero op2));
12576   effect(DEF cr, USE_KILL op2);
12577 
12578   format %{ "negq    $op2\t# overflow check long" %}
12579   ins_encode %{
12580     __ negq($op2$$Register);
12581   %}
12582   ins_pipe(ialu_reg_reg);
12583 %}
12584 
12585 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12586 %{
12587   match(Set cr (OverflowMulI op1 op2));
12588   effect(DEF cr, USE_KILL op1, USE op2);
12589 
12590   format %{ "imull    $op1, $op2\t# overflow check int" %}
12591   ins_encode %{
12592     __ imull($op1$$Register, $op2$$Register);
12593   %}
12594   ins_pipe(ialu_reg_reg_alu0);
12595 %}
12596 
12597 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12598 %{
12599   match(Set cr (OverflowMulI op1 op2));
12600   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12601 
12602   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12603   ins_encode %{
12604     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12605   %}
12606   ins_pipe(ialu_reg_reg_alu0);
12607 %}
12608 
12609 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12610 %{
12611   match(Set cr (OverflowMulL op1 op2));
12612   effect(DEF cr, USE_KILL op1, USE op2);
12613 
12614   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12615   ins_encode %{
12616     __ imulq($op1$$Register, $op2$$Register);
12617   %}
12618   ins_pipe(ialu_reg_reg_alu0);
12619 %}
12620 
12621 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12622 %{
12623   match(Set cr (OverflowMulL op1 op2));
12624   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12625 
12626   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12627   ins_encode %{
12628     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12629   %}
12630   ins_pipe(ialu_reg_reg_alu0);
12631 %}
12632 
12633 
12634 //----------Control Flow Instructions------------------------------------------
12635 // Signed compare Instructions
12636 
12637 // XXX more variants!!
12638 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12639 %{
12640   match(Set cr (CmpI op1 op2));
12641   effect(DEF cr, USE op1, USE op2);
12642 
12643   format %{ "cmpl    $op1, $op2" %}
12644   ins_encode %{
12645     __ cmpl($op1$$Register, $op2$$Register);
12646   %}
12647   ins_pipe(ialu_cr_reg_reg);
12648 %}
12649 
12650 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12651 %{
12652   match(Set cr (CmpI op1 op2));
12653 
12654   format %{ "cmpl    $op1, $op2" %}
12655   ins_encode %{
12656     __ cmpl($op1$$Register, $op2$$constant);
12657   %}
12658   ins_pipe(ialu_cr_reg_imm);
12659 %}
12660 
12661 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12662 %{
12663   match(Set cr (CmpI op1 (LoadI op2)));
12664 
12665   ins_cost(500); // XXX
12666   format %{ "cmpl    $op1, $op2" %}
12667   ins_encode %{
12668     __ cmpl($op1$$Register, $op2$$Address);
12669   %}
12670   ins_pipe(ialu_cr_reg_mem);
12671 %}
12672 
12673 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12674 %{
12675   match(Set cr (CmpI src zero));
12676 
12677   format %{ "testl   $src, $src" %}
12678   ins_encode %{
12679     __ testl($src$$Register, $src$$Register);
12680   %}
12681   ins_pipe(ialu_cr_reg_imm);
12682 %}
12683 
12684 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12685 %{
12686   match(Set cr (CmpI (AndI src con) zero));
12687 
12688   format %{ "testl   $src, $con" %}
12689   ins_encode %{
12690     __ testl($src$$Register, $con$$constant);
12691   %}
12692   ins_pipe(ialu_cr_reg_imm);
12693 %}
12694 
12695 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12696 %{
12697   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12698 
12699   format %{ "testl   $src, $mem" %}
12700   ins_encode %{
12701     __ testl($src$$Register, $mem$$Address);
12702   %}
12703   ins_pipe(ialu_cr_reg_mem);
12704 %}
12705 
12706 // Unsigned compare Instructions; really, same as signed except they
12707 // produce an rFlagsRegU instead of rFlagsReg.
12708 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12709 %{
12710   match(Set cr (CmpU op1 op2));
12711 
12712   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12713   ins_encode %{
12714     __ cmpl($op1$$Register, $op2$$Register);
12715   %}
12716   ins_pipe(ialu_cr_reg_reg);
12717 %}
12718 
12719 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12720 %{
12721   match(Set cr (CmpU op1 op2));
12722 
12723   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12724   ins_encode %{
12725     __ cmpl($op1$$Register, $op2$$constant);
12726   %}
12727   ins_pipe(ialu_cr_reg_imm);
12728 %}
12729 
12730 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12731 %{
12732   match(Set cr (CmpU op1 (LoadI op2)));
12733 
12734   ins_cost(500); // XXX
12735   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12736   ins_encode %{
12737     __ cmpl($op1$$Register, $op2$$Address);
12738   %}
12739   ins_pipe(ialu_cr_reg_mem);
12740 %}
12741 
12742 // // // Cisc-spilled version of cmpU_rReg
12743 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12744 // //%{
12745 // //  match(Set cr (CmpU (LoadI op1) op2));
12746 // //
12747 // //  format %{ "CMPu   $op1,$op2" %}
12748 // //  ins_cost(500);
12749 // //  opcode(0x39);  /* Opcode 39 /r */
12750 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12751 // //%}
12752 
12753 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12754 %{
12755   match(Set cr (CmpU src zero));
12756 
12757   format %{ "testl   $src, $src\t# unsigned" %}
12758   ins_encode %{
12759     __ testl($src$$Register, $src$$Register);
12760   %}
12761   ins_pipe(ialu_cr_reg_imm);
12762 %}
12763 
12764 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12765 %{
12766   match(Set cr (CmpP op1 op2));
12767 
12768   format %{ "cmpq    $op1, $op2\t# ptr" %}
12769   ins_encode %{
12770     __ cmpq($op1$$Register, $op2$$Register);
12771   %}
12772   ins_pipe(ialu_cr_reg_reg);
12773 %}
12774 
12775 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12776 %{
12777   match(Set cr (CmpP op1 (LoadP op2)));
12778   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12779 
12780   ins_cost(500); // XXX
12781   format %{ "cmpq    $op1, $op2\t# ptr" %}
12782   ins_encode %{
12783     __ cmpq($op1$$Register, $op2$$Address);
12784   %}
12785   ins_pipe(ialu_cr_reg_mem);
12786 %}
12787 
12788 // // // Cisc-spilled version of cmpP_rReg
12789 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12790 // //%{
12791 // //  match(Set cr (CmpP (LoadP op1) op2));
12792 // //
12793 // //  format %{ "CMPu   $op1,$op2" %}
12794 // //  ins_cost(500);
12795 // //  opcode(0x39);  /* Opcode 39 /r */
12796 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12797 // //%}
12798 
12799 // XXX this is generalized by compP_rReg_mem???
12800 // Compare raw pointer (used in out-of-heap check).
12801 // Only works because non-oop pointers must be raw pointers
12802 // and raw pointers have no anti-dependencies.
12803 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12804 %{
12805   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12806             n->in(2)->as_Load()->barrier_data() == 0);
12807   match(Set cr (CmpP op1 (LoadP op2)));
12808 
12809   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12810   ins_encode %{
12811     __ cmpq($op1$$Register, $op2$$Address);
12812   %}
12813   ins_pipe(ialu_cr_reg_mem);
12814 %}
12815 
12816 // This will generate a signed flags result. This should be OK since
12817 // any compare to a zero should be eq/neq.
12818 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12819 %{
12820   match(Set cr (CmpP src zero));
12821 
12822   format %{ "testq   $src, $src\t# ptr" %}
12823   ins_encode %{
12824     __ testq($src$$Register, $src$$Register);
12825   %}
12826   ins_pipe(ialu_cr_reg_imm);
12827 %}
12828 
12829 // This will generate a signed flags result. This should be OK since
12830 // any compare to a zero should be eq/neq.
12831 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12832 %{
12833   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12834             n->in(1)->as_Load()->barrier_data() == 0);
12835   match(Set cr (CmpP (LoadP op) zero));
12836 
12837   ins_cost(500); // XXX
12838   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12839   ins_encode %{
12840     __ testq($op$$Address, 0xFFFFFFFF);
12841   %}
12842   ins_pipe(ialu_cr_reg_imm);
12843 %}
12844 
12845 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12846 %{
12847   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12848             n->in(1)->as_Load()->barrier_data() == 0);
12849   match(Set cr (CmpP (LoadP mem) zero));
12850 
12851   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12852   ins_encode %{
12853     __ cmpq(r12, $mem$$Address);
12854   %}
12855   ins_pipe(ialu_cr_reg_mem);
12856 %}
12857 
12858 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12859 %{
12860   match(Set cr (CmpN op1 op2));
12861 
12862   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12863   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12864   ins_pipe(ialu_cr_reg_reg);
12865 %}
12866 
12867 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12868 %{
12869   match(Set cr (CmpN src (LoadN mem)));
12870 
12871   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12872   ins_encode %{
12873     __ cmpl($src$$Register, $mem$$Address);
12874   %}
12875   ins_pipe(ialu_cr_reg_mem);
12876 %}
12877 
12878 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12879   match(Set cr (CmpN op1 op2));
12880 
12881   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12882   ins_encode %{
12883     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12884   %}
12885   ins_pipe(ialu_cr_reg_imm);
12886 %}
12887 
12888 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12889 %{
12890   match(Set cr (CmpN src (LoadN mem)));
12891 
12892   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12893   ins_encode %{
12894     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12895   %}
12896   ins_pipe(ialu_cr_reg_mem);
12897 %}
12898 
12899 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12900   match(Set cr (CmpN op1 op2));
12901 
12902   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12903   ins_encode %{
12904     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12905   %}
12906   ins_pipe(ialu_cr_reg_imm);
12907 %}
12908 
12909 // Disabled because the compressed Klass* in header cannot be safely
12910 // accessed. TODO: Re-enable it as soon as synchronization does not
12911 // overload the upper header bits anymore.
12912 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12913 %{
12914   predicate(false);
12915   match(Set cr (CmpN src (LoadNKlass mem)));
12916 
12917   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12918   ins_encode %{
12919     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12920   %}
12921   ins_pipe(ialu_cr_reg_mem);
12922 %}
12923 
12924 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12925   match(Set cr (CmpN src zero));
12926 
12927   format %{ "testl   $src, $src\t# compressed ptr" %}
12928   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12929   ins_pipe(ialu_cr_reg_imm);
12930 %}
12931 
12932 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12933 %{
12934   predicate(CompressedOops::base() != NULL);
12935   match(Set cr (CmpN (LoadN mem) zero));
12936 
12937   ins_cost(500); // XXX
12938   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12939   ins_encode %{
12940     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12941   %}
12942   ins_pipe(ialu_cr_reg_mem);
12943 %}
12944 
12945 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12946 %{
12947   predicate(CompressedOops::base() == NULL);
12948   match(Set cr (CmpN (LoadN mem) zero));
12949 
12950   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12951   ins_encode %{
12952     __ cmpl(r12, $mem$$Address);
12953   %}
12954   ins_pipe(ialu_cr_reg_mem);
12955 %}
12956 
12957 // Yanked all unsigned pointer compare operations.
12958 // Pointer compares are done with CmpP which is already unsigned.
12959 
12960 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12961 %{
12962   match(Set cr (CmpL op1 op2));
12963 
12964   format %{ "cmpq    $op1, $op2" %}
12965   ins_encode %{
12966     __ cmpq($op1$$Register, $op2$$Register);
12967   %}
12968   ins_pipe(ialu_cr_reg_reg);
12969 %}
12970 
12971 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12972 %{
12973   match(Set cr (CmpL op1 op2));
12974 
12975   format %{ "cmpq    $op1, $op2" %}
12976   ins_encode %{
12977     __ cmpq($op1$$Register, $op2$$constant);
12978   %}
12979   ins_pipe(ialu_cr_reg_imm);
12980 %}
12981 
12982 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12983 %{
12984   match(Set cr (CmpL op1 (LoadL op2)));
12985 
12986   format %{ "cmpq    $op1, $op2" %}
12987   ins_encode %{
12988     __ cmpq($op1$$Register, $op2$$Address);
12989   %}
12990   ins_pipe(ialu_cr_reg_mem);
12991 %}
12992 
12993 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12994 %{
12995   match(Set cr (CmpL src zero));
12996 
12997   format %{ "testq   $src, $src" %}
12998   ins_encode %{
12999     __ testq($src$$Register, $src$$Register);
13000   %}
13001   ins_pipe(ialu_cr_reg_imm);
13002 %}
13003 
13004 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13005 %{
13006   match(Set cr (CmpL (AndL src con) zero));
13007 
13008   format %{ "testq   $src, $con\t# long" %}
13009   ins_encode %{
13010     __ testq($src$$Register, $con$$constant);
13011   %}
13012   ins_pipe(ialu_cr_reg_imm);
13013 %}
13014 
13015 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13016 %{
13017   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13018 
13019   format %{ "testq   $src, $mem" %}
13020   ins_encode %{
13021     __ testq($src$$Register, $mem$$Address);
13022   %}
13023   ins_pipe(ialu_cr_reg_mem);
13024 %}
13025 
13026 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13027 %{
13028   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13029 
13030   format %{ "testq   $src, $mem" %}
13031   ins_encode %{
13032     __ testq($src$$Register, $mem$$Address);
13033   %}
13034   ins_pipe(ialu_cr_reg_mem);
13035 %}
13036 
13037 // Manifest a CmpU result in an integer register.  Very painful.
13038 // This is the test to avoid.
13039 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13040 %{
13041   match(Set dst (CmpU3 src1 src2));
13042   effect(KILL flags);
13043 
13044   ins_cost(275); // XXX
13045   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13046             "movl    $dst, -1\n\t"
13047             "jb,u    done\n\t"
13048             "setne   $dst\n\t"
13049             "movzbl  $dst, $dst\n\t"
13050     "done:" %}
13051   ins_encode %{
13052     Label done;
13053     __ cmpl($src1$$Register, $src2$$Register);
13054     __ movl($dst$$Register, -1);
13055     __ jccb(Assembler::below, done);
13056     __ setne($dst$$Register);
13057     __ movzbl($dst$$Register, $dst$$Register);
13058     __ bind(done);
13059   %}
13060   ins_pipe(pipe_slow);
13061 %}
13062 
13063 // Manifest a CmpL result in an integer register.  Very painful.
13064 // This is the test to avoid.
13065 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13066 %{
13067   match(Set dst (CmpL3 src1 src2));
13068   effect(KILL flags);
13069 
13070   ins_cost(275); // XXX
13071   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13072             "movl    $dst, -1\n\t"
13073             "jl,s    done\n\t"
13074             "setne   $dst\n\t"
13075             "movzbl  $dst, $dst\n\t"
13076     "done:" %}
13077   ins_encode %{
13078     Label done;
13079     __ cmpq($src1$$Register, $src2$$Register);
13080     __ movl($dst$$Register, -1);
13081     __ jccb(Assembler::less, done);
13082     __ setne($dst$$Register);
13083     __ movzbl($dst$$Register, $dst$$Register);
13084     __ bind(done);
13085   %}
13086   ins_pipe(pipe_slow);
13087 %}
13088 
13089 // Manifest a CmpUL result in an integer register.  Very painful.
13090 // This is the test to avoid.
13091 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13092 %{
13093   match(Set dst (CmpUL3 src1 src2));
13094   effect(KILL flags);
13095 
13096   ins_cost(275); // XXX
13097   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13098             "movl    $dst, -1\n\t"
13099             "jb,u    done\n\t"
13100             "setne   $dst\n\t"
13101             "movzbl  $dst, $dst\n\t"
13102     "done:" %}
13103   ins_encode %{
13104     Label done;
13105     __ cmpq($src1$$Register, $src2$$Register);
13106     __ movl($dst$$Register, -1);
13107     __ jccb(Assembler::below, done);
13108     __ setne($dst$$Register);
13109     __ movzbl($dst$$Register, $dst$$Register);
13110     __ bind(done);
13111   %}
13112   ins_pipe(pipe_slow);
13113 %}
13114 
13115 // Unsigned long compare Instructions; really, same as signed long except they
13116 // produce an rFlagsRegU instead of rFlagsReg.
13117 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13118 %{
13119   match(Set cr (CmpUL op1 op2));
13120 
13121   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13122   ins_encode %{
13123     __ cmpq($op1$$Register, $op2$$Register);
13124   %}
13125   ins_pipe(ialu_cr_reg_reg);
13126 %}
13127 
13128 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13129 %{
13130   match(Set cr (CmpUL op1 op2));
13131 
13132   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13133   ins_encode %{
13134     __ cmpq($op1$$Register, $op2$$constant);
13135   %}
13136   ins_pipe(ialu_cr_reg_imm);
13137 %}
13138 
13139 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13140 %{
13141   match(Set cr (CmpUL op1 (LoadL op2)));
13142 
13143   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13144   ins_encode %{
13145     __ cmpq($op1$$Register, $op2$$Address);
13146   %}
13147   ins_pipe(ialu_cr_reg_mem);
13148 %}
13149 
13150 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13151 %{
13152   match(Set cr (CmpUL src zero));
13153 
13154   format %{ "testq   $src, $src\t# unsigned" %}
13155   ins_encode %{
13156     __ testq($src$$Register, $src$$Register);
13157   %}
13158   ins_pipe(ialu_cr_reg_imm);
13159 %}
13160 
13161 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13162 %{
13163   match(Set cr (CmpI (LoadB mem) imm));
13164 
13165   ins_cost(125);
13166   format %{ "cmpb    $mem, $imm" %}
13167   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13168   ins_pipe(ialu_cr_reg_mem);
13169 %}
13170 
13171 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13172 %{
13173   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13174 
13175   ins_cost(125);
13176   format %{ "testb   $mem, $imm\t# ubyte" %}
13177   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13178   ins_pipe(ialu_cr_reg_mem);
13179 %}
13180 
13181 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13182 %{
13183   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13184 
13185   ins_cost(125);
13186   format %{ "testb   $mem, $imm\t# byte" %}
13187   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13188   ins_pipe(ialu_cr_reg_mem);
13189 %}
13190 
13191 //----------Max and Min--------------------------------------------------------
13192 // Min Instructions
13193 
13194 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13195 %{
13196   effect(USE_DEF dst, USE src, USE cr);
13197 
13198   format %{ "cmovlgt $dst, $src\t# min" %}
13199   ins_encode %{
13200     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13201   %}
13202   ins_pipe(pipe_cmov_reg);
13203 %}
13204 
13205 
13206 instruct minI_rReg(rRegI dst, rRegI src)
13207 %{
13208   match(Set dst (MinI dst src));
13209 
13210   ins_cost(200);
13211   expand %{
13212     rFlagsReg cr;
13213     compI_rReg(cr, dst, src);
13214     cmovI_reg_g(dst, src, cr);
13215   %}
13216 %}
13217 
13218 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13219 %{
13220   effect(USE_DEF dst, USE src, USE cr);
13221 
13222   format %{ "cmovllt $dst, $src\t# max" %}
13223   ins_encode %{
13224     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13225   %}
13226   ins_pipe(pipe_cmov_reg);
13227 %}
13228 
13229 
13230 instruct maxI_rReg(rRegI dst, rRegI src)
13231 %{
13232   match(Set dst (MaxI dst src));
13233 
13234   ins_cost(200);
13235   expand %{
13236     rFlagsReg cr;
13237     compI_rReg(cr, dst, src);
13238     cmovI_reg_l(dst, src, cr);
13239   %}
13240 %}
13241 
13242 // ============================================================================
13243 // Branch Instructions
13244 
13245 // Jump Direct - Label defines a relative address from JMP+1
13246 instruct jmpDir(label labl)
13247 %{
13248   match(Goto);
13249   effect(USE labl);
13250 
13251   ins_cost(300);
13252   format %{ "jmp     $labl" %}
13253   size(5);
13254   ins_encode %{
13255     Label* L = $labl$$label;
13256     __ jmp(*L, false); // Always long jump
13257   %}
13258   ins_pipe(pipe_jmp);
13259 %}
13260 
13261 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13262 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13263 %{
13264   match(If cop cr);
13265   effect(USE labl);
13266 
13267   ins_cost(300);
13268   format %{ "j$cop     $labl" %}
13269   size(6);
13270   ins_encode %{
13271     Label* L = $labl$$label;
13272     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13273   %}
13274   ins_pipe(pipe_jcc);
13275 %}
13276 
13277 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13278 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13279 %{
13280   match(CountedLoopEnd cop cr);
13281   effect(USE labl);
13282 
13283   ins_cost(300);
13284   format %{ "j$cop     $labl\t# loop end" %}
13285   size(6);
13286   ins_encode %{
13287     Label* L = $labl$$label;
13288     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13289   %}
13290   ins_pipe(pipe_jcc);
13291 %}
13292 
13293 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13294 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13295   match(CountedLoopEnd cop cmp);
13296   effect(USE labl);
13297 
13298   ins_cost(300);
13299   format %{ "j$cop,u   $labl\t# loop end" %}
13300   size(6);
13301   ins_encode %{
13302     Label* L = $labl$$label;
13303     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13304   %}
13305   ins_pipe(pipe_jcc);
13306 %}
13307 
13308 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13309   match(CountedLoopEnd cop cmp);
13310   effect(USE labl);
13311 
13312   ins_cost(200);
13313   format %{ "j$cop,u   $labl\t# loop end" %}
13314   size(6);
13315   ins_encode %{
13316     Label* L = $labl$$label;
13317     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13318   %}
13319   ins_pipe(pipe_jcc);
13320 %}
13321 
13322 // Jump Direct Conditional - using unsigned comparison
13323 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13324   match(If cop cmp);
13325   effect(USE labl);
13326 
13327   ins_cost(300);
13328   format %{ "j$cop,u   $labl" %}
13329   size(6);
13330   ins_encode %{
13331     Label* L = $labl$$label;
13332     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13333   %}
13334   ins_pipe(pipe_jcc);
13335 %}
13336 
13337 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13338   match(If cop cmp);
13339   effect(USE labl);
13340 
13341   ins_cost(200);
13342   format %{ "j$cop,u   $labl" %}
13343   size(6);
13344   ins_encode %{
13345     Label* L = $labl$$label;
13346     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13347   %}
13348   ins_pipe(pipe_jcc);
13349 %}
13350 
13351 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13352   match(If cop cmp);
13353   effect(USE labl);
13354 
13355   ins_cost(200);
13356   format %{ $$template
13357     if ($cop$$cmpcode == Assembler::notEqual) {
13358       $$emit$$"jp,u    $labl\n\t"
13359       $$emit$$"j$cop,u   $labl"
13360     } else {
13361       $$emit$$"jp,u    done\n\t"
13362       $$emit$$"j$cop,u   $labl\n\t"
13363       $$emit$$"done:"
13364     }
13365   %}
13366   ins_encode %{
13367     Label* l = $labl$$label;
13368     if ($cop$$cmpcode == Assembler::notEqual) {
13369       __ jcc(Assembler::parity, *l, false);
13370       __ jcc(Assembler::notEqual, *l, false);
13371     } else if ($cop$$cmpcode == Assembler::equal) {
13372       Label done;
13373       __ jccb(Assembler::parity, done);
13374       __ jcc(Assembler::equal, *l, false);
13375       __ bind(done);
13376     } else {
13377        ShouldNotReachHere();
13378     }
13379   %}
13380   ins_pipe(pipe_jcc);
13381 %}
13382 
13383 // ============================================================================
13384 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13385 // superklass array for an instance of the superklass.  Set a hidden
13386 // internal cache on a hit (cache is checked with exposed code in
13387 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13388 // encoding ALSO sets flags.
13389 
13390 instruct partialSubtypeCheck(rdi_RegP result,
13391                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13392                              rFlagsReg cr)
13393 %{
13394   match(Set result (PartialSubtypeCheck sub super));
13395   effect(KILL rcx, KILL cr);
13396 
13397   ins_cost(1100);  // slightly larger than the next version
13398   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13399             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13400             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13401             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13402             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13403             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13404             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13405     "miss:\t" %}
13406 
13407   opcode(0x1); // Force a XOR of RDI
13408   ins_encode(enc_PartialSubtypeCheck());
13409   ins_pipe(pipe_slow);
13410 %}
13411 
13412 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13413                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13414                                      immP0 zero,
13415                                      rdi_RegP result)
13416 %{
13417   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13418   effect(KILL rcx, KILL result);
13419 
13420   ins_cost(1000);
13421   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13422             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13423             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13424             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13425             "jne,s   miss\t\t# Missed: flags nz\n\t"
13426             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13427     "miss:\t" %}
13428 
13429   opcode(0x0); // No need to XOR RDI
13430   ins_encode(enc_PartialSubtypeCheck());
13431   ins_pipe(pipe_slow);
13432 %}
13433 
13434 // ============================================================================
13435 // Branch Instructions -- short offset versions
13436 //
13437 // These instructions are used to replace jumps of a long offset (the default
13438 // match) with jumps of a shorter offset.  These instructions are all tagged
13439 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13440 // match rules in general matching.  Instead, the ADLC generates a conversion
13441 // method in the MachNode which can be used to do in-place replacement of the
13442 // long variant with the shorter variant.  The compiler will determine if a
13443 // branch can be taken by the is_short_branch_offset() predicate in the machine
13444 // specific code section of the file.
13445 
13446 // Jump Direct - Label defines a relative address from JMP+1
13447 instruct jmpDir_short(label labl) %{
13448   match(Goto);
13449   effect(USE labl);
13450 
13451   ins_cost(300);
13452   format %{ "jmp,s   $labl" %}
13453   size(2);
13454   ins_encode %{
13455     Label* L = $labl$$label;
13456     __ jmpb(*L);
13457   %}
13458   ins_pipe(pipe_jmp);
13459   ins_short_branch(1);
13460 %}
13461 
13462 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13463 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13464   match(If cop cr);
13465   effect(USE labl);
13466 
13467   ins_cost(300);
13468   format %{ "j$cop,s   $labl" %}
13469   size(2);
13470   ins_encode %{
13471     Label* L = $labl$$label;
13472     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13473   %}
13474   ins_pipe(pipe_jcc);
13475   ins_short_branch(1);
13476 %}
13477 
13478 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13479 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13480   match(CountedLoopEnd cop cr);
13481   effect(USE labl);
13482 
13483   ins_cost(300);
13484   format %{ "j$cop,s   $labl\t# loop end" %}
13485   size(2);
13486   ins_encode %{
13487     Label* L = $labl$$label;
13488     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13489   %}
13490   ins_pipe(pipe_jcc);
13491   ins_short_branch(1);
13492 %}
13493 
13494 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13495 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13496   match(CountedLoopEnd cop cmp);
13497   effect(USE labl);
13498 
13499   ins_cost(300);
13500   format %{ "j$cop,us  $labl\t# loop end" %}
13501   size(2);
13502   ins_encode %{
13503     Label* L = $labl$$label;
13504     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13505   %}
13506   ins_pipe(pipe_jcc);
13507   ins_short_branch(1);
13508 %}
13509 
13510 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13511   match(CountedLoopEnd cop cmp);
13512   effect(USE labl);
13513 
13514   ins_cost(300);
13515   format %{ "j$cop,us  $labl\t# loop end" %}
13516   size(2);
13517   ins_encode %{
13518     Label* L = $labl$$label;
13519     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13520   %}
13521   ins_pipe(pipe_jcc);
13522   ins_short_branch(1);
13523 %}
13524 
13525 // Jump Direct Conditional - using unsigned comparison
13526 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13527   match(If cop cmp);
13528   effect(USE labl);
13529 
13530   ins_cost(300);
13531   format %{ "j$cop,us  $labl" %}
13532   size(2);
13533   ins_encode %{
13534     Label* L = $labl$$label;
13535     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13536   %}
13537   ins_pipe(pipe_jcc);
13538   ins_short_branch(1);
13539 %}
13540 
13541 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13542   match(If cop cmp);
13543   effect(USE labl);
13544 
13545   ins_cost(300);
13546   format %{ "j$cop,us  $labl" %}
13547   size(2);
13548   ins_encode %{
13549     Label* L = $labl$$label;
13550     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13551   %}
13552   ins_pipe(pipe_jcc);
13553   ins_short_branch(1);
13554 %}
13555 
13556 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13557   match(If cop cmp);
13558   effect(USE labl);
13559 
13560   ins_cost(300);
13561   format %{ $$template
13562     if ($cop$$cmpcode == Assembler::notEqual) {
13563       $$emit$$"jp,u,s  $labl\n\t"
13564       $$emit$$"j$cop,u,s  $labl"
13565     } else {
13566       $$emit$$"jp,u,s  done\n\t"
13567       $$emit$$"j$cop,u,s  $labl\n\t"
13568       $$emit$$"done:"
13569     }
13570   %}
13571   size(4);
13572   ins_encode %{
13573     Label* l = $labl$$label;
13574     if ($cop$$cmpcode == Assembler::notEqual) {
13575       __ jccb(Assembler::parity, *l);
13576       __ jccb(Assembler::notEqual, *l);
13577     } else if ($cop$$cmpcode == Assembler::equal) {
13578       Label done;
13579       __ jccb(Assembler::parity, done);
13580       __ jccb(Assembler::equal, *l);
13581       __ bind(done);
13582     } else {
13583        ShouldNotReachHere();
13584     }
13585   %}
13586   ins_pipe(pipe_jcc);
13587   ins_short_branch(1);
13588 %}
13589 
13590 // ============================================================================
13591 // inlined locking and unlocking
13592 
13593 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13594   predicate(Compile::current()->use_rtm());
13595   match(Set cr (FastLock object box));
13596   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13597   ins_cost(300);
13598   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13599   ins_encode %{
13600     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13601                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13602                  _rtm_counters, _stack_rtm_counters,
13603                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13604                  true, ra_->C->profile_rtm());
13605   %}
13606   ins_pipe(pipe_slow);
13607 %}
13608 
13609 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13610   predicate(!Compile::current()->use_rtm());
13611   match(Set cr (FastLock object box));
13612   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13613   ins_cost(300);
13614   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13615   ins_encode %{
13616     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13617                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13618   %}
13619   ins_pipe(pipe_slow);
13620 %}
13621 
13622 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13623   match(Set cr (FastUnlock object box));
13624   effect(TEMP tmp, USE_KILL box);
13625   ins_cost(300);
13626   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13627   ins_encode %{
13628     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13629   %}
13630   ins_pipe(pipe_slow);
13631 %}
13632 
13633 
13634 // ============================================================================
13635 // Safepoint Instructions
13636 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13637 %{
13638   match(SafePoint poll);
13639   effect(KILL cr, USE poll);
13640 
13641   format %{ "testl   rax, [$poll]\t"
13642             "# Safepoint: poll for GC" %}
13643   ins_cost(125);
13644   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13645   ins_encode %{
13646     __ relocate(relocInfo::poll_type);
13647     address pre_pc = __ pc();
13648     __ testl(rax, Address($poll$$Register, 0));
13649     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13650   %}
13651   ins_pipe(ialu_reg_mem);
13652 %}
13653 
13654 instruct mask_all_evexL(kReg dst, rRegL src) %{
13655   match(Set dst (MaskAll src));
13656   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13657   ins_encode %{
13658     int mask_len = Matcher::vector_length(this);
13659     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13660   %}
13661   ins_pipe( pipe_slow );
13662 %}
13663 
13664 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13665   predicate(Matcher::vector_length(n) > 32);
13666   match(Set dst (MaskAll src));
13667   effect(TEMP tmp);
13668   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13669   ins_encode %{
13670     int mask_len = Matcher::vector_length(this);
13671     __ movslq($tmp$$Register, $src$$Register);
13672     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13673   %}
13674   ins_pipe( pipe_slow );
13675 %}
13676 
13677 // ============================================================================
13678 // Procedure Call/Return Instructions
13679 // Call Java Static Instruction
13680 // Note: If this code changes, the corresponding ret_addr_offset() and
13681 //       compute_padding() functions will have to be adjusted.
13682 instruct CallStaticJavaDirect(method meth) %{
13683   match(CallStaticJava);
13684   effect(USE meth);
13685 
13686   ins_cost(300);
13687   format %{ "call,static " %}
13688   opcode(0xE8); /* E8 cd */
13689   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13690   ins_pipe(pipe_slow);
13691   ins_alignment(4);
13692 %}
13693 
13694 // Call Java Dynamic Instruction
13695 // Note: If this code changes, the corresponding ret_addr_offset() and
13696 //       compute_padding() functions will have to be adjusted.
13697 instruct CallDynamicJavaDirect(method meth)
13698 %{
13699   match(CallDynamicJava);
13700   effect(USE meth);
13701 
13702   ins_cost(300);
13703   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13704             "call,dynamic " %}
13705   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13706   ins_pipe(pipe_slow);
13707   ins_alignment(4);
13708 %}
13709 
13710 // Call Runtime Instruction
13711 instruct CallRuntimeDirect(method meth)
13712 %{
13713   match(CallRuntime);
13714   effect(USE meth);
13715 
13716   ins_cost(300);
13717   format %{ "call,runtime " %}
13718   ins_encode(clear_avx, Java_To_Runtime(meth));
13719   ins_pipe(pipe_slow);
13720 %}
13721 
13722 // Call runtime without safepoint
13723 instruct CallLeafDirect(method meth)
13724 %{
13725   match(CallLeaf);
13726   effect(USE meth);
13727 
13728   ins_cost(300);
13729   format %{ "call_leaf,runtime " %}
13730   ins_encode(clear_avx, Java_To_Runtime(meth));
13731   ins_pipe(pipe_slow);
13732 %}
13733 
13734 // Call runtime without safepoint and with vector arguments
13735 instruct CallLeafDirectVector(method meth)
13736 %{
13737   match(CallLeafVector);
13738   effect(USE meth);
13739 
13740   ins_cost(300);
13741   format %{ "call_leaf,vector " %}
13742   ins_encode(Java_To_Runtime(meth));
13743   ins_pipe(pipe_slow);
13744 %}
13745 
13746 // Call runtime without safepoint
13747 instruct CallLeafNoFPDirect(method meth)
13748 %{
13749   match(CallLeafNoFP);
13750   effect(USE meth);
13751 
13752   ins_cost(300);
13753   format %{ "call_leaf_nofp,runtime " %}
13754   ins_encode(clear_avx, Java_To_Runtime(meth));
13755   ins_pipe(pipe_slow);
13756 %}
13757 
13758 // Return Instruction
13759 // Remove the return address & jump to it.
13760 // Notice: We always emit a nop after a ret to make sure there is room
13761 // for safepoint patching
13762 instruct Ret()
13763 %{
13764   match(Return);
13765 
13766   format %{ "ret" %}
13767   ins_encode %{
13768     __ ret(0);
13769   %}
13770   ins_pipe(pipe_jmp);
13771 %}
13772 
13773 // Tail Call; Jump from runtime stub to Java code.
13774 // Also known as an 'interprocedural jump'.
13775 // Target of jump will eventually return to caller.
13776 // TailJump below removes the return address.
13777 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13778 %{
13779   match(TailCall jump_target method_ptr);
13780 
13781   ins_cost(300);
13782   format %{ "jmp     $jump_target\t# rbx holds method" %}
13783   ins_encode %{
13784     __ jmp($jump_target$$Register);
13785   %}
13786   ins_pipe(pipe_jmp);
13787 %}
13788 
13789 // Tail Jump; remove the return address; jump to target.
13790 // TailCall above leaves the return address around.
13791 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13792 %{
13793   match(TailJump jump_target ex_oop);
13794 
13795   ins_cost(300);
13796   format %{ "popq    rdx\t# pop return address\n\t"
13797             "jmp     $jump_target" %}
13798   ins_encode %{
13799     __ popq(as_Register(RDX_enc));
13800     __ jmp($jump_target$$Register);
13801   %}
13802   ins_pipe(pipe_jmp);
13803 %}
13804 
13805 // Create exception oop: created by stack-crawling runtime code.
13806 // Created exception is now available to this handler, and is setup
13807 // just prior to jumping to this handler.  No code emitted.
13808 instruct CreateException(rax_RegP ex_oop)
13809 %{
13810   match(Set ex_oop (CreateEx));
13811 
13812   size(0);
13813   // use the following format syntax
13814   format %{ "# exception oop is in rax; no code emitted" %}
13815   ins_encode();
13816   ins_pipe(empty);
13817 %}
13818 
13819 // Rethrow exception:
13820 // The exception oop will come in the first argument position.
13821 // Then JUMP (not call) to the rethrow stub code.
13822 instruct RethrowException()
13823 %{
13824   match(Rethrow);
13825 
13826   // use the following format syntax
13827   format %{ "jmp     rethrow_stub" %}
13828   ins_encode(enc_rethrow);
13829   ins_pipe(pipe_jmp);
13830 %}
13831 
13832 // ============================================================================
13833 // This name is KNOWN by the ADLC and cannot be changed.
13834 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13835 // for this guy.
13836 instruct tlsLoadP(r15_RegP dst) %{
13837   match(Set dst (ThreadLocal));
13838   effect(DEF dst);
13839 
13840   size(0);
13841   format %{ "# TLS is in R15" %}
13842   ins_encode( /*empty encoding*/ );
13843   ins_pipe(ialu_reg_reg);
13844 %}
13845 
13846 
13847 //----------PEEPHOLE RULES-----------------------------------------------------
13848 // These must follow all instruction definitions as they use the names
13849 // defined in the instructions definitions.
13850 //
13851 // peepmatch ( root_instr_name [preceding_instruction]* );
13852 //
13853 // peepconstraint %{
13854 // (instruction_number.operand_name relational_op instruction_number.operand_name
13855 //  [, ...] );
13856 // // instruction numbers are zero-based using left to right order in peepmatch
13857 //
13858 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13859 // // provide an instruction_number.operand_name for each operand that appears
13860 // // in the replacement instruction's match rule
13861 //
13862 // ---------VM FLAGS---------------------------------------------------------
13863 //
13864 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13865 //
13866 // Each peephole rule is given an identifying number starting with zero and
13867 // increasing by one in the order seen by the parser.  An individual peephole
13868 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13869 // on the command-line.
13870 //
13871 // ---------CURRENT LIMITATIONS----------------------------------------------
13872 //
13873 // Only match adjacent instructions in same basic block
13874 // Only equality constraints
13875 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13876 // Only one replacement instruction
13877 //
13878 // ---------EXAMPLE----------------------------------------------------------
13879 //
13880 // // pertinent parts of existing instructions in architecture description
13881 // instruct movI(rRegI dst, rRegI src)
13882 // %{
13883 //   match(Set dst (CopyI src));
13884 // %}
13885 //
13886 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13887 // %{
13888 //   match(Set dst (AddI dst src));
13889 //   effect(KILL cr);
13890 // %}
13891 //
13892 // // Change (inc mov) to lea
13893 // peephole %{
13894 //   // increment preceded by register-register move
13895 //   peepmatch ( incI_rReg movI );
13896 //   // require that the destination register of the increment
13897 //   // match the destination register of the move
13898 //   peepconstraint ( 0.dst == 1.dst );
13899 //   // construct a replacement instruction that sets
13900 //   // the destination to ( move's source register + one )
13901 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13902 // %}
13903 //
13904 
13905 // Implementation no longer uses movX instructions since
13906 // machine-independent system no longer uses CopyX nodes.
13907 //
13908 // peephole
13909 // %{
13910 //   peepmatch (incI_rReg movI);
13911 //   peepconstraint (0.dst == 1.dst);
13912 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13913 // %}
13914 
13915 // peephole
13916 // %{
13917 //   peepmatch (decI_rReg movI);
13918 //   peepconstraint (0.dst == 1.dst);
13919 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13920 // %}
13921 
13922 // peephole
13923 // %{
13924 //   peepmatch (addI_rReg_imm movI);
13925 //   peepconstraint (0.dst == 1.dst);
13926 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13927 // %}
13928 
13929 // peephole
13930 // %{
13931 //   peepmatch (incL_rReg movL);
13932 //   peepconstraint (0.dst == 1.dst);
13933 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13934 // %}
13935 
13936 // peephole
13937 // %{
13938 //   peepmatch (decL_rReg movL);
13939 //   peepconstraint (0.dst == 1.dst);
13940 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13941 // %}
13942 
13943 // peephole
13944 // %{
13945 //   peepmatch (addL_rReg_imm movL);
13946 //   peepconstraint (0.dst == 1.dst);
13947 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13948 // %}
13949 
13950 // peephole
13951 // %{
13952 //   peepmatch (addP_rReg_imm movP);
13953 //   peepconstraint (0.dst == 1.dst);
13954 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13955 // %}
13956 
13957 // // Change load of spilled value to only a spill
13958 // instruct storeI(memory mem, rRegI src)
13959 // %{
13960 //   match(Set mem (StoreI mem src));
13961 // %}
13962 //
13963 // instruct loadI(rRegI dst, memory mem)
13964 // %{
13965 //   match(Set dst (LoadI mem));
13966 // %}
13967 //
13968 
13969 peephole
13970 %{
13971   peepmatch (loadI storeI);
13972   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13973   peepreplace (storeI(1.mem 1.mem 1.src));
13974 %}
13975 
13976 peephole
13977 %{
13978   peepmatch (loadL storeL);
13979   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13980   peepreplace (storeL(1.mem 1.mem 1.src));
13981 %}
13982 
13983 //----------SMARTSPILL RULES---------------------------------------------------
13984 // These must follow all instruction definitions as they use the names
13985 // defined in the instructions definitions.
--- EOF ---