1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 }
  458 
  459 static bool generate_vzeroupper(Compile* C) {
  460   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  461 }
  462 
  463 static int clear_avx_size() {
  464   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  465 }
  466 
  467 // !!!!! Special hack to get all types of calls to specify the byte offset
  468 //       from the start of the call to the point where the return address
  469 //       will point.
  470 int MachCallStaticJavaNode::ret_addr_offset()
  471 {
  472   int offset = 5; // 5 bytes from start of call to where return address points
  473   offset += clear_avx_size();
  474   return offset;
  475 }
  476 
  477 int MachCallDynamicJavaNode::ret_addr_offset()
  478 {
  479   int offset = 15; // 15 bytes from start of call to where return address points
  480   offset += clear_avx_size();
  481   return offset;
  482 }
  483 
  484 int MachCallRuntimeNode::ret_addr_offset() {
  485   int offset = 13; // movq r10,#addr; callq (r10)
  486   if (this->ideal_Opcode() != Op_CallLeafVector) {
  487     offset += clear_avx_size();
  488   }
  489   return offset;
  490 }
  491 //
  492 // Compute padding required for nodes which need alignment
  493 //
  494 
  495 // The address of the call instruction needs to be 4-byte aligned to
  496 // ensure that it does not span a cache line so that it can be patched.
  497 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  498 {
  499   current_offset += clear_avx_size(); // skip vzeroupper
  500   current_offset += 1; // skip call opcode byte
  501   return align_up(current_offset, alignment_required()) - current_offset;
  502 }
  503 
  504 // The address of the call instruction needs to be 4-byte aligned to
  505 // ensure that it does not span a cache line so that it can be patched.
  506 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  507 {
  508   current_offset += clear_avx_size(); // skip vzeroupper
  509   current_offset += 11; // skip movq instruction + call opcode byte
  510   return align_up(current_offset, alignment_required()) - current_offset;
  511 }
  512 
  513 // EMIT_RM()
  514 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  515   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  516   cbuf.insts()->emit_int8(c);
  517 }
  518 
  519 // EMIT_CC()
  520 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  521   unsigned char c = (unsigned char) (f1 | f2);
  522   cbuf.insts()->emit_int8(c);
  523 }
  524 
  525 // EMIT_OPCODE()
  526 void emit_opcode(CodeBuffer &cbuf, int code) {
  527   cbuf.insts()->emit_int8((unsigned char) code);
  528 }
  529 
  530 // EMIT_OPCODE() w/ relocation information
  531 void emit_opcode(CodeBuffer &cbuf,
  532                  int code, relocInfo::relocType reloc, int offset, int format)
  533 {
  534   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  535   emit_opcode(cbuf, code);
  536 }
  537 
  538 // EMIT_D8()
  539 void emit_d8(CodeBuffer &cbuf, int d8) {
  540   cbuf.insts()->emit_int8((unsigned char) d8);
  541 }
  542 
  543 // EMIT_D16()
  544 void emit_d16(CodeBuffer &cbuf, int d16) {
  545   cbuf.insts()->emit_int16(d16);
  546 }
  547 
  548 // EMIT_D32()
  549 void emit_d32(CodeBuffer &cbuf, int d32) {
  550   cbuf.insts()->emit_int32(d32);
  551 }
  552 
  553 // EMIT_D64()
  554 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  555   cbuf.insts()->emit_int64(d64);
  556 }
  557 
  558 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  559 void emit_d32_reloc(CodeBuffer& cbuf,
  560                     int d32,
  561                     relocInfo::relocType reloc,
  562                     int format)
  563 {
  564   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  565   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  566   cbuf.insts()->emit_int32(d32);
  567 }
  568 
  569 // emit 32 bit value and construct relocation entry from RelocationHolder
  570 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  571 #ifdef ASSERT
  572   if (rspec.reloc()->type() == relocInfo::oop_type &&
  573       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  574     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  575     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  576   }
  577 #endif
  578   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  579   cbuf.insts()->emit_int32(d32);
  580 }
  581 
  582 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  583   address next_ip = cbuf.insts_end() + 4;
  584   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  585                  external_word_Relocation::spec(addr),
  586                  RELOC_DISP32);
  587 }
  588 
  589 
  590 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  591 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  592   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  593   cbuf.insts()->emit_int64(d64);
  594 }
  595 
  596 // emit 64 bit value and construct relocation entry from RelocationHolder
  597 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  598 #ifdef ASSERT
  599   if (rspec.reloc()->type() == relocInfo::oop_type &&
  600       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  601     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  602     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  603   }
  604 #endif
  605   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  606   cbuf.insts()->emit_int64(d64);
  607 }
  608 
  609 // Access stack slot for load or store
  610 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  611 {
  612   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  613   if (-0x80 <= disp && disp < 0x80) {
  614     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  615     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  616     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  617   } else {
  618     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  619     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  620     emit_d32(cbuf, disp);     // Displacement // R/M byte
  621   }
  622 }
  623 
  624    // rRegI ereg, memory mem) %{    // emit_reg_mem
  625 void encode_RegMem(CodeBuffer &cbuf,
  626                    int reg,
  627                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  628 {
  629   assert(disp_reloc == relocInfo::none, "cannot have disp");
  630   int regenc = reg & 7;
  631   int baseenc = base & 7;
  632   int indexenc = index & 7;
  633 
  634   // There is no index & no scale, use form without SIB byte
  635   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  636     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  637     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  638       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  639     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  640       // If 8-bit displacement, mode 0x1
  641       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  642       emit_d8(cbuf, disp);
  643     } else {
  644       // If 32-bit displacement
  645       if (base == -1) { // Special flag for absolute address
  646         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  647         if (disp_reloc != relocInfo::none) {
  648           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  649         } else {
  650           emit_d32(cbuf, disp);
  651         }
  652       } else {
  653         // Normal base + offset
  654         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  655         if (disp_reloc != relocInfo::none) {
  656           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  657         } else {
  658           emit_d32(cbuf, disp);
  659         }
  660       }
  661     }
  662   } else {
  663     // Else, encode with the SIB byte
  664     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  665     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  666       // If no displacement
  667       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  668       emit_rm(cbuf, scale, indexenc, baseenc);
  669     } else {
  670       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  671         // If 8-bit displacement, mode 0x1
  672         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  673         emit_rm(cbuf, scale, indexenc, baseenc);
  674         emit_d8(cbuf, disp);
  675       } else {
  676         // If 32-bit displacement
  677         if (base == 0x04 ) {
  678           emit_rm(cbuf, 0x2, regenc, 0x4);
  679           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  680         } else {
  681           emit_rm(cbuf, 0x2, regenc, 0x4);
  682           emit_rm(cbuf, scale, indexenc, baseenc); // *
  683         }
  684         if (disp_reloc != relocInfo::none) {
  685           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  686         } else {
  687           emit_d32(cbuf, disp);
  688         }
  689       }
  690     }
  691   }
  692 }
  693 
  694 // This could be in MacroAssembler but it's fairly C2 specific
  695 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  696   Label exit;
  697   __ jccb(Assembler::noParity, exit);
  698   __ pushf();
  699   //
  700   // comiss/ucomiss instructions set ZF,PF,CF flags and
  701   // zero OF,AF,SF for NaN values.
  702   // Fixup flags by zeroing ZF,PF so that compare of NaN
  703   // values returns 'less than' result (CF is set).
  704   // Leave the rest of flags unchanged.
  705   //
  706   //    7 6 5 4 3 2 1 0
  707   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  708   //    0 0 1 0 1 0 1 1   (0x2B)
  709   //
  710   __ andq(Address(rsp, 0), 0xffffff2b);
  711   __ popf();
  712   __ bind(exit);
  713 }
  714 
  715 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  716   Label done;
  717   __ movl(dst, -1);
  718   __ jcc(Assembler::parity, done);
  719   __ jcc(Assembler::below, done);
  720   __ setb(Assembler::notEqual, dst);
  721   __ movzbl(dst, dst);
  722   __ bind(done);
  723 }
  724 
  725 // Math.min()    # Math.max()
  726 // --------------------------
  727 // ucomis[s/d]   #
  728 // ja   -> b     # a
  729 // jp   -> NaN   # NaN
  730 // jb   -> a     # b
  731 // je            #
  732 // |-jz -> a | b # a & b
  733 // |    -> a     #
  734 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  735                      XMMRegister a, XMMRegister b,
  736                      XMMRegister xmmt, Register rt,
  737                      bool min, bool single) {
  738 
  739   Label nan, zero, below, above, done;
  740 
  741   if (single)
  742     __ ucomiss(a, b);
  743   else
  744     __ ucomisd(a, b);
  745 
  746   if (dst->encoding() != (min ? b : a)->encoding())
  747     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  748   else
  749     __ jccb(Assembler::above, done);
  750 
  751   __ jccb(Assembler::parity, nan);  // PF=1
  752   __ jccb(Assembler::below, below); // CF=1
  753 
  754   // equal
  755   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  756   if (single) {
  757     __ ucomiss(a, xmmt);
  758     __ jccb(Assembler::equal, zero);
  759 
  760     __ movflt(dst, a);
  761     __ jmp(done);
  762   }
  763   else {
  764     __ ucomisd(a, xmmt);
  765     __ jccb(Assembler::equal, zero);
  766 
  767     __ movdbl(dst, a);
  768     __ jmp(done);
  769   }
  770 
  771   __ bind(zero);
  772   if (min)
  773     __ vpor(dst, a, b, Assembler::AVX_128bit);
  774   else
  775     __ vpand(dst, a, b, Assembler::AVX_128bit);
  776 
  777   __ jmp(done);
  778 
  779   __ bind(above);
  780   if (single)
  781     __ movflt(dst, min ? b : a);
  782   else
  783     __ movdbl(dst, min ? b : a);
  784 
  785   __ jmp(done);
  786 
  787   __ bind(nan);
  788   if (single) {
  789     __ movl(rt, 0x7fc00000); // Float.NaN
  790     __ movdl(dst, rt);
  791   }
  792   else {
  793     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  794     __ movdq(dst, rt);
  795   }
  796   __ jmp(done);
  797 
  798   __ bind(below);
  799   if (single)
  800     __ movflt(dst, min ? a : b);
  801   else
  802     __ movdbl(dst, min ? a : b);
  803 
  804   __ bind(done);
  805 }
  806 
  807 //=============================================================================
  808 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  809 
  810 int ConstantTable::calculate_table_base_offset() const {
  811   return 0;  // absolute addressing, no offset
  812 }
  813 
  814 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  815 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  816   ShouldNotReachHere();
  817 }
  818 
  819 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  820   // Empty encoding
  821 }
  822 
  823 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  824   return 0;
  825 }
  826 
  827 #ifndef PRODUCT
  828 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  829   st->print("# MachConstantBaseNode (empty encoding)");
  830 }
  831 #endif
  832 
  833 
  834 //=============================================================================
  835 #ifndef PRODUCT
  836 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  837   Compile* C = ra_->C;
  838 
  839   int framesize = C->output()->frame_size_in_bytes();
  840   int bangsize = C->output()->bang_size_in_bytes();
  841   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  842   // Remove wordSize for return addr which is already pushed.
  843   framesize -= wordSize;
  844 
  845   if (C->output()->need_stack_bang(bangsize)) {
  846     framesize -= wordSize;
  847     st->print("# stack bang (%d bytes)", bangsize);
  848     st->print("\n\t");
  849     st->print("pushq   rbp\t# Save rbp");
  850     if (PreserveFramePointer) {
  851         st->print("\n\t");
  852         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  853     }
  854     if (framesize) {
  855       st->print("\n\t");
  856       st->print("subq    rsp, #%d\t# Create frame",framesize);
  857     }
  858   } else {
  859     st->print("subq    rsp, #%d\t# Create frame",framesize);
  860     st->print("\n\t");
  861     framesize -= wordSize;
  862     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  863     if (PreserveFramePointer) {
  864       st->print("\n\t");
  865       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  866       if (framesize > 0) {
  867         st->print("\n\t");
  868         st->print("addq    rbp, #%d", framesize);
  869       }
  870     }
  871   }
  872 
  873   if (VerifyStackAtCalls) {
  874     st->print("\n\t");
  875     framesize -= wordSize;
  876     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  877 #ifdef ASSERT
  878     st->print("\n\t");
  879     st->print("# stack alignment check");
  880 #endif
  881   }
  882   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  883     st->print("\n\t");
  884     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  885     st->print("\n\t");
  886     st->print("je      fast_entry\t");
  887     st->print("\n\t");
  888     st->print("call    #nmethod_entry_barrier_stub\t");
  889     st->print("\n\tfast_entry:");
  890   }
  891   st->cr();
  892 }
  893 #endif
  894 
  895 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  896   Compile* C = ra_->C;
  897   C2_MacroAssembler _masm(&cbuf);
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   int bangsize = C->output()->bang_size_in_bytes();
  901 
  902   if (C->clinit_barrier_on_entry()) {
  903     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  904     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  905 
  906     Label L_skip_barrier;
  907     Register klass = rscratch1;
  908 
  909     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  910     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  911 
  912     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  913 
  914     __ bind(L_skip_barrier);
  915   }
  916 
  917   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
  918 
  919   C->output()->set_frame_complete(cbuf.insts_size());
  920 
  921   if (C->has_mach_constant_base_node()) {
  922     // NOTE: We set the table base offset here because users might be
  923     // emitted before MachConstantBaseNode.
  924     ConstantTable& constant_table = C->output()->constant_table();
  925     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  926   }
  927 }
  928 
  929 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  930 {
  931   return MachNode::size(ra_); // too many variables; just compute it
  932                               // the hard way
  933 }
  934 
  935 int MachPrologNode::reloc() const
  936 {
  937   return 0; // a large enough number
  938 }
  939 
  940 //=============================================================================
  941 #ifndef PRODUCT
  942 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  943 {
  944   Compile* C = ra_->C;
  945   if (generate_vzeroupper(C)) {
  946     st->print("vzeroupper");
  947     st->cr(); st->print("\t");
  948   }
  949 
  950   int framesize = C->output()->frame_size_in_bytes();
  951   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  952   // Remove word for return adr already pushed
  953   // and RBP
  954   framesize -= 2*wordSize;
  955 
  956   if (framesize) {
  957     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  958     st->print("\t");
  959   }
  960 
  961   st->print_cr("popq    rbp");
  962   if (do_polling() && C->is_method_compilation()) {
  963     st->print("\t");
  964     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  965                  "ja       #safepoint_stub\t"
  966                  "# Safepoint: poll for GC");
  967   }
  968 }
  969 #endif
  970 
  971 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  972 {
  973   Compile* C = ra_->C;
  974   MacroAssembler _masm(&cbuf);
  975 
  976   if (generate_vzeroupper(C)) {
  977     // Clear upper bits of YMM registers when current compiled code uses
  978     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  979     __ vzeroupper();
  980   }
  981 
  982   int framesize = C->output()->frame_size_in_bytes();
  983   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  984   // Remove word for return adr already pushed
  985   // and RBP
  986   framesize -= 2*wordSize;
  987 
  988   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  989 
  990   if (framesize) {
  991     emit_opcode(cbuf, Assembler::REX_W);
  992     if (framesize < 0x80) {
  993       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
  994       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  995       emit_d8(cbuf, framesize);
  996     } else {
  997       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
  998       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
  999       emit_d32(cbuf, framesize);
 1000     }
 1001   }
 1002 
 1003   // popq rbp
 1004   emit_opcode(cbuf, 0x58 | RBP_enc);
 1005 
 1006   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1007     __ reserved_stack_check();
 1008   }
 1009 
 1010   if (do_polling() && C->is_method_compilation()) {
 1011     MacroAssembler _masm(&cbuf);
 1012     Label dummy_label;
 1013     Label* code_stub = &dummy_label;
 1014     if (!C->output()->in_scratch_emit_size()) {
 1015       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1016     }
 1017     __ relocate(relocInfo::poll_return_type);
 1018     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1019   }
 1020 }
 1021 
 1022 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 1023 {
 1024   return MachNode::size(ra_); // too many variables; just compute it
 1025                               // the hard way
 1026 }
 1027 
 1028 int MachEpilogNode::reloc() const
 1029 {
 1030   return 2; // a large enough number
 1031 }
 1032 
 1033 const Pipeline* MachEpilogNode::pipeline() const
 1034 {
 1035   return MachNode::pipeline_class();
 1036 }
 1037 
 1038 //=============================================================================
 1039 
 1040 enum RC {
 1041   rc_bad,
 1042   rc_int,
 1043   rc_kreg,
 1044   rc_float,
 1045   rc_stack
 1046 };
 1047 
 1048 static enum RC rc_class(OptoReg::Name reg)
 1049 {
 1050   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1051 
 1052   if (OptoReg::is_stack(reg)) return rc_stack;
 1053 
 1054   VMReg r = OptoReg::as_VMReg(reg);
 1055 
 1056   if (r->is_Register()) return rc_int;
 1057 
 1058   if (r->is_KRegister()) return rc_kreg;
 1059 
 1060   assert(r->is_XMMRegister(), "must be");
 1061   return rc_float;
 1062 }
 1063 
 1064 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1065 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1066                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1067 
 1068 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1069                      int stack_offset, int reg, uint ireg, outputStream* st);
 1070 
 1071 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1072                                       int dst_offset, uint ireg, outputStream* st) {
 1073   if (cbuf) {
 1074     MacroAssembler _masm(cbuf);
 1075     switch (ireg) {
 1076     case Op_VecS:
 1077       __ movq(Address(rsp, -8), rax);
 1078       __ movl(rax, Address(rsp, src_offset));
 1079       __ movl(Address(rsp, dst_offset), rax);
 1080       __ movq(rax, Address(rsp, -8));
 1081       break;
 1082     case Op_VecD:
 1083       __ pushq(Address(rsp, src_offset));
 1084       __ popq (Address(rsp, dst_offset));
 1085       break;
 1086     case Op_VecX:
 1087       __ pushq(Address(rsp, src_offset));
 1088       __ popq (Address(rsp, dst_offset));
 1089       __ pushq(Address(rsp, src_offset+8));
 1090       __ popq (Address(rsp, dst_offset+8));
 1091       break;
 1092     case Op_VecY:
 1093       __ vmovdqu(Address(rsp, -32), xmm0);
 1094       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1095       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1096       __ vmovdqu(xmm0, Address(rsp, -32));
 1097       break;
 1098     case Op_VecZ:
 1099       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1100       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1101       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1102       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1103       break;
 1104     default:
 1105       ShouldNotReachHere();
 1106     }
 1107 #ifndef PRODUCT
 1108   } else {
 1109     switch (ireg) {
 1110     case Op_VecS:
 1111       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1112                 "movl    rax, [rsp + #%d]\n\t"
 1113                 "movl    [rsp + #%d], rax\n\t"
 1114                 "movq    rax, [rsp - #8]",
 1115                 src_offset, dst_offset);
 1116       break;
 1117     case Op_VecD:
 1118       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1119                 "popq    [rsp + #%d]",
 1120                 src_offset, dst_offset);
 1121       break;
 1122      case Op_VecX:
 1123       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1124                 "popq    [rsp + #%d]\n\t"
 1125                 "pushq   [rsp + #%d]\n\t"
 1126                 "popq    [rsp + #%d]",
 1127                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1128       break;
 1129     case Op_VecY:
 1130       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1131                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1132                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1133                 "vmovdqu xmm0, [rsp - #32]",
 1134                 src_offset, dst_offset);
 1135       break;
 1136     case Op_VecZ:
 1137       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1138                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1139                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1140                 "vmovdqu xmm0, [rsp - #64]",
 1141                 src_offset, dst_offset);
 1142       break;
 1143     default:
 1144       ShouldNotReachHere();
 1145     }
 1146 #endif
 1147   }
 1148 }
 1149 
 1150 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1151                                        PhaseRegAlloc* ra_,
 1152                                        bool do_size,
 1153                                        outputStream* st) const {
 1154   assert(cbuf != NULL || st  != NULL, "sanity");
 1155   // Get registers to move
 1156   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1157   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1158   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1159   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1160 
 1161   enum RC src_second_rc = rc_class(src_second);
 1162   enum RC src_first_rc = rc_class(src_first);
 1163   enum RC dst_second_rc = rc_class(dst_second);
 1164   enum RC dst_first_rc = rc_class(dst_first);
 1165 
 1166   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1167          "must move at least 1 register" );
 1168 
 1169   if (src_first == dst_first && src_second == dst_second) {
 1170     // Self copy, no move
 1171     return 0;
 1172   }
 1173   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1174     uint ireg = ideal_reg();
 1175     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1176     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1177     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1178       // mem -> mem
 1179       int src_offset = ra_->reg2offset(src_first);
 1180       int dst_offset = ra_->reg2offset(dst_first);
 1181       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1182     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1183       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1184     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1185       int stack_offset = ra_->reg2offset(dst_first);
 1186       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1187     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1188       int stack_offset = ra_->reg2offset(src_first);
 1189       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1190     } else {
 1191       ShouldNotReachHere();
 1192     }
 1193     return 0;
 1194   }
 1195   if (src_first_rc == rc_stack) {
 1196     // mem ->
 1197     if (dst_first_rc == rc_stack) {
 1198       // mem -> mem
 1199       assert(src_second != dst_first, "overlap");
 1200       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1201           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1202         // 64-bit
 1203         int src_offset = ra_->reg2offset(src_first);
 1204         int dst_offset = ra_->reg2offset(dst_first);
 1205         if (cbuf) {
 1206           MacroAssembler _masm(cbuf);
 1207           __ pushq(Address(rsp, src_offset));
 1208           __ popq (Address(rsp, dst_offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1212                     "popq    [rsp + #%d]",
 1213                      src_offset, dst_offset);
 1214 #endif
 1215         }
 1216       } else {
 1217         // 32-bit
 1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1220         // No pushl/popl, so:
 1221         int src_offset = ra_->reg2offset(src_first);
 1222         int dst_offset = ra_->reg2offset(dst_first);
 1223         if (cbuf) {
 1224           MacroAssembler _masm(cbuf);
 1225           __ movq(Address(rsp, -8), rax);
 1226           __ movl(rax, Address(rsp, src_offset));
 1227           __ movl(Address(rsp, dst_offset), rax);
 1228           __ movq(rax, Address(rsp, -8));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1232                     "movl    rax, [rsp + #%d]\n\t"
 1233                     "movl    [rsp + #%d], rax\n\t"
 1234                     "movq    rax, [rsp - #8]",
 1235                      src_offset, dst_offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_int) {
 1241       // mem -> gpr
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (cbuf) {
 1247           MacroAssembler _masm(cbuf);
 1248           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1249 #ifndef PRODUCT
 1250         } else {
 1251           st->print("movq    %s, [rsp + #%d]\t# spill",
 1252                      Matcher::regName[dst_first],
 1253                      offset);
 1254 #endif
 1255         }
 1256       } else {
 1257         // 32-bit
 1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1260         int offset = ra_->reg2offset(src_first);
 1261         if (cbuf) {
 1262           MacroAssembler _masm(cbuf);
 1263           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1264 #ifndef PRODUCT
 1265         } else {
 1266           st->print("movl    %s, [rsp + #%d]\t# spill",
 1267                      Matcher::regName[dst_first],
 1268                      offset);
 1269 #endif
 1270         }
 1271       }
 1272       return 0;
 1273     } else if (dst_first_rc == rc_float) {
 1274       // mem-> xmm
 1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1277         // 64-bit
 1278         int offset = ra_->reg2offset(src_first);
 1279         if (cbuf) {
 1280           MacroAssembler _masm(cbuf);
 1281           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("%s  %s, [rsp + #%d]\t# spill",
 1285                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1286                      Matcher::regName[dst_first],
 1287                      offset);
 1288 #endif
 1289         }
 1290       } else {
 1291         // 32-bit
 1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1294         int offset = ra_->reg2offset(src_first);
 1295         if (cbuf) {
 1296           MacroAssembler _masm(cbuf);
 1297           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movss   %s, [rsp + #%d]\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      offset);
 1303 #endif
 1304         }
 1305       }
 1306       return 0;
 1307     } else if (dst_first_rc == rc_kreg) {
 1308       // mem -> kreg
 1309       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1310           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1311         // 64-bit
 1312         int offset = ra_->reg2offset(src_first);
 1313         if (cbuf) {
 1314           MacroAssembler _masm(cbuf);
 1315           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1316 #ifndef PRODUCT
 1317         } else {
 1318           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1319                      Matcher::regName[dst_first],
 1320                      offset);
 1321 #endif
 1322         }
 1323       }
 1324       return 0;
 1325     }
 1326   } else if (src_first_rc == rc_int) {
 1327     // gpr ->
 1328     if (dst_first_rc == rc_stack) {
 1329       // gpr -> mem
 1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1332         // 64-bit
 1333         int offset = ra_->reg2offset(dst_first);
 1334         if (cbuf) {
 1335           MacroAssembler _masm(cbuf);
 1336           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1337 #ifndef PRODUCT
 1338         } else {
 1339           st->print("movq    [rsp + #%d], %s\t# spill",
 1340                      offset,
 1341                      Matcher::regName[src_first]);
 1342 #endif
 1343         }
 1344       } else {
 1345         // 32-bit
 1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1348         int offset = ra_->reg2offset(dst_first);
 1349         if (cbuf) {
 1350           MacroAssembler _masm(cbuf);
 1351           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1352 #ifndef PRODUCT
 1353         } else {
 1354           st->print("movl    [rsp + #%d], %s\t# spill",
 1355                      offset,
 1356                      Matcher::regName[src_first]);
 1357 #endif
 1358         }
 1359       }
 1360       return 0;
 1361     } else if (dst_first_rc == rc_int) {
 1362       // gpr -> gpr
 1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1365         // 64-bit
 1366         if (cbuf) {
 1367           MacroAssembler _masm(cbuf);
 1368           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1369                   as_Register(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movq    %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377         return 0;
 1378       } else {
 1379         // 32-bit
 1380         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1381         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1382         if (cbuf) {
 1383           MacroAssembler _masm(cbuf);
 1384           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1385                   as_Register(Matcher::_regEncode[src_first]));
 1386 #ifndef PRODUCT
 1387         } else {
 1388           st->print("movl    %s, %s\t# spill",
 1389                      Matcher::regName[dst_first],
 1390                      Matcher::regName[src_first]);
 1391 #endif
 1392         }
 1393         return 0;
 1394       }
 1395     } else if (dst_first_rc == rc_float) {
 1396       // gpr -> xmm
 1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1399         // 64-bit
 1400         if (cbuf) {
 1401           MacroAssembler _masm(cbuf);
 1402           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1403 #ifndef PRODUCT
 1404         } else {
 1405           st->print("movdq   %s, %s\t# spill",
 1406                      Matcher::regName[dst_first],
 1407                      Matcher::regName[src_first]);
 1408 #endif
 1409         }
 1410       } else {
 1411         // 32-bit
 1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1414         if (cbuf) {
 1415           MacroAssembler _masm(cbuf);
 1416           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1417 #ifndef PRODUCT
 1418         } else {
 1419           st->print("movdl   %s, %s\t# spill",
 1420                      Matcher::regName[dst_first],
 1421                      Matcher::regName[src_first]);
 1422 #endif
 1423         }
 1424       }
 1425       return 0;
 1426     } else if (dst_first_rc == rc_kreg) {
 1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1429         // 64-bit
 1430         if (cbuf) {
 1431           MacroAssembler _masm(cbuf);
 1432           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1433   #ifndef PRODUCT
 1434         } else {
 1435            st->print("kmovq   %s, %s\t# spill",
 1436                        Matcher::regName[dst_first],
 1437                        Matcher::regName[src_first]);
 1438   #endif
 1439         }
 1440       }
 1441       Unimplemented();
 1442       return 0;
 1443     }
 1444   } else if (src_first_rc == rc_float) {
 1445     // xmm ->
 1446     if (dst_first_rc == rc_stack) {
 1447       // xmm -> mem
 1448       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1449           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1450         // 64-bit
 1451         int offset = ra_->reg2offset(dst_first);
 1452         if (cbuf) {
 1453           MacroAssembler _masm(cbuf);
 1454           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1455 #ifndef PRODUCT
 1456         } else {
 1457           st->print("movsd   [rsp + #%d], %s\t# spill",
 1458                      offset,
 1459                      Matcher::regName[src_first]);
 1460 #endif
 1461         }
 1462       } else {
 1463         // 32-bit
 1464         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1465         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1466         int offset = ra_->reg2offset(dst_first);
 1467         if (cbuf) {
 1468           MacroAssembler _masm(cbuf);
 1469           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1470 #ifndef PRODUCT
 1471         } else {
 1472           st->print("movss   [rsp + #%d], %s\t# spill",
 1473                      offset,
 1474                      Matcher::regName[src_first]);
 1475 #endif
 1476         }
 1477       }
 1478       return 0;
 1479     } else if (dst_first_rc == rc_int) {
 1480       // xmm -> gpr
 1481       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1482           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1483         // 64-bit
 1484         if (cbuf) {
 1485           MacroAssembler _masm(cbuf);
 1486           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1487 #ifndef PRODUCT
 1488         } else {
 1489           st->print("movdq   %s, %s\t# spill",
 1490                      Matcher::regName[dst_first],
 1491                      Matcher::regName[src_first]);
 1492 #endif
 1493         }
 1494       } else {
 1495         // 32-bit
 1496         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1497         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1498         if (cbuf) {
 1499           MacroAssembler _masm(cbuf);
 1500           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1501 #ifndef PRODUCT
 1502         } else {
 1503           st->print("movdl   %s, %s\t# spill",
 1504                      Matcher::regName[dst_first],
 1505                      Matcher::regName[src_first]);
 1506 #endif
 1507         }
 1508       }
 1509       return 0;
 1510     } else if (dst_first_rc == rc_float) {
 1511       // xmm -> xmm
 1512       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1513           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1514         // 64-bit
 1515         if (cbuf) {
 1516           MacroAssembler _masm(cbuf);
 1517           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1518 #ifndef PRODUCT
 1519         } else {
 1520           st->print("%s  %s, %s\t# spill",
 1521                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1522                      Matcher::regName[dst_first],
 1523                      Matcher::regName[src_first]);
 1524 #endif
 1525         }
 1526       } else {
 1527         // 32-bit
 1528         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1529         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1530         if (cbuf) {
 1531           MacroAssembler _masm(cbuf);
 1532           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1533 #ifndef PRODUCT
 1534         } else {
 1535           st->print("%s  %s, %s\t# spill",
 1536                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1537                      Matcher::regName[dst_first],
 1538                      Matcher::regName[src_first]);
 1539 #endif
 1540         }
 1541       }
 1542       return 0;
 1543     } else if (dst_first_rc == rc_kreg) {
 1544       assert(false, "Illegal spilling");
 1545       return 0;
 1546     }
 1547   } else if (src_first_rc == rc_kreg) {
 1548     if (dst_first_rc == rc_stack) {
 1549       // mem -> kreg
 1550       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1551           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1552         // 64-bit
 1553         int offset = ra_->reg2offset(dst_first);
 1554         if (cbuf) {
 1555           MacroAssembler _masm(cbuf);
 1556           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1557 #ifndef PRODUCT
 1558         } else {
 1559           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1560                      offset,
 1561                      Matcher::regName[src_first]);
 1562 #endif
 1563         }
 1564       }
 1565       return 0;
 1566     } else if (dst_first_rc == rc_int) {
 1567       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1568           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1569         // 64-bit
 1570         if (cbuf) {
 1571           MacroAssembler _masm(cbuf);
 1572           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1573 #ifndef PRODUCT
 1574         } else {
 1575          st->print("kmovq   %s, %s\t# spill",
 1576                      Matcher::regName[dst_first],
 1577                      Matcher::regName[src_first]);
 1578 #endif
 1579         }
 1580       }
 1581       Unimplemented();
 1582       return 0;
 1583     } else if (dst_first_rc == rc_kreg) {
 1584       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1585           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1586         // 64-bit
 1587         if (cbuf) {
 1588           MacroAssembler _masm(cbuf);
 1589           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1590 #ifndef PRODUCT
 1591         } else {
 1592          st->print("kmovq   %s, %s\t# spill",
 1593                      Matcher::regName[dst_first],
 1594                      Matcher::regName[src_first]);
 1595 #endif
 1596         }
 1597       }
 1598       return 0;
 1599     } else if (dst_first_rc == rc_float) {
 1600       assert(false, "Illegal spill");
 1601       return 0;
 1602     }
 1603   }
 1604 
 1605   assert(0," foo ");
 1606   Unimplemented();
 1607   return 0;
 1608 }
 1609 
 1610 #ifndef PRODUCT
 1611 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1612   implementation(NULL, ra_, false, st);
 1613 }
 1614 #endif
 1615 
 1616 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1617   implementation(&cbuf, ra_, false, NULL);
 1618 }
 1619 
 1620 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1621   return MachNode::size(ra_);
 1622 }
 1623 
 1624 //=============================================================================
 1625 #ifndef PRODUCT
 1626 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1627 {
 1628   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1629   int reg = ra_->get_reg_first(this);
 1630   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1631             Matcher::regName[reg], offset);
 1632 }
 1633 #endif
 1634 
 1635 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1636 {
 1637   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1638   int reg = ra_->get_encode(this);
 1639   if (offset >= 0x80) {
 1640     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1641     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1642     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1643     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1644     emit_d32(cbuf, offset);
 1645   } else {
 1646     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1647     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1648     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1649     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1650     emit_d8(cbuf, offset);
 1651   }
 1652 }
 1653 
 1654 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1655 {
 1656   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1657   return (offset < 0x80) ? 5 : 8; // REX
 1658 }
 1659 
 1660 //=============================================================================
 1661 #ifndef PRODUCT
 1662 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1663 {
 1664   if (UseCompressedClassPointers) {
 1665     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1666     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1667     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1668   } else {
 1669     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1670                  "# Inline cache check");
 1671   }
 1672   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1673   st->print_cr("\tnop\t# nops to align entry point");
 1674 }
 1675 #endif
 1676 
 1677 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1678 {
 1679   MacroAssembler masm(&cbuf);
 1680   uint insts_size = cbuf.insts_size();
 1681   if (UseCompressedClassPointers) {
 1682     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1683     masm.cmpptr(rax, rscratch1);
 1684   } else {
 1685     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1686   }
 1687 
 1688   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1689 
 1690   /* WARNING these NOPs are critical so that verified entry point is properly
 1691      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1692   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1693   if (OptoBreakpoint) {
 1694     // Leave space for int3
 1695     nops_cnt -= 1;
 1696   }
 1697   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1698   if (nops_cnt > 0)
 1699     masm.nop(nops_cnt);
 1700 }
 1701 
 1702 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1703 {
 1704   return MachNode::size(ra_); // too many variables; just compute it
 1705                               // the hard way
 1706 }
 1707 
 1708 
 1709 //=============================================================================
 1710 
 1711 const bool Matcher::supports_vector_calling_convention(void) {
 1712   if (EnableVectorSupport && UseVectorStubs) {
 1713     return true;
 1714   }
 1715   return false;
 1716 }
 1717 
 1718 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1719   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1720   int lo = XMM0_num;
 1721   int hi = XMM0b_num;
 1722   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1723   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1724   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1725   return OptoRegPair(hi, lo);
 1726 }
 1727 
 1728 // Is this branch offset short enough that a short branch can be used?
 1729 //
 1730 // NOTE: If the platform does not provide any short branch variants, then
 1731 //       this method should return false for offset 0.
 1732 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1733   // The passed offset is relative to address of the branch.
 1734   // On 86 a branch displacement is calculated relative to address
 1735   // of a next instruction.
 1736   offset -= br_size;
 1737 
 1738   // the short version of jmpConUCF2 contains multiple branches,
 1739   // making the reach slightly less
 1740   if (rule == jmpConUCF2_rule)
 1741     return (-126 <= offset && offset <= 125);
 1742   return (-128 <= offset && offset <= 127);
 1743 }
 1744 
 1745 // Return whether or not this register is ever used as an argument.
 1746 // This function is used on startup to build the trampoline stubs in
 1747 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1748 // call in the trampoline, and arguments in those registers not be
 1749 // available to the callee.
 1750 bool Matcher::can_be_java_arg(int reg)
 1751 {
 1752   return
 1753     reg ==  RDI_num || reg == RDI_H_num ||
 1754     reg ==  RSI_num || reg == RSI_H_num ||
 1755     reg ==  RDX_num || reg == RDX_H_num ||
 1756     reg ==  RCX_num || reg == RCX_H_num ||
 1757     reg ==   R8_num || reg ==  R8_H_num ||
 1758     reg ==   R9_num || reg ==  R9_H_num ||
 1759     reg ==  R12_num || reg == R12_H_num ||
 1760     reg == XMM0_num || reg == XMM0b_num ||
 1761     reg == XMM1_num || reg == XMM1b_num ||
 1762     reg == XMM2_num || reg == XMM2b_num ||
 1763     reg == XMM3_num || reg == XMM3b_num ||
 1764     reg == XMM4_num || reg == XMM4b_num ||
 1765     reg == XMM5_num || reg == XMM5b_num ||
 1766     reg == XMM6_num || reg == XMM6b_num ||
 1767     reg == XMM7_num || reg == XMM7b_num;
 1768 }
 1769 
 1770 bool Matcher::is_spillable_arg(int reg)
 1771 {
 1772   return can_be_java_arg(reg);
 1773 }
 1774 
 1775 uint Matcher::int_pressure_limit()
 1776 {
 1777   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1778 }
 1779 
 1780 uint Matcher::float_pressure_limit()
 1781 {
 1782   // After experiment around with different values, the following default threshold
 1783   // works best for LCM's register pressure scheduling on x64.
 1784   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1785   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1786   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1787 }
 1788 
 1789 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1790   // In 64 bit mode a code which use multiply when
 1791   // devisor is constant is faster than hardware
 1792   // DIV instruction (it uses MulHiL).
 1793   return false;
 1794 }
 1795 
 1796 // Register for DIVI projection of divmodI
 1797 RegMask Matcher::divI_proj_mask() {
 1798   return INT_RAX_REG_mask();
 1799 }
 1800 
 1801 // Register for MODI projection of divmodI
 1802 RegMask Matcher::modI_proj_mask() {
 1803   return INT_RDX_REG_mask();
 1804 }
 1805 
 1806 // Register for DIVL projection of divmodL
 1807 RegMask Matcher::divL_proj_mask() {
 1808   return LONG_RAX_REG_mask();
 1809 }
 1810 
 1811 // Register for MODL projection of divmodL
 1812 RegMask Matcher::modL_proj_mask() {
 1813   return LONG_RDX_REG_mask();
 1814 }
 1815 
 1816 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1817 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1818     return NO_REG_mask();
 1819 }
 1820 
 1821 %}
 1822 
 1823 //----------ENCODING BLOCK-----------------------------------------------------
 1824 // This block specifies the encoding classes used by the compiler to
 1825 // output byte streams.  Encoding classes are parameterized macros
 1826 // used by Machine Instruction Nodes in order to generate the bit
 1827 // encoding of the instruction.  Operands specify their base encoding
 1828 // interface with the interface keyword.  There are currently
 1829 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1830 // COND_INTER.  REG_INTER causes an operand to generate a function
 1831 // which returns its register number when queried.  CONST_INTER causes
 1832 // an operand to generate a function which returns the value of the
 1833 // constant when queried.  MEMORY_INTER causes an operand to generate
 1834 // four functions which return the Base Register, the Index Register,
 1835 // the Scale Value, and the Offset Value of the operand when queried.
 1836 // COND_INTER causes an operand to generate six functions which return
 1837 // the encoding code (ie - encoding bits for the instruction)
 1838 // associated with each basic boolean condition for a conditional
 1839 // instruction.
 1840 //
 1841 // Instructions specify two basic values for encoding.  Again, a
 1842 // function is available to check if the constant displacement is an
 1843 // oop. They use the ins_encode keyword to specify their encoding
 1844 // classes (which must be a sequence of enc_class names, and their
 1845 // parameters, specified in the encoding block), and they use the
 1846 // opcode keyword to specify, in order, their primary, secondary, and
 1847 // tertiary opcode.  Only the opcode sections which a particular
 1848 // instruction needs for encoding need to be specified.
 1849 encode %{
 1850   // Build emit functions for each basic byte or larger field in the
 1851   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1852   // from C++ code in the enc_class source block.  Emit functions will
 1853   // live in the main source block for now.  In future, we can
 1854   // generalize this by adding a syntax that specifies the sizes of
 1855   // fields in an order, so that the adlc can build the emit functions
 1856   // automagically
 1857 
 1858   // Emit primary opcode
 1859   enc_class OpcP
 1860   %{
 1861     emit_opcode(cbuf, $primary);
 1862   %}
 1863 
 1864   // Emit secondary opcode
 1865   enc_class OpcS
 1866   %{
 1867     emit_opcode(cbuf, $secondary);
 1868   %}
 1869 
 1870   // Emit tertiary opcode
 1871   enc_class OpcT
 1872   %{
 1873     emit_opcode(cbuf, $tertiary);
 1874   %}
 1875 
 1876   // Emit opcode directly
 1877   enc_class Opcode(immI d8)
 1878   %{
 1879     emit_opcode(cbuf, $d8$$constant);
 1880   %}
 1881 
 1882   // Emit size prefix
 1883   enc_class SizePrefix
 1884   %{
 1885     emit_opcode(cbuf, 0x66);
 1886   %}
 1887 
 1888   enc_class reg(rRegI reg)
 1889   %{
 1890     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1891   %}
 1892 
 1893   enc_class reg_reg(rRegI dst, rRegI src)
 1894   %{
 1895     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1896   %}
 1897 
 1898   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1899   %{
 1900     emit_opcode(cbuf, $opcode$$constant);
 1901     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1902   %}
 1903 
 1904   enc_class cdql_enc(no_rax_rdx_RegI div)
 1905   %{
 1906     // Full implementation of Java idiv and irem; checks for
 1907     // special case as described in JVM spec., p.243 & p.271.
 1908     //
 1909     //         normal case                           special case
 1910     //
 1911     // input : rax: dividend                         min_int
 1912     //         reg: divisor                          -1
 1913     //
 1914     // output: rax: quotient  (= rax idiv reg)       min_int
 1915     //         rdx: remainder (= rax irem reg)       0
 1916     //
 1917     //  Code sequnce:
 1918     //
 1919     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1920     //    5:   75 07/08                jne    e <normal>
 1921     //    7:   33 d2                   xor    %edx,%edx
 1922     //  [div >= 8 -> offset + 1]
 1923     //  [REX_B]
 1924     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1925     //    c:   74 03/04                je     11 <done>
 1926     // 000000000000000e <normal>:
 1927     //    e:   99                      cltd
 1928     //  [div >= 8 -> offset + 1]
 1929     //  [REX_B]
 1930     //    f:   f7 f9                   idiv   $div
 1931     // 0000000000000011 <done>:
 1932     MacroAssembler _masm(&cbuf);
 1933     Label normal;
 1934     Label done;
 1935 
 1936     // cmp    $0x80000000,%eax
 1937     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1938 
 1939     // jne    e <normal>
 1940     __ jccb(Assembler::notEqual, normal);
 1941 
 1942     // xor    %edx,%edx
 1943     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1944 
 1945     // cmp    $0xffffffffffffffff,%ecx
 1946     __ cmpl($div$$Register, -1);
 1947 
 1948     // je     11 <done>
 1949     __ jccb(Assembler::equal, done);
 1950 
 1951     // <normal>
 1952     // cltd
 1953     __ bind(normal);
 1954     __ cdql();
 1955 
 1956     // idivl
 1957     // <done>
 1958     __ idivl($div$$Register);
 1959     __ bind(done);
 1960   %}
 1961 
 1962   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1963   %{
 1964     // Full implementation of Java ldiv and lrem; checks for
 1965     // special case as described in JVM spec., p.243 & p.271.
 1966     //
 1967     //         normal case                           special case
 1968     //
 1969     // input : rax: dividend                         min_long
 1970     //         reg: divisor                          -1
 1971     //
 1972     // output: rax: quotient  (= rax idiv reg)       min_long
 1973     //         rdx: remainder (= rax irem reg)       0
 1974     //
 1975     //  Code sequnce:
 1976     //
 1977     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1978     //    7:   00 00 80
 1979     //    a:   48 39 d0                cmp    %rdx,%rax
 1980     //    d:   75 08                   jne    17 <normal>
 1981     //    f:   33 d2                   xor    %edx,%edx
 1982     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1983     //   15:   74 05                   je     1c <done>
 1984     // 0000000000000017 <normal>:
 1985     //   17:   48 99                   cqto
 1986     //   19:   48 f7 f9                idiv   $div
 1987     // 000000000000001c <done>:
 1988     MacroAssembler _masm(&cbuf);
 1989     Label normal;
 1990     Label done;
 1991 
 1992     // mov    $0x8000000000000000,%rdx
 1993     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1994 
 1995     // cmp    %rdx,%rax
 1996     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1997 
 1998     // jne    17 <normal>
 1999     __ jccb(Assembler::notEqual, normal);
 2000 
 2001     // xor    %edx,%edx
 2002     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2003 
 2004     // cmp    $0xffffffffffffffff,$div
 2005     __ cmpq($div$$Register, -1);
 2006 
 2007     // je     1e <done>
 2008     __ jccb(Assembler::equal, done);
 2009 
 2010     // <normal>
 2011     // cqto
 2012     __ bind(normal);
 2013     __ cdqq();
 2014 
 2015     // idivq (note: must be emitted by the user of this rule)
 2016     // <done>
 2017     __ idivq($div$$Register);
 2018     __ bind(done);
 2019   %}
 2020 
 2021   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2022   enc_class OpcSE(immI imm)
 2023   %{
 2024     // Emit primary opcode and set sign-extend bit
 2025     // Check for 8-bit immediate, and set sign extend bit in opcode
 2026     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2027       emit_opcode(cbuf, $primary | 0x02);
 2028     } else {
 2029       // 32-bit immediate
 2030       emit_opcode(cbuf, $primary);
 2031     }
 2032   %}
 2033 
 2034   enc_class OpcSErm(rRegI dst, immI imm)
 2035   %{
 2036     // OpcSEr/m
 2037     int dstenc = $dst$$reg;
 2038     if (dstenc >= 8) {
 2039       emit_opcode(cbuf, Assembler::REX_B);
 2040       dstenc -= 8;
 2041     }
 2042     // Emit primary opcode and set sign-extend bit
 2043     // Check for 8-bit immediate, and set sign extend bit in opcode
 2044     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2045       emit_opcode(cbuf, $primary | 0x02);
 2046     } else {
 2047       // 32-bit immediate
 2048       emit_opcode(cbuf, $primary);
 2049     }
 2050     // Emit r/m byte with secondary opcode, after primary opcode.
 2051     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2052   %}
 2053 
 2054   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2055   %{
 2056     // OpcSEr/m
 2057     int dstenc = $dst$$reg;
 2058     if (dstenc < 8) {
 2059       emit_opcode(cbuf, Assembler::REX_W);
 2060     } else {
 2061       emit_opcode(cbuf, Assembler::REX_WB);
 2062       dstenc -= 8;
 2063     }
 2064     // Emit primary opcode and set sign-extend bit
 2065     // Check for 8-bit immediate, and set sign extend bit in opcode
 2066     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2067       emit_opcode(cbuf, $primary | 0x02);
 2068     } else {
 2069       // 32-bit immediate
 2070       emit_opcode(cbuf, $primary);
 2071     }
 2072     // Emit r/m byte with secondary opcode, after primary opcode.
 2073     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2074   %}
 2075 
 2076   enc_class Con8or32(immI imm)
 2077   %{
 2078     // Check for 8-bit immediate, and set sign extend bit in opcode
 2079     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2080       $$$emit8$imm$$constant;
 2081     } else {
 2082       // 32-bit immediate
 2083       $$$emit32$imm$$constant;
 2084     }
 2085   %}
 2086 
 2087   enc_class opc2_reg(rRegI dst)
 2088   %{
 2089     // BSWAP
 2090     emit_cc(cbuf, $secondary, $dst$$reg);
 2091   %}
 2092 
 2093   enc_class opc3_reg(rRegI dst)
 2094   %{
 2095     // BSWAP
 2096     emit_cc(cbuf, $tertiary, $dst$$reg);
 2097   %}
 2098 
 2099   enc_class reg_opc(rRegI div)
 2100   %{
 2101     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2102     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2103   %}
 2104 
 2105   enc_class enc_cmov(cmpOp cop)
 2106   %{
 2107     // CMOV
 2108     $$$emit8$primary;
 2109     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2110   %}
 2111 
 2112   enc_class enc_PartialSubtypeCheck()
 2113   %{
 2114     Register Rrdi = as_Register(RDI_enc); // result register
 2115     Register Rrax = as_Register(RAX_enc); // super class
 2116     Register Rrcx = as_Register(RCX_enc); // killed
 2117     Register Rrsi = as_Register(RSI_enc); // sub class
 2118     Label miss;
 2119     const bool set_cond_codes = true;
 2120 
 2121     MacroAssembler _masm(&cbuf);
 2122     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2123                                      NULL, &miss,
 2124                                      /*set_cond_codes:*/ true);
 2125     if ($primary) {
 2126       __ xorptr(Rrdi, Rrdi);
 2127     }
 2128     __ bind(miss);
 2129   %}
 2130 
 2131   enc_class clear_avx %{
 2132     debug_only(int off0 = cbuf.insts_size());
 2133     if (generate_vzeroupper(Compile::current())) {
 2134       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2135       // Clear upper bits of YMM registers when current compiled code uses
 2136       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2137       MacroAssembler _masm(&cbuf);
 2138       __ vzeroupper();
 2139     }
 2140     debug_only(int off1 = cbuf.insts_size());
 2141     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2142   %}
 2143 
 2144   enc_class Java_To_Runtime(method meth) %{
 2145     // No relocation needed
 2146     MacroAssembler _masm(&cbuf);
 2147     __ mov64(r10, (int64_t) $meth$$method);
 2148     __ call(r10);
 2149     __ post_call_nop();
 2150   %}
 2151 
 2152   enc_class Java_Static_Call(method meth)
 2153   %{
 2154     // JAVA STATIC CALL
 2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2156     // determine who we intended to call.
 2157     MacroAssembler _masm(&cbuf);
 2158     cbuf.set_insts_mark();
 2159     $$$emit8$primary;
 2160 
 2161     if (!_method) {
 2162       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2163                      runtime_call_Relocation::spec(),
 2164                      RELOC_DISP32);
 2165     } else {
 2166       int method_index = resolved_method_index(cbuf);
 2167       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2168                                                   : static_call_Relocation::spec(method_index);
 2169       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2170                      rspec, RELOC_DISP32);
 2171       address mark = cbuf.insts_mark();
 2172       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 2173         // Calls of the same statically bound method can share
 2174         // a stub to the interpreter.
 2175         cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
 2176       } else {
 2177         // Emit stubs for static call.
 2178         address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2179         if (stub == NULL) {
 2180           ciEnv::current()->record_failure("CodeCache is full");
 2181           return;
 2182         }
 2183       }
 2184     }
 2185     _masm.clear_inst_mark();
 2186     __ post_call_nop();
 2187   %}
 2188 
 2189   enc_class Java_Dynamic_Call(method meth) %{
 2190     MacroAssembler _masm(&cbuf);
 2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2192     __ post_call_nop();
 2193   %}
 2194 
 2195   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2196   %{
 2197     // SAL, SAR, SHR
 2198     int dstenc = $dst$$reg;
 2199     if (dstenc >= 8) {
 2200       emit_opcode(cbuf, Assembler::REX_B);
 2201       dstenc -= 8;
 2202     }
 2203     $$$emit8$primary;
 2204     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2205     $$$emit8$shift$$constant;
 2206   %}
 2207 
 2208   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2209   %{
 2210     // SAL, SAR, SHR
 2211     int dstenc = $dst$$reg;
 2212     if (dstenc < 8) {
 2213       emit_opcode(cbuf, Assembler::REX_W);
 2214     } else {
 2215       emit_opcode(cbuf, Assembler::REX_WB);
 2216       dstenc -= 8;
 2217     }
 2218     $$$emit8$primary;
 2219     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2220     $$$emit8$shift$$constant;
 2221   %}
 2222 
 2223   enc_class load_immI(rRegI dst, immI src)
 2224   %{
 2225     int dstenc = $dst$$reg;
 2226     if (dstenc >= 8) {
 2227       emit_opcode(cbuf, Assembler::REX_B);
 2228       dstenc -= 8;
 2229     }
 2230     emit_opcode(cbuf, 0xB8 | dstenc);
 2231     $$$emit32$src$$constant;
 2232   %}
 2233 
 2234   enc_class load_immL(rRegL dst, immL src)
 2235   %{
 2236     int dstenc = $dst$$reg;
 2237     if (dstenc < 8) {
 2238       emit_opcode(cbuf, Assembler::REX_W);
 2239     } else {
 2240       emit_opcode(cbuf, Assembler::REX_WB);
 2241       dstenc -= 8;
 2242     }
 2243     emit_opcode(cbuf, 0xB8 | dstenc);
 2244     emit_d64(cbuf, $src$$constant);
 2245   %}
 2246 
 2247   enc_class load_immUL32(rRegL dst, immUL32 src)
 2248   %{
 2249     // same as load_immI, but this time we care about zeroes in the high word
 2250     int dstenc = $dst$$reg;
 2251     if (dstenc >= 8) {
 2252       emit_opcode(cbuf, Assembler::REX_B);
 2253       dstenc -= 8;
 2254     }
 2255     emit_opcode(cbuf, 0xB8 | dstenc);
 2256     $$$emit32$src$$constant;
 2257   %}
 2258 
 2259   enc_class load_immL32(rRegL dst, immL32 src)
 2260   %{
 2261     int dstenc = $dst$$reg;
 2262     if (dstenc < 8) {
 2263       emit_opcode(cbuf, Assembler::REX_W);
 2264     } else {
 2265       emit_opcode(cbuf, Assembler::REX_WB);
 2266       dstenc -= 8;
 2267     }
 2268     emit_opcode(cbuf, 0xC7);
 2269     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2270     $$$emit32$src$$constant;
 2271   %}
 2272 
 2273   enc_class load_immP31(rRegP dst, immP32 src)
 2274   %{
 2275     // same as load_immI, but this time we care about zeroes in the high word
 2276     int dstenc = $dst$$reg;
 2277     if (dstenc >= 8) {
 2278       emit_opcode(cbuf, Assembler::REX_B);
 2279       dstenc -= 8;
 2280     }
 2281     emit_opcode(cbuf, 0xB8 | dstenc);
 2282     $$$emit32$src$$constant;
 2283   %}
 2284 
 2285   enc_class load_immP(rRegP dst, immP src)
 2286   %{
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc < 8) {
 2289       emit_opcode(cbuf, Assembler::REX_W);
 2290     } else {
 2291       emit_opcode(cbuf, Assembler::REX_WB);
 2292       dstenc -= 8;
 2293     }
 2294     emit_opcode(cbuf, 0xB8 | dstenc);
 2295     // This next line should be generated from ADLC
 2296     if ($src->constant_reloc() != relocInfo::none) {
 2297       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2298     } else {
 2299       emit_d64(cbuf, $src$$constant);
 2300     }
 2301   %}
 2302 
 2303   enc_class Con32(immI src)
 2304   %{
 2305     // Output immediate
 2306     $$$emit32$src$$constant;
 2307   %}
 2308 
 2309   enc_class Con32F_as_bits(immF src)
 2310   %{
 2311     // Output Float immediate bits
 2312     jfloat jf = $src$$constant;
 2313     jint jf_as_bits = jint_cast(jf);
 2314     emit_d32(cbuf, jf_as_bits);
 2315   %}
 2316 
 2317   enc_class Con16(immI src)
 2318   %{
 2319     // Output immediate
 2320     $$$emit16$src$$constant;
 2321   %}
 2322 
 2323   // How is this different from Con32??? XXX
 2324   enc_class Con_d32(immI src)
 2325   %{
 2326     emit_d32(cbuf,$src$$constant);
 2327   %}
 2328 
 2329   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2330     // Output immediate memory reference
 2331     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2332     emit_d32(cbuf, 0x00);
 2333   %}
 2334 
 2335   enc_class lock_prefix()
 2336   %{
 2337     emit_opcode(cbuf, 0xF0); // lock
 2338   %}
 2339 
 2340   enc_class REX_mem(memory mem)
 2341   %{
 2342     if ($mem$$base >= 8) {
 2343       if ($mem$$index < 8) {
 2344         emit_opcode(cbuf, Assembler::REX_B);
 2345       } else {
 2346         emit_opcode(cbuf, Assembler::REX_XB);
 2347       }
 2348     } else {
 2349       if ($mem$$index >= 8) {
 2350         emit_opcode(cbuf, Assembler::REX_X);
 2351       }
 2352     }
 2353   %}
 2354 
 2355   enc_class REX_mem_wide(memory mem)
 2356   %{
 2357     if ($mem$$base >= 8) {
 2358       if ($mem$$index < 8) {
 2359         emit_opcode(cbuf, Assembler::REX_WB);
 2360       } else {
 2361         emit_opcode(cbuf, Assembler::REX_WXB);
 2362       }
 2363     } else {
 2364       if ($mem$$index < 8) {
 2365         emit_opcode(cbuf, Assembler::REX_W);
 2366       } else {
 2367         emit_opcode(cbuf, Assembler::REX_WX);
 2368       }
 2369     }
 2370   %}
 2371 
 2372   // for byte regs
 2373   enc_class REX_breg(rRegI reg)
 2374   %{
 2375     if ($reg$$reg >= 4) {
 2376       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2377     }
 2378   %}
 2379 
 2380   // for byte regs
 2381   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2382   %{
 2383     if ($dst$$reg < 8) {
 2384       if ($src$$reg >= 4) {
 2385         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2386       }
 2387     } else {
 2388       if ($src$$reg < 8) {
 2389         emit_opcode(cbuf, Assembler::REX_R);
 2390       } else {
 2391         emit_opcode(cbuf, Assembler::REX_RB);
 2392       }
 2393     }
 2394   %}
 2395 
 2396   // for byte regs
 2397   enc_class REX_breg_mem(rRegI reg, memory mem)
 2398   %{
 2399     if ($reg$$reg < 8) {
 2400       if ($mem$$base < 8) {
 2401         if ($mem$$index >= 8) {
 2402           emit_opcode(cbuf, Assembler::REX_X);
 2403         } else if ($reg$$reg >= 4) {
 2404           emit_opcode(cbuf, Assembler::REX);
 2405         }
 2406       } else {
 2407         if ($mem$$index < 8) {
 2408           emit_opcode(cbuf, Assembler::REX_B);
 2409         } else {
 2410           emit_opcode(cbuf, Assembler::REX_XB);
 2411         }
 2412       }
 2413     } else {
 2414       if ($mem$$base < 8) {
 2415         if ($mem$$index < 8) {
 2416           emit_opcode(cbuf, Assembler::REX_R);
 2417         } else {
 2418           emit_opcode(cbuf, Assembler::REX_RX);
 2419         }
 2420       } else {
 2421         if ($mem$$index < 8) {
 2422           emit_opcode(cbuf, Assembler::REX_RB);
 2423         } else {
 2424           emit_opcode(cbuf, Assembler::REX_RXB);
 2425         }
 2426       }
 2427     }
 2428   %}
 2429 
 2430   enc_class REX_reg(rRegI reg)
 2431   %{
 2432     if ($reg$$reg >= 8) {
 2433       emit_opcode(cbuf, Assembler::REX_B);
 2434     }
 2435   %}
 2436 
 2437   enc_class REX_reg_wide(rRegI reg)
 2438   %{
 2439     if ($reg$$reg < 8) {
 2440       emit_opcode(cbuf, Assembler::REX_W);
 2441     } else {
 2442       emit_opcode(cbuf, Assembler::REX_WB);
 2443     }
 2444   %}
 2445 
 2446   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2447   %{
 2448     if ($dst$$reg < 8) {
 2449       if ($src$$reg >= 8) {
 2450         emit_opcode(cbuf, Assembler::REX_B);
 2451       }
 2452     } else {
 2453       if ($src$$reg < 8) {
 2454         emit_opcode(cbuf, Assembler::REX_R);
 2455       } else {
 2456         emit_opcode(cbuf, Assembler::REX_RB);
 2457       }
 2458     }
 2459   %}
 2460 
 2461   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2462   %{
 2463     if ($dst$$reg < 8) {
 2464       if ($src$$reg < 8) {
 2465         emit_opcode(cbuf, Assembler::REX_W);
 2466       } else {
 2467         emit_opcode(cbuf, Assembler::REX_WB);
 2468       }
 2469     } else {
 2470       if ($src$$reg < 8) {
 2471         emit_opcode(cbuf, Assembler::REX_WR);
 2472       } else {
 2473         emit_opcode(cbuf, Assembler::REX_WRB);
 2474       }
 2475     }
 2476   %}
 2477 
 2478   enc_class REX_reg_mem(rRegI reg, memory mem)
 2479   %{
 2480     if ($reg$$reg < 8) {
 2481       if ($mem$$base < 8) {
 2482         if ($mem$$index >= 8) {
 2483           emit_opcode(cbuf, Assembler::REX_X);
 2484         }
 2485       } else {
 2486         if ($mem$$index < 8) {
 2487           emit_opcode(cbuf, Assembler::REX_B);
 2488         } else {
 2489           emit_opcode(cbuf, Assembler::REX_XB);
 2490         }
 2491       }
 2492     } else {
 2493       if ($mem$$base < 8) {
 2494         if ($mem$$index < 8) {
 2495           emit_opcode(cbuf, Assembler::REX_R);
 2496         } else {
 2497           emit_opcode(cbuf, Assembler::REX_RX);
 2498         }
 2499       } else {
 2500         if ($mem$$index < 8) {
 2501           emit_opcode(cbuf, Assembler::REX_RB);
 2502         } else {
 2503           emit_opcode(cbuf, Assembler::REX_RXB);
 2504         }
 2505       }
 2506     }
 2507   %}
 2508 
 2509   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2510   %{
 2511     if ($reg$$reg < 8) {
 2512       if ($mem$$base < 8) {
 2513         if ($mem$$index < 8) {
 2514           emit_opcode(cbuf, Assembler::REX_W);
 2515         } else {
 2516           emit_opcode(cbuf, Assembler::REX_WX);
 2517         }
 2518       } else {
 2519         if ($mem$$index < 8) {
 2520           emit_opcode(cbuf, Assembler::REX_WB);
 2521         } else {
 2522           emit_opcode(cbuf, Assembler::REX_WXB);
 2523         }
 2524       }
 2525     } else {
 2526       if ($mem$$base < 8) {
 2527         if ($mem$$index < 8) {
 2528           emit_opcode(cbuf, Assembler::REX_WR);
 2529         } else {
 2530           emit_opcode(cbuf, Assembler::REX_WRX);
 2531         }
 2532       } else {
 2533         if ($mem$$index < 8) {
 2534           emit_opcode(cbuf, Assembler::REX_WRB);
 2535         } else {
 2536           emit_opcode(cbuf, Assembler::REX_WRXB);
 2537         }
 2538       }
 2539     }
 2540   %}
 2541 
 2542   enc_class reg_mem(rRegI ereg, memory mem)
 2543   %{
 2544     // High registers handle in encode_RegMem
 2545     int reg = $ereg$$reg;
 2546     int base = $mem$$base;
 2547     int index = $mem$$index;
 2548     int scale = $mem$$scale;
 2549     int disp = $mem$$disp;
 2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2551 
 2552     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2553   %}
 2554 
 2555   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2556   %{
 2557     int rm_byte_opcode = $rm_opcode$$constant;
 2558 
 2559     // High registers handle in encode_RegMem
 2560     int base = $mem$$base;
 2561     int index = $mem$$index;
 2562     int scale = $mem$$scale;
 2563     int displace = $mem$$disp;
 2564 
 2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2566                                             // working with static
 2567                                             // globals
 2568     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2569                   disp_reloc);
 2570   %}
 2571 
 2572   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2573   %{
 2574     int reg_encoding = $dst$$reg;
 2575     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2576     int index        = 0x04;            // 0x04 indicates no index
 2577     int scale        = 0x00;            // 0x00 indicates no scale
 2578     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2579     relocInfo::relocType disp_reloc = relocInfo::none;
 2580     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2581                   disp_reloc);
 2582   %}
 2583 
 2584   enc_class neg_reg(rRegI dst)
 2585   %{
 2586     int dstenc = $dst$$reg;
 2587     if (dstenc >= 8) {
 2588       emit_opcode(cbuf, Assembler::REX_B);
 2589       dstenc -= 8;
 2590     }
 2591     // NEG $dst
 2592     emit_opcode(cbuf, 0xF7);
 2593     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2594   %}
 2595 
 2596   enc_class neg_reg_wide(rRegI dst)
 2597   %{
 2598     int dstenc = $dst$$reg;
 2599     if (dstenc < 8) {
 2600       emit_opcode(cbuf, Assembler::REX_W);
 2601     } else {
 2602       emit_opcode(cbuf, Assembler::REX_WB);
 2603       dstenc -= 8;
 2604     }
 2605     // NEG $dst
 2606     emit_opcode(cbuf, 0xF7);
 2607     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2608   %}
 2609 
 2610   enc_class setLT_reg(rRegI dst)
 2611   %{
 2612     int dstenc = $dst$$reg;
 2613     if (dstenc >= 8) {
 2614       emit_opcode(cbuf, Assembler::REX_B);
 2615       dstenc -= 8;
 2616     } else if (dstenc >= 4) {
 2617       emit_opcode(cbuf, Assembler::REX);
 2618     }
 2619     // SETLT $dst
 2620     emit_opcode(cbuf, 0x0F);
 2621     emit_opcode(cbuf, 0x9C);
 2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2623   %}
 2624 
 2625   enc_class setNZ_reg(rRegI dst)
 2626   %{
 2627     int dstenc = $dst$$reg;
 2628     if (dstenc >= 8) {
 2629       emit_opcode(cbuf, Assembler::REX_B);
 2630       dstenc -= 8;
 2631     } else if (dstenc >= 4) {
 2632       emit_opcode(cbuf, Assembler::REX);
 2633     }
 2634     // SETNZ $dst
 2635     emit_opcode(cbuf, 0x0F);
 2636     emit_opcode(cbuf, 0x95);
 2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2638   %}
 2639 
 2640 
 2641   // Compare the lonogs and set -1, 0, or 1 into dst
 2642   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2643   %{
 2644     int src1enc = $src1$$reg;
 2645     int src2enc = $src2$$reg;
 2646     int dstenc = $dst$$reg;
 2647 
 2648     // cmpq $src1, $src2
 2649     if (src1enc < 8) {
 2650       if (src2enc < 8) {
 2651         emit_opcode(cbuf, Assembler::REX_W);
 2652       } else {
 2653         emit_opcode(cbuf, Assembler::REX_WB);
 2654       }
 2655     } else {
 2656       if (src2enc < 8) {
 2657         emit_opcode(cbuf, Assembler::REX_WR);
 2658       } else {
 2659         emit_opcode(cbuf, Assembler::REX_WRB);
 2660       }
 2661     }
 2662     emit_opcode(cbuf, 0x3B);
 2663     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2664 
 2665     // movl $dst, -1
 2666     if (dstenc >= 8) {
 2667       emit_opcode(cbuf, Assembler::REX_B);
 2668     }
 2669     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2670     emit_d32(cbuf, -1);
 2671 
 2672     // jl,s done
 2673     emit_opcode(cbuf, 0x7C);
 2674     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2675 
 2676     // setne $dst
 2677     if (dstenc >= 4) {
 2678       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2679     }
 2680     emit_opcode(cbuf, 0x0F);
 2681     emit_opcode(cbuf, 0x95);
 2682     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2683 
 2684     // movzbl $dst, $dst
 2685     if (dstenc >= 4) {
 2686       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2687     }
 2688     emit_opcode(cbuf, 0x0F);
 2689     emit_opcode(cbuf, 0xB6);
 2690     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2691   %}
 2692 
 2693   enc_class Push_ResultXD(regD dst) %{
 2694     MacroAssembler _masm(&cbuf);
 2695     __ fstp_d(Address(rsp, 0));
 2696     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2697     __ addptr(rsp, 8);
 2698   %}
 2699 
 2700   enc_class Push_SrcXD(regD src) %{
 2701     MacroAssembler _masm(&cbuf);
 2702     __ subptr(rsp, 8);
 2703     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2704     __ fld_d(Address(rsp, 0));
 2705   %}
 2706 
 2707 
 2708   enc_class enc_rethrow()
 2709   %{
 2710     cbuf.set_insts_mark();
 2711     emit_opcode(cbuf, 0xE9); // jmp entry
 2712     emit_d32_reloc(cbuf,
 2713                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2714                    runtime_call_Relocation::spec(),
 2715                    RELOC_DISP32);
 2716   %}
 2717 
 2718 %}
 2719 
 2720 
 2721 
 2722 //----------FRAME--------------------------------------------------------------
 2723 // Definition of frame structure and management information.
 2724 //
 2725 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2726 //                             |   (to get allocators register number
 2727 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2728 //  r   CALLER     |        |
 2729 //  o     |        +--------+      pad to even-align allocators stack-slot
 2730 //  w     V        |  pad0  |        numbers; owned by CALLER
 2731 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2732 //  h     ^        |   in   |  5
 2733 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2734 //  |     |        |        |  3
 2735 //  |     |        +--------+
 2736 //  V     |        | old out|      Empty on Intel, window on Sparc
 2737 //        |    old |preserve|      Must be even aligned.
 2738 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2739 //        |        |   in   |  3   area for Intel ret address
 2740 //     Owned by    |preserve|      Empty on Sparc.
 2741 //       SELF      +--------+
 2742 //        |        |  pad2  |  2   pad to align old SP
 2743 //        |        +--------+  1
 2744 //        |        | locks  |  0
 2745 //        |        +--------+----> OptoReg::stack0(), even aligned
 2746 //        |        |  pad1  | 11   pad to align new SP
 2747 //        |        +--------+
 2748 //        |        |        | 10
 2749 //        |        | spills |  9   spills
 2750 //        V        |        |  8   (pad0 slot for callee)
 2751 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2752 //        ^        |  out   |  7
 2753 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2754 //     Owned by    +--------+
 2755 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2756 //        |    new |preserve|      Must be even-aligned.
 2757 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2758 //        |        |        |
 2759 //
 2760 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2761 //         known from SELF's arguments and the Java calling convention.
 2762 //         Region 6-7 is determined per call site.
 2763 // Note 2: If the calling convention leaves holes in the incoming argument
 2764 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2765 //         are owned by the CALLEE.  Holes should not be necessary in the
 2766 //         incoming area, as the Java calling convention is completely under
 2767 //         the control of the AD file.  Doubles can be sorted and packed to
 2768 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 2769 //         varargs C calling conventions.
 2770 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2771 //         even aligned with pad0 as needed.
 2772 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2773 //         region 6-11 is even aligned; it may be padded out more so that
 2774 //         the region from SP to FP meets the minimum stack alignment.
 2775 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2776 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2777 //         SP meets the minimum alignment.
 2778 
 2779 frame
 2780 %{
 2781   // These three registers define part of the calling convention
 2782   // between compiled code and the interpreter.
 2783   inline_cache_reg(RAX);                // Inline Cache Register
 2784 
 2785   // Optional: name the operand used by cisc-spilling to access
 2786   // [stack_pointer + offset]
 2787   cisc_spilling_operand_name(indOffset32);
 2788 
 2789   // Number of stack slots consumed by locking an object
 2790   sync_stack_slots(2);
 2791 
 2792   // Compiled code's Frame Pointer
 2793   frame_pointer(RSP);
 2794 
 2795   // Interpreter stores its frame pointer in a register which is
 2796   // stored to the stack by I2CAdaptors.
 2797   // I2CAdaptors convert from interpreted java to compiled java.
 2798   interpreter_frame_pointer(RBP);
 2799 
 2800   // Stack alignment requirement
 2801   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2802 
 2803   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2804   // for calls to C.  Supports the var-args backing area for register parms.
 2805   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2806 
 2807   // The after-PROLOG location of the return address.  Location of
 2808   // return address specifies a type (REG or STACK) and a number
 2809   // representing the register number (i.e. - use a register name) or
 2810   // stack slot.
 2811   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2812   // Otherwise, it is above the locks and verification slot and alignment word
 2813   return_addr(STACK - 2 +
 2814               align_up((Compile::current()->in_preserve_stack_slots() +
 2815                         Compile::current()->fixed_slots()),
 2816                        stack_alignment_in_slots()));
 2817 
 2818   // Location of compiled Java return values.  Same as C for now.
 2819   return_value
 2820   %{
 2821     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2822            "only return normal values");
 2823 
 2824     static const int lo[Op_RegL + 1] = {
 2825       0,
 2826       0,
 2827       RAX_num,  // Op_RegN
 2828       RAX_num,  // Op_RegI
 2829       RAX_num,  // Op_RegP
 2830       XMM0_num, // Op_RegF
 2831       XMM0_num, // Op_RegD
 2832       RAX_num   // Op_RegL
 2833     };
 2834     static const int hi[Op_RegL + 1] = {
 2835       0,
 2836       0,
 2837       OptoReg::Bad, // Op_RegN
 2838       OptoReg::Bad, // Op_RegI
 2839       RAX_H_num,    // Op_RegP
 2840       OptoReg::Bad, // Op_RegF
 2841       XMM0b_num,    // Op_RegD
 2842       RAX_H_num     // Op_RegL
 2843     };
 2844     // Excluded flags and vector registers.
 2845     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2846     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2847   %}
 2848 %}
 2849 
 2850 //----------ATTRIBUTES---------------------------------------------------------
 2851 //----------Operand Attributes-------------------------------------------------
 2852 op_attrib op_cost(0);        // Required cost attribute
 2853 
 2854 //----------Instruction Attributes---------------------------------------------
 2855 ins_attrib ins_cost(100);       // Required cost attribute
 2856 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2857 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2858                                 // a non-matching short branch variant
 2859                                 // of some long branch?
 2860 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2861                                 // be a power of 2) specifies the
 2862                                 // alignment that some part of the
 2863                                 // instruction (not necessarily the
 2864                                 // start) requires.  If > 1, a
 2865                                 // compute_padding() function must be
 2866                                 // provided for the instruction
 2867 
 2868 //----------OPERANDS-----------------------------------------------------------
 2869 // Operand definitions must precede instruction definitions for correct parsing
 2870 // in the ADLC because operands constitute user defined types which are used in
 2871 // instruction definitions.
 2872 
 2873 //----------Simple Operands----------------------------------------------------
 2874 // Immediate Operands
 2875 // Integer Immediate
 2876 operand immI()
 2877 %{
 2878   match(ConI);
 2879 
 2880   op_cost(10);
 2881   format %{ %}
 2882   interface(CONST_INTER);
 2883 %}
 2884 
 2885 // Constant for test vs zero
 2886 operand immI_0()
 2887 %{
 2888   predicate(n->get_int() == 0);
 2889   match(ConI);
 2890 
 2891   op_cost(0);
 2892   format %{ %}
 2893   interface(CONST_INTER);
 2894 %}
 2895 
 2896 // Constant for increment
 2897 operand immI_1()
 2898 %{
 2899   predicate(n->get_int() == 1);
 2900   match(ConI);
 2901 
 2902   op_cost(0);
 2903   format %{ %}
 2904   interface(CONST_INTER);
 2905 %}
 2906 
 2907 // Constant for decrement
 2908 operand immI_M1()
 2909 %{
 2910   predicate(n->get_int() == -1);
 2911   match(ConI);
 2912 
 2913   op_cost(0);
 2914   format %{ %}
 2915   interface(CONST_INTER);
 2916 %}
 2917 
 2918 operand immI_2()
 2919 %{
 2920   predicate(n->get_int() == 2);
 2921   match(ConI);
 2922 
 2923   op_cost(0);
 2924   format %{ %}
 2925   interface(CONST_INTER);
 2926 %}
 2927 
 2928 operand immI_4()
 2929 %{
 2930   predicate(n->get_int() == 4);
 2931   match(ConI);
 2932 
 2933   op_cost(0);
 2934   format %{ %}
 2935   interface(CONST_INTER);
 2936 %}
 2937 
 2938 operand immI_8()
 2939 %{
 2940   predicate(n->get_int() == 8);
 2941   match(ConI);
 2942 
 2943   op_cost(0);
 2944   format %{ %}
 2945   interface(CONST_INTER);
 2946 %}
 2947 
 2948 // Valid scale values for addressing modes
 2949 operand immI2()
 2950 %{
 2951   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2952   match(ConI);
 2953 
 2954   format %{ %}
 2955   interface(CONST_INTER);
 2956 %}
 2957 
 2958 operand immU7()
 2959 %{
 2960   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2961   match(ConI);
 2962 
 2963   op_cost(5);
 2964   format %{ %}
 2965   interface(CONST_INTER);
 2966 %}
 2967 
 2968 operand immI8()
 2969 %{
 2970   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2971   match(ConI);
 2972 
 2973   op_cost(5);
 2974   format %{ %}
 2975   interface(CONST_INTER);
 2976 %}
 2977 
 2978 operand immU8()
 2979 %{
 2980   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2981   match(ConI);
 2982 
 2983   op_cost(5);
 2984   format %{ %}
 2985   interface(CONST_INTER);
 2986 %}
 2987 
 2988 operand immI16()
 2989 %{
 2990   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2991   match(ConI);
 2992 
 2993   op_cost(10);
 2994   format %{ %}
 2995   interface(CONST_INTER);
 2996 %}
 2997 
 2998 // Int Immediate non-negative
 2999 operand immU31()
 3000 %{
 3001   predicate(n->get_int() >= 0);
 3002   match(ConI);
 3003 
 3004   op_cost(0);
 3005   format %{ %}
 3006   interface(CONST_INTER);
 3007 %}
 3008 
 3009 // Constant for long shifts
 3010 operand immI_32()
 3011 %{
 3012   predicate( n->get_int() == 32 );
 3013   match(ConI);
 3014 
 3015   op_cost(0);
 3016   format %{ %}
 3017   interface(CONST_INTER);
 3018 %}
 3019 
 3020 // Constant for long shifts
 3021 operand immI_64()
 3022 %{
 3023   predicate( n->get_int() == 64 );
 3024   match(ConI);
 3025 
 3026   op_cost(0);
 3027   format %{ %}
 3028   interface(CONST_INTER);
 3029 %}
 3030 
 3031 // Pointer Immediate
 3032 operand immP()
 3033 %{
 3034   match(ConP);
 3035 
 3036   op_cost(10);
 3037   format %{ %}
 3038   interface(CONST_INTER);
 3039 %}
 3040 
 3041 // NULL Pointer Immediate
 3042 operand immP0()
 3043 %{
 3044   predicate(n->get_ptr() == 0);
 3045   match(ConP);
 3046 
 3047   op_cost(5);
 3048   format %{ %}
 3049   interface(CONST_INTER);
 3050 %}
 3051 
 3052 // Pointer Immediate
 3053 operand immN() %{
 3054   match(ConN);
 3055 
 3056   op_cost(10);
 3057   format %{ %}
 3058   interface(CONST_INTER);
 3059 %}
 3060 
 3061 operand immNKlass() %{
 3062   match(ConNKlass);
 3063 
 3064   op_cost(10);
 3065   format %{ %}
 3066   interface(CONST_INTER);
 3067 %}
 3068 
 3069 // NULL Pointer Immediate
 3070 operand immN0() %{
 3071   predicate(n->get_narrowcon() == 0);
 3072   match(ConN);
 3073 
 3074   op_cost(5);
 3075   format %{ %}
 3076   interface(CONST_INTER);
 3077 %}
 3078 
 3079 operand immP31()
 3080 %{
 3081   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3082             && (n->get_ptr() >> 31) == 0);
 3083   match(ConP);
 3084 
 3085   op_cost(5);
 3086   format %{ %}
 3087   interface(CONST_INTER);
 3088 %}
 3089 
 3090 
 3091 // Long Immediate
 3092 operand immL()
 3093 %{
 3094   match(ConL);
 3095 
 3096   op_cost(20);
 3097   format %{ %}
 3098   interface(CONST_INTER);
 3099 %}
 3100 
 3101 // Long Immediate 8-bit
 3102 operand immL8()
 3103 %{
 3104   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3105   match(ConL);
 3106 
 3107   op_cost(5);
 3108   format %{ %}
 3109   interface(CONST_INTER);
 3110 %}
 3111 
 3112 // Long Immediate 32-bit unsigned
 3113 operand immUL32()
 3114 %{
 3115   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3116   match(ConL);
 3117 
 3118   op_cost(10);
 3119   format %{ %}
 3120   interface(CONST_INTER);
 3121 %}
 3122 
 3123 // Long Immediate 32-bit signed
 3124 operand immL32()
 3125 %{
 3126   predicate(n->get_long() == (int) (n->get_long()));
 3127   match(ConL);
 3128 
 3129   op_cost(15);
 3130   format %{ %}
 3131   interface(CONST_INTER);
 3132 %}
 3133 
 3134 operand immL_Pow2()
 3135 %{
 3136   predicate(is_power_of_2((julong)n->get_long()));
 3137   match(ConL);
 3138 
 3139   op_cost(15);
 3140   format %{ %}
 3141   interface(CONST_INTER);
 3142 %}
 3143 
 3144 operand immL_NotPow2()
 3145 %{
 3146   predicate(is_power_of_2((julong)~n->get_long()));
 3147   match(ConL);
 3148 
 3149   op_cost(15);
 3150   format %{ %}
 3151   interface(CONST_INTER);
 3152 %}
 3153 
 3154 // Long Immediate zero
 3155 operand immL0()
 3156 %{
 3157   predicate(n->get_long() == 0L);
 3158   match(ConL);
 3159 
 3160   op_cost(10);
 3161   format %{ %}
 3162   interface(CONST_INTER);
 3163 %}
 3164 
 3165 // Constant for increment
 3166 operand immL1()
 3167 %{
 3168   predicate(n->get_long() == 1);
 3169   match(ConL);
 3170 
 3171   format %{ %}
 3172   interface(CONST_INTER);
 3173 %}
 3174 
 3175 // Constant for decrement
 3176 operand immL_M1()
 3177 %{
 3178   predicate(n->get_long() == -1);
 3179   match(ConL);
 3180 
 3181   format %{ %}
 3182   interface(CONST_INTER);
 3183 %}
 3184 
 3185 // Long Immediate: the value 10
 3186 operand immL10()
 3187 %{
 3188   predicate(n->get_long() == 10);
 3189   match(ConL);
 3190 
 3191   format %{ %}
 3192   interface(CONST_INTER);
 3193 %}
 3194 
 3195 // Long immediate from 0 to 127.
 3196 // Used for a shorter form of long mul by 10.
 3197 operand immL_127()
 3198 %{
 3199   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3200   match(ConL);
 3201 
 3202   op_cost(10);
 3203   format %{ %}
 3204   interface(CONST_INTER);
 3205 %}
 3206 
 3207 // Long Immediate: low 32-bit mask
 3208 operand immL_32bits()
 3209 %{
 3210   predicate(n->get_long() == 0xFFFFFFFFL);
 3211   match(ConL);
 3212   op_cost(20);
 3213 
 3214   format %{ %}
 3215   interface(CONST_INTER);
 3216 %}
 3217 
 3218 // Int Immediate: 2^n-1, positive
 3219 operand immI_Pow2M1()
 3220 %{
 3221   predicate((n->get_int() > 0)
 3222             && is_power_of_2(n->get_int() + 1));
 3223   match(ConI);
 3224 
 3225   op_cost(20);
 3226   format %{ %}
 3227   interface(CONST_INTER);
 3228 %}
 3229 
 3230 // Float Immediate zero
 3231 operand immF0()
 3232 %{
 3233   predicate(jint_cast(n->getf()) == 0);
 3234   match(ConF);
 3235 
 3236   op_cost(5);
 3237   format %{ %}
 3238   interface(CONST_INTER);
 3239 %}
 3240 
 3241 // Float Immediate
 3242 operand immF()
 3243 %{
 3244   match(ConF);
 3245 
 3246   op_cost(15);
 3247   format %{ %}
 3248   interface(CONST_INTER);
 3249 %}
 3250 
 3251 // Double Immediate zero
 3252 operand immD0()
 3253 %{
 3254   predicate(jlong_cast(n->getd()) == 0);
 3255   match(ConD);
 3256 
 3257   op_cost(5);
 3258   format %{ %}
 3259   interface(CONST_INTER);
 3260 %}
 3261 
 3262 // Double Immediate
 3263 operand immD()
 3264 %{
 3265   match(ConD);
 3266 
 3267   op_cost(15);
 3268   format %{ %}
 3269   interface(CONST_INTER);
 3270 %}
 3271 
 3272 // Immediates for special shifts (sign extend)
 3273 
 3274 // Constants for increment
 3275 operand immI_16()
 3276 %{
 3277   predicate(n->get_int() == 16);
 3278   match(ConI);
 3279 
 3280   format %{ %}
 3281   interface(CONST_INTER);
 3282 %}
 3283 
 3284 operand immI_24()
 3285 %{
 3286   predicate(n->get_int() == 24);
 3287   match(ConI);
 3288 
 3289   format %{ %}
 3290   interface(CONST_INTER);
 3291 %}
 3292 
 3293 // Constant for byte-wide masking
 3294 operand immI_255()
 3295 %{
 3296   predicate(n->get_int() == 255);
 3297   match(ConI);
 3298 
 3299   format %{ %}
 3300   interface(CONST_INTER);
 3301 %}
 3302 
 3303 // Constant for short-wide masking
 3304 operand immI_65535()
 3305 %{
 3306   predicate(n->get_int() == 65535);
 3307   match(ConI);
 3308 
 3309   format %{ %}
 3310   interface(CONST_INTER);
 3311 %}
 3312 
 3313 // Constant for byte-wide masking
 3314 operand immL_255()
 3315 %{
 3316   predicate(n->get_long() == 255);
 3317   match(ConL);
 3318 
 3319   format %{ %}
 3320   interface(CONST_INTER);
 3321 %}
 3322 
 3323 // Constant for short-wide masking
 3324 operand immL_65535()
 3325 %{
 3326   predicate(n->get_long() == 65535);
 3327   match(ConL);
 3328 
 3329   format %{ %}
 3330   interface(CONST_INTER);
 3331 %}
 3332 
 3333 operand kReg()
 3334 %{
 3335   constraint(ALLOC_IN_RC(vectmask_reg));
 3336   match(RegVectMask);
 3337   format %{%}
 3338   interface(REG_INTER);
 3339 %}
 3340 
 3341 operand kReg_K1()
 3342 %{
 3343   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3344   match(RegVectMask);
 3345   format %{%}
 3346   interface(REG_INTER);
 3347 %}
 3348 
 3349 operand kReg_K2()
 3350 %{
 3351   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3352   match(RegVectMask);
 3353   format %{%}
 3354   interface(REG_INTER);
 3355 %}
 3356 
 3357 // Special Registers
 3358 operand kReg_K3()
 3359 %{
 3360   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3361   match(RegVectMask);
 3362   format %{%}
 3363   interface(REG_INTER);
 3364 %}
 3365 
 3366 operand kReg_K4()
 3367 %{
 3368   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3369   match(RegVectMask);
 3370   format %{%}
 3371   interface(REG_INTER);
 3372 %}
 3373 
 3374 operand kReg_K5()
 3375 %{
 3376   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3377   match(RegVectMask);
 3378   format %{%}
 3379   interface(REG_INTER);
 3380 %}
 3381 
 3382 operand kReg_K6()
 3383 %{
 3384   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3385   match(RegVectMask);
 3386   format %{%}
 3387   interface(REG_INTER);
 3388 %}
 3389 
 3390 // Special Registers
 3391 operand kReg_K7()
 3392 %{
 3393   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3394   match(RegVectMask);
 3395   format %{%}
 3396   interface(REG_INTER);
 3397 %}
 3398 
 3399 // Register Operands
 3400 // Integer Register
 3401 operand rRegI()
 3402 %{
 3403   constraint(ALLOC_IN_RC(int_reg));
 3404   match(RegI);
 3405 
 3406   match(rax_RegI);
 3407   match(rbx_RegI);
 3408   match(rcx_RegI);
 3409   match(rdx_RegI);
 3410   match(rdi_RegI);
 3411 
 3412   format %{ %}
 3413   interface(REG_INTER);
 3414 %}
 3415 
 3416 // Special Registers
 3417 operand rax_RegI()
 3418 %{
 3419   constraint(ALLOC_IN_RC(int_rax_reg));
 3420   match(RegI);
 3421   match(rRegI);
 3422 
 3423   format %{ "RAX" %}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand rbx_RegI()
 3429 %{
 3430   constraint(ALLOC_IN_RC(int_rbx_reg));
 3431   match(RegI);
 3432   match(rRegI);
 3433 
 3434   format %{ "RBX" %}
 3435   interface(REG_INTER);
 3436 %}
 3437 
 3438 operand rcx_RegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_rcx_reg));
 3441   match(RegI);
 3442   match(rRegI);
 3443 
 3444   format %{ "RCX" %}
 3445   interface(REG_INTER);
 3446 %}
 3447 
 3448 operand rdx_RegI()
 3449 %{
 3450   constraint(ALLOC_IN_RC(int_rdx_reg));
 3451   match(RegI);
 3452   match(rRegI);
 3453 
 3454   format %{ "RDX" %}
 3455   interface(REG_INTER);
 3456 %}
 3457 
 3458 operand rdi_RegI()
 3459 %{
 3460   constraint(ALLOC_IN_RC(int_rdi_reg));
 3461   match(RegI);
 3462   match(rRegI);
 3463 
 3464   format %{ "RDI" %}
 3465   interface(REG_INTER);
 3466 %}
 3467 
 3468 operand no_rax_rdx_RegI()
 3469 %{
 3470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3471   match(RegI);
 3472   match(rbx_RegI);
 3473   match(rcx_RegI);
 3474   match(rdi_RegI);
 3475 
 3476   format %{ %}
 3477   interface(REG_INTER);
 3478 %}
 3479 
 3480 operand no_rbp_r13_RegI()
 3481 %{
 3482   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3483   match(RegI);
 3484   match(rRegI);
 3485   match(rax_RegI);
 3486   match(rbx_RegI);
 3487   match(rcx_RegI);
 3488   match(rdx_RegI);
 3489   match(rdi_RegI);
 3490 
 3491   format %{ %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 // Pointer Register
 3496 operand any_RegP()
 3497 %{
 3498   constraint(ALLOC_IN_RC(any_reg));
 3499   match(RegP);
 3500   match(rax_RegP);
 3501   match(rbx_RegP);
 3502   match(rdi_RegP);
 3503   match(rsi_RegP);
 3504   match(rbp_RegP);
 3505   match(r15_RegP);
 3506   match(rRegP);
 3507 
 3508   format %{ %}
 3509   interface(REG_INTER);
 3510 %}
 3511 
 3512 operand rRegP()
 3513 %{
 3514   constraint(ALLOC_IN_RC(ptr_reg));
 3515   match(RegP);
 3516   match(rax_RegP);
 3517   match(rbx_RegP);
 3518   match(rdi_RegP);
 3519   match(rsi_RegP);
 3520   match(rbp_RegP);  // See Q&A below about
 3521   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3522 
 3523   format %{ %}
 3524   interface(REG_INTER);
 3525 %}
 3526 
 3527 operand rRegN() %{
 3528   constraint(ALLOC_IN_RC(int_reg));
 3529   match(RegN);
 3530 
 3531   format %{ %}
 3532   interface(REG_INTER);
 3533 %}
 3534 
 3535 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3536 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3537 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3538 // The output of an instruction is controlled by the allocator, which respects
 3539 // register class masks, not match rules.  Unless an instruction mentions
 3540 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3541 // by the allocator as an input.
 3542 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3543 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3544 // result, RBP is not included in the output of the instruction either.
 3545 
 3546 operand no_rax_RegP()
 3547 %{
 3548   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3549   match(RegP);
 3550   match(rbx_RegP);
 3551   match(rsi_RegP);
 3552   match(rdi_RegP);
 3553 
 3554   format %{ %}
 3555   interface(REG_INTER);
 3556 %}
 3557 
 3558 // This operand is not allowed to use RBP even if
 3559 // RBP is not used to hold the frame pointer.
 3560 operand no_rbp_RegP()
 3561 %{
 3562   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3563   match(RegP);
 3564   match(rbx_RegP);
 3565   match(rsi_RegP);
 3566   match(rdi_RegP);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 operand no_rax_rbx_RegP()
 3573 %{
 3574   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3575   match(RegP);
 3576   match(rsi_RegP);
 3577   match(rdi_RegP);
 3578 
 3579   format %{ %}
 3580   interface(REG_INTER);
 3581 %}
 3582 
 3583 // Special Registers
 3584 // Return a pointer value
 3585 operand rax_RegP()
 3586 %{
 3587   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3588   match(RegP);
 3589   match(rRegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // Special Registers
 3596 // Return a compressed pointer value
 3597 operand rax_RegN()
 3598 %{
 3599   constraint(ALLOC_IN_RC(int_rax_reg));
 3600   match(RegN);
 3601   match(rRegN);
 3602 
 3603   format %{ %}
 3604   interface(REG_INTER);
 3605 %}
 3606 
 3607 // Used in AtomicAdd
 3608 operand rbx_RegP()
 3609 %{
 3610   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3611   match(RegP);
 3612   match(rRegP);
 3613 
 3614   format %{ %}
 3615   interface(REG_INTER);
 3616 %}
 3617 
 3618 operand rsi_RegP()
 3619 %{
 3620   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3621   match(RegP);
 3622   match(rRegP);
 3623 
 3624   format %{ %}
 3625   interface(REG_INTER);
 3626 %}
 3627 
 3628 operand rbp_RegP()
 3629 %{
 3630   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3631   match(RegP);
 3632   match(rRegP);
 3633 
 3634   format %{ %}
 3635   interface(REG_INTER);
 3636 %}
 3637 
 3638 // Used in rep stosq
 3639 operand rdi_RegP()
 3640 %{
 3641   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3642   match(RegP);
 3643   match(rRegP);
 3644 
 3645   format %{ %}
 3646   interface(REG_INTER);
 3647 %}
 3648 
 3649 operand r15_RegP()
 3650 %{
 3651   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3652   match(RegP);
 3653   match(rRegP);
 3654 
 3655   format %{ %}
 3656   interface(REG_INTER);
 3657 %}
 3658 
 3659 operand rRegL()
 3660 %{
 3661   constraint(ALLOC_IN_RC(long_reg));
 3662   match(RegL);
 3663   match(rax_RegL);
 3664   match(rdx_RegL);
 3665 
 3666   format %{ %}
 3667   interface(REG_INTER);
 3668 %}
 3669 
 3670 // Special Registers
 3671 operand no_rax_rdx_RegL()
 3672 %{
 3673   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3674   match(RegL);
 3675   match(rRegL);
 3676 
 3677   format %{ %}
 3678   interface(REG_INTER);
 3679 %}
 3680 
 3681 operand rax_RegL()
 3682 %{
 3683   constraint(ALLOC_IN_RC(long_rax_reg));
 3684   match(RegL);
 3685   match(rRegL);
 3686 
 3687   format %{ "RAX" %}
 3688   interface(REG_INTER);
 3689 %}
 3690 
 3691 operand rcx_RegL()
 3692 %{
 3693   constraint(ALLOC_IN_RC(long_rcx_reg));
 3694   match(RegL);
 3695   match(rRegL);
 3696 
 3697   format %{ %}
 3698   interface(REG_INTER);
 3699 %}
 3700 
 3701 operand rdx_RegL()
 3702 %{
 3703   constraint(ALLOC_IN_RC(long_rdx_reg));
 3704   match(RegL);
 3705   match(rRegL);
 3706 
 3707   format %{ %}
 3708   interface(REG_INTER);
 3709 %}
 3710 
 3711 operand no_rbp_r13_RegL()
 3712 %{
 3713   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3714   match(RegL);
 3715   match(rRegL);
 3716   match(rax_RegL);
 3717   match(rcx_RegL);
 3718   match(rdx_RegL);
 3719 
 3720   format %{ %}
 3721   interface(REG_INTER);
 3722 %}
 3723 
 3724 // Flags register, used as output of compare instructions
 3725 operand rFlagsReg()
 3726 %{
 3727   constraint(ALLOC_IN_RC(int_flags));
 3728   match(RegFlags);
 3729 
 3730   format %{ "RFLAGS" %}
 3731   interface(REG_INTER);
 3732 %}
 3733 
 3734 // Flags register, used as output of FLOATING POINT compare instructions
 3735 operand rFlagsRegU()
 3736 %{
 3737   constraint(ALLOC_IN_RC(int_flags));
 3738   match(RegFlags);
 3739 
 3740   format %{ "RFLAGS_U" %}
 3741   interface(REG_INTER);
 3742 %}
 3743 
 3744 operand rFlagsRegUCF() %{
 3745   constraint(ALLOC_IN_RC(int_flags));
 3746   match(RegFlags);
 3747   predicate(false);
 3748 
 3749   format %{ "RFLAGS_U_CF" %}
 3750   interface(REG_INTER);
 3751 %}
 3752 
 3753 // Float register operands
 3754 operand regF() %{
 3755    constraint(ALLOC_IN_RC(float_reg));
 3756    match(RegF);
 3757 
 3758    format %{ %}
 3759    interface(REG_INTER);
 3760 %}
 3761 
 3762 // Float register operands
 3763 operand legRegF() %{
 3764    constraint(ALLOC_IN_RC(float_reg_legacy));
 3765    match(RegF);
 3766 
 3767    format %{ %}
 3768    interface(REG_INTER);
 3769 %}
 3770 
 3771 // Float register operands
 3772 operand vlRegF() %{
 3773    constraint(ALLOC_IN_RC(float_reg_vl));
 3774    match(RegF);
 3775 
 3776    format %{ %}
 3777    interface(REG_INTER);
 3778 %}
 3779 
 3780 // Double register operands
 3781 operand regD() %{
 3782    constraint(ALLOC_IN_RC(double_reg));
 3783    match(RegD);
 3784 
 3785    format %{ %}
 3786    interface(REG_INTER);
 3787 %}
 3788 
 3789 // Double register operands
 3790 operand legRegD() %{
 3791    constraint(ALLOC_IN_RC(double_reg_legacy));
 3792    match(RegD);
 3793 
 3794    format %{ %}
 3795    interface(REG_INTER);
 3796 %}
 3797 
 3798 // Double register operands
 3799 operand vlRegD() %{
 3800    constraint(ALLOC_IN_RC(double_reg_vl));
 3801    match(RegD);
 3802 
 3803    format %{ %}
 3804    interface(REG_INTER);
 3805 %}
 3806 
 3807 //----------Memory Operands----------------------------------------------------
 3808 // Direct Memory Operand
 3809 // operand direct(immP addr)
 3810 // %{
 3811 //   match(addr);
 3812 
 3813 //   format %{ "[$addr]" %}
 3814 //   interface(MEMORY_INTER) %{
 3815 //     base(0xFFFFFFFF);
 3816 //     index(0x4);
 3817 //     scale(0x0);
 3818 //     disp($addr);
 3819 //   %}
 3820 // %}
 3821 
 3822 // Indirect Memory Operand
 3823 operand indirect(any_RegP reg)
 3824 %{
 3825   constraint(ALLOC_IN_RC(ptr_reg));
 3826   match(reg);
 3827 
 3828   format %{ "[$reg]" %}
 3829   interface(MEMORY_INTER) %{
 3830     base($reg);
 3831     index(0x4);
 3832     scale(0x0);
 3833     disp(0x0);
 3834   %}
 3835 %}
 3836 
 3837 // Indirect Memory Plus Short Offset Operand
 3838 operand indOffset8(any_RegP reg, immL8 off)
 3839 %{
 3840   constraint(ALLOC_IN_RC(ptr_reg));
 3841   match(AddP reg off);
 3842 
 3843   format %{ "[$reg + $off (8-bit)]" %}
 3844   interface(MEMORY_INTER) %{
 3845     base($reg);
 3846     index(0x4);
 3847     scale(0x0);
 3848     disp($off);
 3849   %}
 3850 %}
 3851 
 3852 // Indirect Memory Plus Long Offset Operand
 3853 operand indOffset32(any_RegP reg, immL32 off)
 3854 %{
 3855   constraint(ALLOC_IN_RC(ptr_reg));
 3856   match(AddP reg off);
 3857 
 3858   format %{ "[$reg + $off (32-bit)]" %}
 3859   interface(MEMORY_INTER) %{
 3860     base($reg);
 3861     index(0x4);
 3862     scale(0x0);
 3863     disp($off);
 3864   %}
 3865 %}
 3866 
 3867 // Indirect Memory Plus Index Register Plus Offset Operand
 3868 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3869 %{
 3870   constraint(ALLOC_IN_RC(ptr_reg));
 3871   match(AddP (AddP reg lreg) off);
 3872 
 3873   op_cost(10);
 3874   format %{"[$reg + $off + $lreg]" %}
 3875   interface(MEMORY_INTER) %{
 3876     base($reg);
 3877     index($lreg);
 3878     scale(0x0);
 3879     disp($off);
 3880   %}
 3881 %}
 3882 
 3883 // Indirect Memory Plus Index Register Plus Offset Operand
 3884 operand indIndex(any_RegP reg, rRegL lreg)
 3885 %{
 3886   constraint(ALLOC_IN_RC(ptr_reg));
 3887   match(AddP reg lreg);
 3888 
 3889   op_cost(10);
 3890   format %{"[$reg + $lreg]" %}
 3891   interface(MEMORY_INTER) %{
 3892     base($reg);
 3893     index($lreg);
 3894     scale(0x0);
 3895     disp(0x0);
 3896   %}
 3897 %}
 3898 
 3899 // Indirect Memory Times Scale Plus Index Register
 3900 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3901 %{
 3902   constraint(ALLOC_IN_RC(ptr_reg));
 3903   match(AddP reg (LShiftL lreg scale));
 3904 
 3905   op_cost(10);
 3906   format %{"[$reg + $lreg << $scale]" %}
 3907   interface(MEMORY_INTER) %{
 3908     base($reg);
 3909     index($lreg);
 3910     scale($scale);
 3911     disp(0x0);
 3912   %}
 3913 %}
 3914 
 3915 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3916 %{
 3917   constraint(ALLOC_IN_RC(ptr_reg));
 3918   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3919   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3920 
 3921   op_cost(10);
 3922   format %{"[$reg + pos $idx << $scale]" %}
 3923   interface(MEMORY_INTER) %{
 3924     base($reg);
 3925     index($idx);
 3926     scale($scale);
 3927     disp(0x0);
 3928   %}
 3929 %}
 3930 
 3931 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3932 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3933 %{
 3934   constraint(ALLOC_IN_RC(ptr_reg));
 3935   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3936 
 3937   op_cost(10);
 3938   format %{"[$reg + $off + $lreg << $scale]" %}
 3939   interface(MEMORY_INTER) %{
 3940     base($reg);
 3941     index($lreg);
 3942     scale($scale);
 3943     disp($off);
 3944   %}
 3945 %}
 3946 
 3947 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3948 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3949 %{
 3950   constraint(ALLOC_IN_RC(ptr_reg));
 3951   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3952   match(AddP (AddP reg (ConvI2L idx)) off);
 3953 
 3954   op_cost(10);
 3955   format %{"[$reg + $off + $idx]" %}
 3956   interface(MEMORY_INTER) %{
 3957     base($reg);
 3958     index($idx);
 3959     scale(0x0);
 3960     disp($off);
 3961   %}
 3962 %}
 3963 
 3964 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3965 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3966 %{
 3967   constraint(ALLOC_IN_RC(ptr_reg));
 3968   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3969   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3970 
 3971   op_cost(10);
 3972   format %{"[$reg + $off + $idx << $scale]" %}
 3973   interface(MEMORY_INTER) %{
 3974     base($reg);
 3975     index($idx);
 3976     scale($scale);
 3977     disp($off);
 3978   %}
 3979 %}
 3980 
 3981 // Indirect Narrow Oop Plus Offset Operand
 3982 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3983 // we can't free r12 even with CompressedOops::base() == NULL.
 3984 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3985   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3986   constraint(ALLOC_IN_RC(ptr_reg));
 3987   match(AddP (DecodeN reg) off);
 3988 
 3989   op_cost(10);
 3990   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3991   interface(MEMORY_INTER) %{
 3992     base(0xc); // R12
 3993     index($reg);
 3994     scale(0x3);
 3995     disp($off);
 3996   %}
 3997 %}
 3998 
 3999 // Indirect Memory Operand
 4000 operand indirectNarrow(rRegN reg)
 4001 %{
 4002   predicate(CompressedOops::shift() == 0);
 4003   constraint(ALLOC_IN_RC(ptr_reg));
 4004   match(DecodeN reg);
 4005 
 4006   format %{ "[$reg]" %}
 4007   interface(MEMORY_INTER) %{
 4008     base($reg);
 4009     index(0x4);
 4010     scale(0x0);
 4011     disp(0x0);
 4012   %}
 4013 %}
 4014 
 4015 // Indirect Memory Plus Short Offset Operand
 4016 operand indOffset8Narrow(rRegN reg, immL8 off)
 4017 %{
 4018   predicate(CompressedOops::shift() == 0);
 4019   constraint(ALLOC_IN_RC(ptr_reg));
 4020   match(AddP (DecodeN reg) off);
 4021 
 4022   format %{ "[$reg + $off (8-bit)]" %}
 4023   interface(MEMORY_INTER) %{
 4024     base($reg);
 4025     index(0x4);
 4026     scale(0x0);
 4027     disp($off);
 4028   %}
 4029 %}
 4030 
 4031 // Indirect Memory Plus Long Offset Operand
 4032 operand indOffset32Narrow(rRegN reg, immL32 off)
 4033 %{
 4034   predicate(CompressedOops::shift() == 0);
 4035   constraint(ALLOC_IN_RC(ptr_reg));
 4036   match(AddP (DecodeN reg) off);
 4037 
 4038   format %{ "[$reg + $off (32-bit)]" %}
 4039   interface(MEMORY_INTER) %{
 4040     base($reg);
 4041     index(0x4);
 4042     scale(0x0);
 4043     disp($off);
 4044   %}
 4045 %}
 4046 
 4047 // Indirect Memory Plus Index Register Plus Offset Operand
 4048 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4049 %{
 4050   predicate(CompressedOops::shift() == 0);
 4051   constraint(ALLOC_IN_RC(ptr_reg));
 4052   match(AddP (AddP (DecodeN reg) lreg) off);
 4053 
 4054   op_cost(10);
 4055   format %{"[$reg + $off + $lreg]" %}
 4056   interface(MEMORY_INTER) %{
 4057     base($reg);
 4058     index($lreg);
 4059     scale(0x0);
 4060     disp($off);
 4061   %}
 4062 %}
 4063 
 4064 // Indirect Memory Plus Index Register Plus Offset Operand
 4065 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4066 %{
 4067   predicate(CompressedOops::shift() == 0);
 4068   constraint(ALLOC_IN_RC(ptr_reg));
 4069   match(AddP (DecodeN reg) lreg);
 4070 
 4071   op_cost(10);
 4072   format %{"[$reg + $lreg]" %}
 4073   interface(MEMORY_INTER) %{
 4074     base($reg);
 4075     index($lreg);
 4076     scale(0x0);
 4077     disp(0x0);
 4078   %}
 4079 %}
 4080 
 4081 // Indirect Memory Times Scale Plus Index Register
 4082 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4083 %{
 4084   predicate(CompressedOops::shift() == 0);
 4085   constraint(ALLOC_IN_RC(ptr_reg));
 4086   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4087 
 4088   op_cost(10);
 4089   format %{"[$reg + $lreg << $scale]" %}
 4090   interface(MEMORY_INTER) %{
 4091     base($reg);
 4092     index($lreg);
 4093     scale($scale);
 4094     disp(0x0);
 4095   %}
 4096 %}
 4097 
 4098 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4099 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4100 %{
 4101   predicate(CompressedOops::shift() == 0);
 4102   constraint(ALLOC_IN_RC(ptr_reg));
 4103   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4104 
 4105   op_cost(10);
 4106   format %{"[$reg + $off + $lreg << $scale]" %}
 4107   interface(MEMORY_INTER) %{
 4108     base($reg);
 4109     index($lreg);
 4110     scale($scale);
 4111     disp($off);
 4112   %}
 4113 %}
 4114 
 4115 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4116 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4117 %{
 4118   constraint(ALLOC_IN_RC(ptr_reg));
 4119   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4120   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4121 
 4122   op_cost(10);
 4123   format %{"[$reg + $off + $idx]" %}
 4124   interface(MEMORY_INTER) %{
 4125     base($reg);
 4126     index($idx);
 4127     scale(0x0);
 4128     disp($off);
 4129   %}
 4130 %}
 4131 
 4132 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4133 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4134 %{
 4135   constraint(ALLOC_IN_RC(ptr_reg));
 4136   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4137   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4138 
 4139   op_cost(10);
 4140   format %{"[$reg + $off + $idx << $scale]" %}
 4141   interface(MEMORY_INTER) %{
 4142     base($reg);
 4143     index($idx);
 4144     scale($scale);
 4145     disp($off);
 4146   %}
 4147 %}
 4148 
 4149 //----------Special Memory Operands--------------------------------------------
 4150 // Stack Slot Operand - This operand is used for loading and storing temporary
 4151 //                      values on the stack where a match requires a value to
 4152 //                      flow through memory.
 4153 operand stackSlotP(sRegP reg)
 4154 %{
 4155   constraint(ALLOC_IN_RC(stack_slots));
 4156   // No match rule because this operand is only generated in matching
 4157 
 4158   format %{ "[$reg]" %}
 4159   interface(MEMORY_INTER) %{
 4160     base(0x4);   // RSP
 4161     index(0x4);  // No Index
 4162     scale(0x0);  // No Scale
 4163     disp($reg);  // Stack Offset
 4164   %}
 4165 %}
 4166 
 4167 operand stackSlotI(sRegI reg)
 4168 %{
 4169   constraint(ALLOC_IN_RC(stack_slots));
 4170   // No match rule because this operand is only generated in matching
 4171 
 4172   format %{ "[$reg]" %}
 4173   interface(MEMORY_INTER) %{
 4174     base(0x4);   // RSP
 4175     index(0x4);  // No Index
 4176     scale(0x0);  // No Scale
 4177     disp($reg);  // Stack Offset
 4178   %}
 4179 %}
 4180 
 4181 operand stackSlotF(sRegF reg)
 4182 %{
 4183   constraint(ALLOC_IN_RC(stack_slots));
 4184   // No match rule because this operand is only generated in matching
 4185 
 4186   format %{ "[$reg]" %}
 4187   interface(MEMORY_INTER) %{
 4188     base(0x4);   // RSP
 4189     index(0x4);  // No Index
 4190     scale(0x0);  // No Scale
 4191     disp($reg);  // Stack Offset
 4192   %}
 4193 %}
 4194 
 4195 operand stackSlotD(sRegD reg)
 4196 %{
 4197   constraint(ALLOC_IN_RC(stack_slots));
 4198   // No match rule because this operand is only generated in matching
 4199 
 4200   format %{ "[$reg]" %}
 4201   interface(MEMORY_INTER) %{
 4202     base(0x4);   // RSP
 4203     index(0x4);  // No Index
 4204     scale(0x0);  // No Scale
 4205     disp($reg);  // Stack Offset
 4206   %}
 4207 %}
 4208 operand stackSlotL(sRegL reg)
 4209 %{
 4210   constraint(ALLOC_IN_RC(stack_slots));
 4211   // No match rule because this operand is only generated in matching
 4212 
 4213   format %{ "[$reg]" %}
 4214   interface(MEMORY_INTER) %{
 4215     base(0x4);   // RSP
 4216     index(0x4);  // No Index
 4217     scale(0x0);  // No Scale
 4218     disp($reg);  // Stack Offset
 4219   %}
 4220 %}
 4221 
 4222 //----------Conditional Branch Operands----------------------------------------
 4223 // Comparison Op  - This is the operation of the comparison, and is limited to
 4224 //                  the following set of codes:
 4225 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4226 //
 4227 // Other attributes of the comparison, such as unsignedness, are specified
 4228 // by the comparison instruction that sets a condition code flags register.
 4229 // That result is represented by a flags operand whose subtype is appropriate
 4230 // to the unsignedness (etc.) of the comparison.
 4231 //
 4232 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4233 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4234 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4235 
 4236 // Comparison Code
 4237 operand cmpOp()
 4238 %{
 4239   match(Bool);
 4240 
 4241   format %{ "" %}
 4242   interface(COND_INTER) %{
 4243     equal(0x4, "e");
 4244     not_equal(0x5, "ne");
 4245     less(0xC, "l");
 4246     greater_equal(0xD, "ge");
 4247     less_equal(0xE, "le");
 4248     greater(0xF, "g");
 4249     overflow(0x0, "o");
 4250     no_overflow(0x1, "no");
 4251   %}
 4252 %}
 4253 
 4254 // Comparison Code, unsigned compare.  Used by FP also, with
 4255 // C2 (unordered) turned into GT or LT already.  The other bits
 4256 // C0 and C3 are turned into Carry & Zero flags.
 4257 operand cmpOpU()
 4258 %{
 4259   match(Bool);
 4260 
 4261   format %{ "" %}
 4262   interface(COND_INTER) %{
 4263     equal(0x4, "e");
 4264     not_equal(0x5, "ne");
 4265     less(0x2, "b");
 4266     greater_equal(0x3, "ae");
 4267     less_equal(0x6, "be");
 4268     greater(0x7, "a");
 4269     overflow(0x0, "o");
 4270     no_overflow(0x1, "no");
 4271   %}
 4272 %}
 4273 
 4274 
 4275 // Floating comparisons that don't require any fixup for the unordered case,
 4276 // If both inputs of the comparison are the same, ZF is always set so we
 4277 // don't need to use cmpOpUCF2 for eq/ne
 4278 operand cmpOpUCF() %{
 4279   match(Bool);
 4280   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4281             n->as_Bool()->_test._test == BoolTest::ge ||
 4282             n->as_Bool()->_test._test == BoolTest::le ||
 4283             n->as_Bool()->_test._test == BoolTest::gt ||
 4284             n->in(1)->in(1) == n->in(1)->in(2));
 4285   format %{ "" %}
 4286   interface(COND_INTER) %{
 4287     equal(0xb, "np");
 4288     not_equal(0xa, "p");
 4289     less(0x2, "b");
 4290     greater_equal(0x3, "ae");
 4291     less_equal(0x6, "be");
 4292     greater(0x7, "a");
 4293     overflow(0x0, "o");
 4294     no_overflow(0x1, "no");
 4295   %}
 4296 %}
 4297 
 4298 
 4299 // Floating comparisons that can be fixed up with extra conditional jumps
 4300 operand cmpOpUCF2() %{
 4301   match(Bool);
 4302   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 4303              n->as_Bool()->_test._test == BoolTest::eq) &&
 4304             n->in(1)->in(1) != n->in(1)->in(2));
 4305   format %{ "" %}
 4306   interface(COND_INTER) %{
 4307     equal(0x4, "e");
 4308     not_equal(0x5, "ne");
 4309     less(0x2, "b");
 4310     greater_equal(0x3, "ae");
 4311     less_equal(0x6, "be");
 4312     greater(0x7, "a");
 4313     overflow(0x0, "o");
 4314     no_overflow(0x1, "no");
 4315   %}
 4316 %}
 4317 
 4318 //----------OPERAND CLASSES----------------------------------------------------
 4319 // Operand Classes are groups of operands that are used as to simplify
 4320 // instruction definitions by not requiring the AD writer to specify separate
 4321 // instructions for every form of operand when the instruction accepts
 4322 // multiple operand types with the same basic encoding and format.  The classic
 4323 // case of this is memory operands.
 4324 
 4325 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4326                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4327                indCompressedOopOffset,
 4328                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4329                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4330                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4331 
 4332 //----------PIPELINE-----------------------------------------------------------
 4333 // Rules which define the behavior of the target architectures pipeline.
 4334 pipeline %{
 4335 
 4336 //----------ATTRIBUTES---------------------------------------------------------
 4337 attributes %{
 4338   variable_size_instructions;        // Fixed size instructions
 4339   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4340   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4341   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4342   instruction_fetch_units = 1;       // of 16 bytes
 4343 
 4344   // List of nop instructions
 4345   nops( MachNop );
 4346 %}
 4347 
 4348 //----------RESOURCES----------------------------------------------------------
 4349 // Resources are the functional units available to the machine
 4350 
 4351 // Generic P2/P3 pipeline
 4352 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4353 // 3 instructions decoded per cycle.
 4354 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4355 // 3 ALU op, only ALU0 handles mul instructions.
 4356 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4357            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4358            BR, FPU,
 4359            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4360 
 4361 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4362 // Pipeline Description specifies the stages in the machine's pipeline
 4363 
 4364 // Generic P2/P3 pipeline
 4365 pipe_desc(S0, S1, S2, S3, S4, S5);
 4366 
 4367 //----------PIPELINE CLASSES---------------------------------------------------
 4368 // Pipeline Classes describe the stages in which input and output are
 4369 // referenced by the hardware pipeline.
 4370 
 4371 // Naming convention: ialu or fpu
 4372 // Then: _reg
 4373 // Then: _reg if there is a 2nd register
 4374 // Then: _long if it's a pair of instructions implementing a long
 4375 // Then: _fat if it requires the big decoder
 4376 //   Or: _mem if it requires the big decoder and a memory unit.
 4377 
 4378 // Integer ALU reg operation
 4379 pipe_class ialu_reg(rRegI dst)
 4380 %{
 4381     single_instruction;
 4382     dst    : S4(write);
 4383     dst    : S3(read);
 4384     DECODE : S0;        // any decoder
 4385     ALU    : S3;        // any alu
 4386 %}
 4387 
 4388 // Long ALU reg operation
 4389 pipe_class ialu_reg_long(rRegL dst)
 4390 %{
 4391     instruction_count(2);
 4392     dst    : S4(write);
 4393     dst    : S3(read);
 4394     DECODE : S0(2);     // any 2 decoders
 4395     ALU    : S3(2);     // both alus
 4396 %}
 4397 
 4398 // Integer ALU reg operation using big decoder
 4399 pipe_class ialu_reg_fat(rRegI dst)
 4400 %{
 4401     single_instruction;
 4402     dst    : S4(write);
 4403     dst    : S3(read);
 4404     D0     : S0;        // big decoder only
 4405     ALU    : S3;        // any alu
 4406 %}
 4407 
 4408 // Integer ALU reg-reg operation
 4409 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4410 %{
 4411     single_instruction;
 4412     dst    : S4(write);
 4413     src    : S3(read);
 4414     DECODE : S0;        // any decoder
 4415     ALU    : S3;        // any alu
 4416 %}
 4417 
 4418 // Integer ALU reg-reg operation
 4419 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4420 %{
 4421     single_instruction;
 4422     dst    : S4(write);
 4423     src    : S3(read);
 4424     D0     : S0;        // big decoder only
 4425     ALU    : S3;        // any alu
 4426 %}
 4427 
 4428 // Integer ALU reg-mem operation
 4429 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4430 %{
 4431     single_instruction;
 4432     dst    : S5(write);
 4433     mem    : S3(read);
 4434     D0     : S0;        // big decoder only
 4435     ALU    : S4;        // any alu
 4436     MEM    : S3;        // any mem
 4437 %}
 4438 
 4439 // Integer mem operation (prefetch)
 4440 pipe_class ialu_mem(memory mem)
 4441 %{
 4442     single_instruction;
 4443     mem    : S3(read);
 4444     D0     : S0;        // big decoder only
 4445     MEM    : S3;        // any mem
 4446 %}
 4447 
 4448 // Integer Store to Memory
 4449 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4450 %{
 4451     single_instruction;
 4452     mem    : S3(read);
 4453     src    : S5(read);
 4454     D0     : S0;        // big decoder only
 4455     ALU    : S4;        // any alu
 4456     MEM    : S3;
 4457 %}
 4458 
 4459 // // Long Store to Memory
 4460 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4461 // %{
 4462 //     instruction_count(2);
 4463 //     mem    : S3(read);
 4464 //     src    : S5(read);
 4465 //     D0     : S0(2);          // big decoder only; twice
 4466 //     ALU    : S4(2);     // any 2 alus
 4467 //     MEM    : S3(2);  // Both mems
 4468 // %}
 4469 
 4470 // Integer Store to Memory
 4471 pipe_class ialu_mem_imm(memory mem)
 4472 %{
 4473     single_instruction;
 4474     mem    : S3(read);
 4475     D0     : S0;        // big decoder only
 4476     ALU    : S4;        // any alu
 4477     MEM    : S3;
 4478 %}
 4479 
 4480 // Integer ALU0 reg-reg operation
 4481 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4482 %{
 4483     single_instruction;
 4484     dst    : S4(write);
 4485     src    : S3(read);
 4486     D0     : S0;        // Big decoder only
 4487     ALU0   : S3;        // only alu0
 4488 %}
 4489 
 4490 // Integer ALU0 reg-mem operation
 4491 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4492 %{
 4493     single_instruction;
 4494     dst    : S5(write);
 4495     mem    : S3(read);
 4496     D0     : S0;        // big decoder only
 4497     ALU0   : S4;        // ALU0 only
 4498     MEM    : S3;        // any mem
 4499 %}
 4500 
 4501 // Integer ALU reg-reg operation
 4502 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4503 %{
 4504     single_instruction;
 4505     cr     : S4(write);
 4506     src1   : S3(read);
 4507     src2   : S3(read);
 4508     DECODE : S0;        // any decoder
 4509     ALU    : S3;        // any alu
 4510 %}
 4511 
 4512 // Integer ALU reg-imm operation
 4513 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4514 %{
 4515     single_instruction;
 4516     cr     : S4(write);
 4517     src1   : S3(read);
 4518     DECODE : S0;        // any decoder
 4519     ALU    : S3;        // any alu
 4520 %}
 4521 
 4522 // Integer ALU reg-mem operation
 4523 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4524 %{
 4525     single_instruction;
 4526     cr     : S4(write);
 4527     src1   : S3(read);
 4528     src2   : S3(read);
 4529     D0     : S0;        // big decoder only
 4530     ALU    : S4;        // any alu
 4531     MEM    : S3;
 4532 %}
 4533 
 4534 // Conditional move reg-reg
 4535 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4536 %{
 4537     instruction_count(4);
 4538     y      : S4(read);
 4539     q      : S3(read);
 4540     p      : S3(read);
 4541     DECODE : S0(4);     // any decoder
 4542 %}
 4543 
 4544 // Conditional move reg-reg
 4545 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4546 %{
 4547     single_instruction;
 4548     dst    : S4(write);
 4549     src    : S3(read);
 4550     cr     : S3(read);
 4551     DECODE : S0;        // any decoder
 4552 %}
 4553 
 4554 // Conditional move reg-mem
 4555 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4556 %{
 4557     single_instruction;
 4558     dst    : S4(write);
 4559     src    : S3(read);
 4560     cr     : S3(read);
 4561     DECODE : S0;        // any decoder
 4562     MEM    : S3;
 4563 %}
 4564 
 4565 // Conditional move reg-reg long
 4566 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4567 %{
 4568     single_instruction;
 4569     dst    : S4(write);
 4570     src    : S3(read);
 4571     cr     : S3(read);
 4572     DECODE : S0(2);     // any 2 decoders
 4573 %}
 4574 
 4575 // XXX
 4576 // // Conditional move double reg-reg
 4577 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4578 // %{
 4579 //     single_instruction;
 4580 //     dst    : S4(write);
 4581 //     src    : S3(read);
 4582 //     cr     : S3(read);
 4583 //     DECODE : S0;     // any decoder
 4584 // %}
 4585 
 4586 // Float reg-reg operation
 4587 pipe_class fpu_reg(regD dst)
 4588 %{
 4589     instruction_count(2);
 4590     dst    : S3(read);
 4591     DECODE : S0(2);     // any 2 decoders
 4592     FPU    : S3;
 4593 %}
 4594 
 4595 // Float reg-reg operation
 4596 pipe_class fpu_reg_reg(regD dst, regD src)
 4597 %{
 4598     instruction_count(2);
 4599     dst    : S4(write);
 4600     src    : S3(read);
 4601     DECODE : S0(2);     // any 2 decoders
 4602     FPU    : S3;
 4603 %}
 4604 
 4605 // Float reg-reg operation
 4606 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4607 %{
 4608     instruction_count(3);
 4609     dst    : S4(write);
 4610     src1   : S3(read);
 4611     src2   : S3(read);
 4612     DECODE : S0(3);     // any 3 decoders
 4613     FPU    : S3(2);
 4614 %}
 4615 
 4616 // Float reg-reg operation
 4617 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4618 %{
 4619     instruction_count(4);
 4620     dst    : S4(write);
 4621     src1   : S3(read);
 4622     src2   : S3(read);
 4623     src3   : S3(read);
 4624     DECODE : S0(4);     // any 3 decoders
 4625     FPU    : S3(2);
 4626 %}
 4627 
 4628 // Float reg-reg operation
 4629 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4630 %{
 4631     instruction_count(4);
 4632     dst    : S4(write);
 4633     src1   : S3(read);
 4634     src2   : S3(read);
 4635     src3   : S3(read);
 4636     DECODE : S1(3);     // any 3 decoders
 4637     D0     : S0;        // Big decoder only
 4638     FPU    : S3(2);
 4639     MEM    : S3;
 4640 %}
 4641 
 4642 // Float reg-mem operation
 4643 pipe_class fpu_reg_mem(regD dst, memory mem)
 4644 %{
 4645     instruction_count(2);
 4646     dst    : S5(write);
 4647     mem    : S3(read);
 4648     D0     : S0;        // big decoder only
 4649     DECODE : S1;        // any decoder for FPU POP
 4650     FPU    : S4;
 4651     MEM    : S3;        // any mem
 4652 %}
 4653 
 4654 // Float reg-mem operation
 4655 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4656 %{
 4657     instruction_count(3);
 4658     dst    : S5(write);
 4659     src1   : S3(read);
 4660     mem    : S3(read);
 4661     D0     : S0;        // big decoder only
 4662     DECODE : S1(2);     // any decoder for FPU POP
 4663     FPU    : S4;
 4664     MEM    : S3;        // any mem
 4665 %}
 4666 
 4667 // Float mem-reg operation
 4668 pipe_class fpu_mem_reg(memory mem, regD src)
 4669 %{
 4670     instruction_count(2);
 4671     src    : S5(read);
 4672     mem    : S3(read);
 4673     DECODE : S0;        // any decoder for FPU PUSH
 4674     D0     : S1;        // big decoder only
 4675     FPU    : S4;
 4676     MEM    : S3;        // any mem
 4677 %}
 4678 
 4679 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4680 %{
 4681     instruction_count(3);
 4682     src1   : S3(read);
 4683     src2   : S3(read);
 4684     mem    : S3(read);
 4685     DECODE : S0(2);     // any decoder for FPU PUSH
 4686     D0     : S1;        // big decoder only
 4687     FPU    : S4;
 4688     MEM    : S3;        // any mem
 4689 %}
 4690 
 4691 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4692 %{
 4693     instruction_count(3);
 4694     src1   : S3(read);
 4695     src2   : S3(read);
 4696     mem    : S4(read);
 4697     DECODE : S0;        // any decoder for FPU PUSH
 4698     D0     : S0(2);     // big decoder only
 4699     FPU    : S4;
 4700     MEM    : S3(2);     // any mem
 4701 %}
 4702 
 4703 pipe_class fpu_mem_mem(memory dst, memory src1)
 4704 %{
 4705     instruction_count(2);
 4706     src1   : S3(read);
 4707     dst    : S4(read);
 4708     D0     : S0(2);     // big decoder only
 4709     MEM    : S3(2);     // any mem
 4710 %}
 4711 
 4712 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4713 %{
 4714     instruction_count(3);
 4715     src1   : S3(read);
 4716     src2   : S3(read);
 4717     dst    : S4(read);
 4718     D0     : S0(3);     // big decoder only
 4719     FPU    : S4;
 4720     MEM    : S3(3);     // any mem
 4721 %}
 4722 
 4723 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4724 %{
 4725     instruction_count(3);
 4726     src1   : S4(read);
 4727     mem    : S4(read);
 4728     DECODE : S0;        // any decoder for FPU PUSH
 4729     D0     : S0(2);     // big decoder only
 4730     FPU    : S4;
 4731     MEM    : S3(2);     // any mem
 4732 %}
 4733 
 4734 // Float load constant
 4735 pipe_class fpu_reg_con(regD dst)
 4736 %{
 4737     instruction_count(2);
 4738     dst    : S5(write);
 4739     D0     : S0;        // big decoder only for the load
 4740     DECODE : S1;        // any decoder for FPU POP
 4741     FPU    : S4;
 4742     MEM    : S3;        // any mem
 4743 %}
 4744 
 4745 // Float load constant
 4746 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4747 %{
 4748     instruction_count(3);
 4749     dst    : S5(write);
 4750     src    : S3(read);
 4751     D0     : S0;        // big decoder only for the load
 4752     DECODE : S1(2);     // any decoder for FPU POP
 4753     FPU    : S4;
 4754     MEM    : S3;        // any mem
 4755 %}
 4756 
 4757 // UnConditional branch
 4758 pipe_class pipe_jmp(label labl)
 4759 %{
 4760     single_instruction;
 4761     BR   : S3;
 4762 %}
 4763 
 4764 // Conditional branch
 4765 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4766 %{
 4767     single_instruction;
 4768     cr    : S1(read);
 4769     BR    : S3;
 4770 %}
 4771 
 4772 // Allocation idiom
 4773 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4774 %{
 4775     instruction_count(1); force_serialization;
 4776     fixed_latency(6);
 4777     heap_ptr : S3(read);
 4778     DECODE   : S0(3);
 4779     D0       : S2;
 4780     MEM      : S3;
 4781     ALU      : S3(2);
 4782     dst      : S5(write);
 4783     BR       : S5;
 4784 %}
 4785 
 4786 // Generic big/slow expanded idiom
 4787 pipe_class pipe_slow()
 4788 %{
 4789     instruction_count(10); multiple_bundles; force_serialization;
 4790     fixed_latency(100);
 4791     D0  : S0(2);
 4792     MEM : S3(2);
 4793 %}
 4794 
 4795 // The real do-nothing guy
 4796 pipe_class empty()
 4797 %{
 4798     instruction_count(0);
 4799 %}
 4800 
 4801 // Define the class for the Nop node
 4802 define
 4803 %{
 4804    MachNop = empty;
 4805 %}
 4806 
 4807 %}
 4808 
 4809 //----------INSTRUCTIONS-------------------------------------------------------
 4810 //
 4811 // match      -- States which machine-independent subtree may be replaced
 4812 //               by this instruction.
 4813 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4814 //               selection to identify a minimum cost tree of machine
 4815 //               instructions that matches a tree of machine-independent
 4816 //               instructions.
 4817 // format     -- A string providing the disassembly for this instruction.
 4818 //               The value of an instruction's operand may be inserted
 4819 //               by referring to it with a '$' prefix.
 4820 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4821 //               to within an encode class as $primary, $secondary, and $tertiary
 4822 //               rrspectively.  The primary opcode is commonly used to
 4823 //               indicate the type of machine instruction, while secondary
 4824 //               and tertiary are often used for prefix options or addressing
 4825 //               modes.
 4826 // ins_encode -- A list of encode classes with parameters. The encode class
 4827 //               name must have been defined in an 'enc_class' specification
 4828 //               in the encode section of the architecture description.
 4829 
 4830 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4831 // Load Float
 4832 instruct MoveF2VL(vlRegF dst, regF src) %{
 4833   match(Set dst src);
 4834   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4835   ins_encode %{
 4836     ShouldNotReachHere();
 4837   %}
 4838   ins_pipe( fpu_reg_reg );
 4839 %}
 4840 
 4841 // Load Float
 4842 instruct MoveF2LEG(legRegF dst, regF src) %{
 4843   match(Set dst src);
 4844   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4845   ins_encode %{
 4846     ShouldNotReachHere();
 4847   %}
 4848   ins_pipe( fpu_reg_reg );
 4849 %}
 4850 
 4851 // Load Float
 4852 instruct MoveVL2F(regF dst, vlRegF src) %{
 4853   match(Set dst src);
 4854   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4855   ins_encode %{
 4856     ShouldNotReachHere();
 4857   %}
 4858   ins_pipe( fpu_reg_reg );
 4859 %}
 4860 
 4861 // Load Float
 4862 instruct MoveLEG2F(regF dst, legRegF src) %{
 4863   match(Set dst src);
 4864   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4865   ins_encode %{
 4866     ShouldNotReachHere();
 4867   %}
 4868   ins_pipe( fpu_reg_reg );
 4869 %}
 4870 
 4871 // Load Double
 4872 instruct MoveD2VL(vlRegD dst, regD src) %{
 4873   match(Set dst src);
 4874   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4875   ins_encode %{
 4876     ShouldNotReachHere();
 4877   %}
 4878   ins_pipe( fpu_reg_reg );
 4879 %}
 4880 
 4881 // Load Double
 4882 instruct MoveD2LEG(legRegD dst, regD src) %{
 4883   match(Set dst src);
 4884   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4885   ins_encode %{
 4886     ShouldNotReachHere();
 4887   %}
 4888   ins_pipe( fpu_reg_reg );
 4889 %}
 4890 
 4891 // Load Double
 4892 instruct MoveVL2D(regD dst, vlRegD src) %{
 4893   match(Set dst src);
 4894   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4895   ins_encode %{
 4896     ShouldNotReachHere();
 4897   %}
 4898   ins_pipe( fpu_reg_reg );
 4899 %}
 4900 
 4901 // Load Double
 4902 instruct MoveLEG2D(regD dst, legRegD src) %{
 4903   match(Set dst src);
 4904   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4905   ins_encode %{
 4906     ShouldNotReachHere();
 4907   %}
 4908   ins_pipe( fpu_reg_reg );
 4909 %}
 4910 
 4911 //----------Load/Store/Move Instructions---------------------------------------
 4912 //----------Load Instructions--------------------------------------------------
 4913 
 4914 // Load Byte (8 bit signed)
 4915 instruct loadB(rRegI dst, memory mem)
 4916 %{
 4917   match(Set dst (LoadB mem));
 4918 
 4919   ins_cost(125);
 4920   format %{ "movsbl  $dst, $mem\t# byte" %}
 4921 
 4922   ins_encode %{
 4923     __ movsbl($dst$$Register, $mem$$Address);
 4924   %}
 4925 
 4926   ins_pipe(ialu_reg_mem);
 4927 %}
 4928 
 4929 // Load Byte (8 bit signed) into Long Register
 4930 instruct loadB2L(rRegL dst, memory mem)
 4931 %{
 4932   match(Set dst (ConvI2L (LoadB mem)));
 4933 
 4934   ins_cost(125);
 4935   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4936 
 4937   ins_encode %{
 4938     __ movsbq($dst$$Register, $mem$$Address);
 4939   %}
 4940 
 4941   ins_pipe(ialu_reg_mem);
 4942 %}
 4943 
 4944 // Load Unsigned Byte (8 bit UNsigned)
 4945 instruct loadUB(rRegI dst, memory mem)
 4946 %{
 4947   match(Set dst (LoadUB mem));
 4948 
 4949   ins_cost(125);
 4950   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4951 
 4952   ins_encode %{
 4953     __ movzbl($dst$$Register, $mem$$Address);
 4954   %}
 4955 
 4956   ins_pipe(ialu_reg_mem);
 4957 %}
 4958 
 4959 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4960 instruct loadUB2L(rRegL dst, memory mem)
 4961 %{
 4962   match(Set dst (ConvI2L (LoadUB mem)));
 4963 
 4964   ins_cost(125);
 4965   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4966 
 4967   ins_encode %{
 4968     __ movzbq($dst$$Register, $mem$$Address);
 4969   %}
 4970 
 4971   ins_pipe(ialu_reg_mem);
 4972 %}
 4973 
 4974 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4975 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4976   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4977   effect(KILL cr);
 4978 
 4979   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4980             "andl    $dst, right_n_bits($mask, 8)" %}
 4981   ins_encode %{
 4982     Register Rdst = $dst$$Register;
 4983     __ movzbq(Rdst, $mem$$Address);
 4984     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4985   %}
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 // Load Short (16 bit signed)
 4990 instruct loadS(rRegI dst, memory mem)
 4991 %{
 4992   match(Set dst (LoadS mem));
 4993 
 4994   ins_cost(125);
 4995   format %{ "movswl $dst, $mem\t# short" %}
 4996 
 4997   ins_encode %{
 4998     __ movswl($dst$$Register, $mem$$Address);
 4999   %}
 5000 
 5001   ins_pipe(ialu_reg_mem);
 5002 %}
 5003 
 5004 // Load Short (16 bit signed) to Byte (8 bit signed)
 5005 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5006   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5007 
 5008   ins_cost(125);
 5009   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5010   ins_encode %{
 5011     __ movsbl($dst$$Register, $mem$$Address);
 5012   %}
 5013   ins_pipe(ialu_reg_mem);
 5014 %}
 5015 
 5016 // Load Short (16 bit signed) into Long Register
 5017 instruct loadS2L(rRegL dst, memory mem)
 5018 %{
 5019   match(Set dst (ConvI2L (LoadS mem)));
 5020 
 5021   ins_cost(125);
 5022   format %{ "movswq $dst, $mem\t# short -> long" %}
 5023 
 5024   ins_encode %{
 5025     __ movswq($dst$$Register, $mem$$Address);
 5026   %}
 5027 
 5028   ins_pipe(ialu_reg_mem);
 5029 %}
 5030 
 5031 // Load Unsigned Short/Char (16 bit UNsigned)
 5032 instruct loadUS(rRegI dst, memory mem)
 5033 %{
 5034   match(Set dst (LoadUS mem));
 5035 
 5036   ins_cost(125);
 5037   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5038 
 5039   ins_encode %{
 5040     __ movzwl($dst$$Register, $mem$$Address);
 5041   %}
 5042 
 5043   ins_pipe(ialu_reg_mem);
 5044 %}
 5045 
 5046 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5047 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5048   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5049 
 5050   ins_cost(125);
 5051   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5052   ins_encode %{
 5053     __ movsbl($dst$$Register, $mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_reg_mem);
 5056 %}
 5057 
 5058 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5059 instruct loadUS2L(rRegL dst, memory mem)
 5060 %{
 5061   match(Set dst (ConvI2L (LoadUS mem)));
 5062 
 5063   ins_cost(125);
 5064   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5065 
 5066   ins_encode %{
 5067     __ movzwq($dst$$Register, $mem$$Address);
 5068   %}
 5069 
 5070   ins_pipe(ialu_reg_mem);
 5071 %}
 5072 
 5073 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5074 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5075   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5076 
 5077   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5078   ins_encode %{
 5079     __ movzbq($dst$$Register, $mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_reg_mem);
 5082 %}
 5083 
 5084 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5085 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5086   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5087   effect(KILL cr);
 5088 
 5089   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5090             "andl    $dst, right_n_bits($mask, 16)" %}
 5091   ins_encode %{
 5092     Register Rdst = $dst$$Register;
 5093     __ movzwq(Rdst, $mem$$Address);
 5094     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5095   %}
 5096   ins_pipe(ialu_reg_mem);
 5097 %}
 5098 
 5099 // Load Integer
 5100 instruct loadI(rRegI dst, memory mem)
 5101 %{
 5102   match(Set dst (LoadI mem));
 5103 
 5104   ins_cost(125);
 5105   format %{ "movl    $dst, $mem\t# int" %}
 5106 
 5107   ins_encode %{
 5108     __ movl($dst$$Register, $mem$$Address);
 5109   %}
 5110 
 5111   ins_pipe(ialu_reg_mem);
 5112 %}
 5113 
 5114 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5115 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5116   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5117 
 5118   ins_cost(125);
 5119   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5120   ins_encode %{
 5121     __ movsbl($dst$$Register, $mem$$Address);
 5122   %}
 5123   ins_pipe(ialu_reg_mem);
 5124 %}
 5125 
 5126 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5127 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5128   match(Set dst (AndI (LoadI mem) mask));
 5129 
 5130   ins_cost(125);
 5131   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5132   ins_encode %{
 5133     __ movzbl($dst$$Register, $mem$$Address);
 5134   %}
 5135   ins_pipe(ialu_reg_mem);
 5136 %}
 5137 
 5138 // Load Integer (32 bit signed) to Short (16 bit signed)
 5139 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5140   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5141 
 5142   ins_cost(125);
 5143   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5144   ins_encode %{
 5145     __ movswl($dst$$Register, $mem$$Address);
 5146   %}
 5147   ins_pipe(ialu_reg_mem);
 5148 %}
 5149 
 5150 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5151 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5152   match(Set dst (AndI (LoadI mem) mask));
 5153 
 5154   ins_cost(125);
 5155   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5156   ins_encode %{
 5157     __ movzwl($dst$$Register, $mem$$Address);
 5158   %}
 5159   ins_pipe(ialu_reg_mem);
 5160 %}
 5161 
 5162 // Load Integer into Long Register
 5163 instruct loadI2L(rRegL dst, memory mem)
 5164 %{
 5165   match(Set dst (ConvI2L (LoadI mem)));
 5166 
 5167   ins_cost(125);
 5168   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5169 
 5170   ins_encode %{
 5171     __ movslq($dst$$Register, $mem$$Address);
 5172   %}
 5173 
 5174   ins_pipe(ialu_reg_mem);
 5175 %}
 5176 
 5177 // Load Integer with mask 0xFF into Long Register
 5178 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5179   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5180 
 5181   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5182   ins_encode %{
 5183     __ movzbq($dst$$Register, $mem$$Address);
 5184   %}
 5185   ins_pipe(ialu_reg_mem);
 5186 %}
 5187 
 5188 // Load Integer with mask 0xFFFF into Long Register
 5189 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5190   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5191 
 5192   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5193   ins_encode %{
 5194     __ movzwq($dst$$Register, $mem$$Address);
 5195   %}
 5196   ins_pipe(ialu_reg_mem);
 5197 %}
 5198 
 5199 // Load Integer with a 31-bit mask into Long Register
 5200 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5201   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5202   effect(KILL cr);
 5203 
 5204   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5205             "andl    $dst, $mask" %}
 5206   ins_encode %{
 5207     Register Rdst = $dst$$Register;
 5208     __ movl(Rdst, $mem$$Address);
 5209     __ andl(Rdst, $mask$$constant);
 5210   %}
 5211   ins_pipe(ialu_reg_mem);
 5212 %}
 5213 
 5214 // Load Unsigned Integer into Long Register
 5215 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5216 %{
 5217   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5218 
 5219   ins_cost(125);
 5220   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5221 
 5222   ins_encode %{
 5223     __ movl($dst$$Register, $mem$$Address);
 5224   %}
 5225 
 5226   ins_pipe(ialu_reg_mem);
 5227 %}
 5228 
 5229 // Load Long
 5230 instruct loadL(rRegL dst, memory mem)
 5231 %{
 5232   match(Set dst (LoadL mem));
 5233 
 5234   ins_cost(125);
 5235   format %{ "movq    $dst, $mem\t# long" %}
 5236 
 5237   ins_encode %{
 5238     __ movq($dst$$Register, $mem$$Address);
 5239   %}
 5240 
 5241   ins_pipe(ialu_reg_mem); // XXX
 5242 %}
 5243 
 5244 // Load Range
 5245 instruct loadRange(rRegI dst, memory mem)
 5246 %{
 5247   match(Set dst (LoadRange mem));
 5248 
 5249   ins_cost(125); // XXX
 5250   format %{ "movl    $dst, $mem\t# range" %}
 5251   ins_encode %{
 5252     __ movl($dst$$Register, $mem$$Address);
 5253   %}
 5254   ins_pipe(ialu_reg_mem);
 5255 %}
 5256 
 5257 // Load Pointer
 5258 instruct loadP(rRegP dst, memory mem)
 5259 %{
 5260   match(Set dst (LoadP mem));
 5261   predicate(n->as_Load()->barrier_data() == 0);
 5262 
 5263   ins_cost(125); // XXX
 5264   format %{ "movq    $dst, $mem\t# ptr" %}
 5265   ins_encode %{
 5266     __ movq($dst$$Register, $mem$$Address);
 5267   %}
 5268   ins_pipe(ialu_reg_mem); // XXX
 5269 %}
 5270 
 5271 // Load Compressed Pointer
 5272 instruct loadN(rRegN dst, memory mem)
 5273 %{
 5274    match(Set dst (LoadN mem));
 5275 
 5276    ins_cost(125); // XXX
 5277    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5278    ins_encode %{
 5279      __ movl($dst$$Register, $mem$$Address);
 5280    %}
 5281    ins_pipe(ialu_reg_mem); // XXX
 5282 %}
 5283 
 5284 
 5285 // Load Klass Pointer
 5286 instruct loadKlass(rRegP dst, memory mem)
 5287 %{
 5288   match(Set dst (LoadKlass mem));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movq    $dst, $mem\t# class" %}
 5292   ins_encode %{
 5293     __ movq($dst$$Register, $mem$$Address);
 5294   %}
 5295   ins_pipe(ialu_reg_mem); // XXX
 5296 %}
 5297 
 5298 // Load narrow Klass Pointer
 5299 instruct loadNKlass(rRegN dst, indOffset8 mem, rFlagsReg cr)
 5300 %{
 5301   match(Set dst (LoadNKlass mem));
 5302   effect(TEMP_DEF dst, KILL cr);
 5303   ins_cost(125); // XXX
 5304   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5305   ins_encode %{
 5306     assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset 4, but got: %d", $mem$$disp);
 5307     assert($mem$$index == 4, "expect no index register: %d", $mem$$index);
 5308     __ load_nklass($dst$$Register, $mem$$base$$Register);
 5309   %}
 5310   ins_pipe(pipe_slow); // XXX
 5311 %}
 5312 
 5313 // Load Float
 5314 instruct loadF(regF dst, memory mem)
 5315 %{
 5316   match(Set dst (LoadF mem));
 5317 
 5318   ins_cost(145); // XXX
 5319   format %{ "movss   $dst, $mem\t# float" %}
 5320   ins_encode %{
 5321     __ movflt($dst$$XMMRegister, $mem$$Address);
 5322   %}
 5323   ins_pipe(pipe_slow); // XXX
 5324 %}
 5325 
 5326 // Load Double
 5327 instruct loadD_partial(regD dst, memory mem)
 5328 %{
 5329   predicate(!UseXmmLoadAndClearUpper);
 5330   match(Set dst (LoadD mem));
 5331 
 5332   ins_cost(145); // XXX
 5333   format %{ "movlpd  $dst, $mem\t# double" %}
 5334   ins_encode %{
 5335     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5336   %}
 5337   ins_pipe(pipe_slow); // XXX
 5338 %}
 5339 
 5340 instruct loadD(regD dst, memory mem)
 5341 %{
 5342   predicate(UseXmmLoadAndClearUpper);
 5343   match(Set dst (LoadD mem));
 5344 
 5345   ins_cost(145); // XXX
 5346   format %{ "movsd   $dst, $mem\t# double" %}
 5347   ins_encode %{
 5348     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5349   %}
 5350   ins_pipe(pipe_slow); // XXX
 5351 %}
 5352 
 5353 
 5354 // Following pseudo code describes the algorithm for max[FD]:
 5355 // Min algorithm is on similar lines
 5356 //  btmp = (b < +0.0) ? a : b
 5357 //  atmp = (b < +0.0) ? b : a
 5358 //  Tmp  = Max_Float(atmp , btmp)
 5359 //  Res  = (atmp == NaN) ? atmp : Tmp
 5360 
 5361 // max = java.lang.Math.max(float a, float b)
 5362 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5363   predicate(UseAVX > 0 && !n->is_reduction());
 5364   match(Set dst (MaxF a b));
 5365   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5366   format %{
 5367      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5368      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5369      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5370      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5371      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5372   %}
 5373   ins_encode %{
 5374     int vector_len = Assembler::AVX_128bit;
 5375     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5376     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5377     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5378     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5379     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5380  %}
 5381   ins_pipe( pipe_slow );
 5382 %}
 5383 
 5384 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5385   predicate(UseAVX > 0 && n->is_reduction());
 5386   match(Set dst (MaxF a b));
 5387   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5388 
 5389   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5390   ins_encode %{
 5391     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5392                     false /*min*/, true /*single*/);
 5393   %}
 5394   ins_pipe( pipe_slow );
 5395 %}
 5396 
 5397 // max = java.lang.Math.max(double a, double b)
 5398 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5399   predicate(UseAVX > 0 && !n->is_reduction());
 5400   match(Set dst (MaxD a b));
 5401   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5402   format %{
 5403      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5404      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5405      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5406      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5407      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5408   %}
 5409   ins_encode %{
 5410     int vector_len = Assembler::AVX_128bit;
 5411     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5412     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5413     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5414     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5415     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5416   %}
 5417   ins_pipe( pipe_slow );
 5418 %}
 5419 
 5420 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5421   predicate(UseAVX > 0 && n->is_reduction());
 5422   match(Set dst (MaxD a b));
 5423   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5424 
 5425   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5426   ins_encode %{
 5427     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5428                     false /*min*/, false /*single*/);
 5429   %}
 5430   ins_pipe( pipe_slow );
 5431 %}
 5432 
 5433 // min = java.lang.Math.min(float a, float b)
 5434 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5435   predicate(UseAVX > 0 && !n->is_reduction());
 5436   match(Set dst (MinF a b));
 5437   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5438   format %{
 5439      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5440      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5441      "vminss           $tmp,$atmp,$btmp           \n\t"
 5442      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5443      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5444   %}
 5445   ins_encode %{
 5446     int vector_len = Assembler::AVX_128bit;
 5447     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5448     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5449     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5450     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5451     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5452   %}
 5453   ins_pipe( pipe_slow );
 5454 %}
 5455 
 5456 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5457   predicate(UseAVX > 0 && n->is_reduction());
 5458   match(Set dst (MinF a b));
 5459   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5460 
 5461   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5462   ins_encode %{
 5463     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5464                     true /*min*/, true /*single*/);
 5465   %}
 5466   ins_pipe( pipe_slow );
 5467 %}
 5468 
 5469 // min = java.lang.Math.min(double a, double b)
 5470 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5471   predicate(UseAVX > 0 && !n->is_reduction());
 5472   match(Set dst (MinD a b));
 5473   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5474   format %{
 5475      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5476      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5477      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5478      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5479      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5480   %}
 5481   ins_encode %{
 5482     int vector_len = Assembler::AVX_128bit;
 5483     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5484     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5485     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5486     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5487     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5488   %}
 5489   ins_pipe( pipe_slow );
 5490 %}
 5491 
 5492 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5493   predicate(UseAVX > 0 && n->is_reduction());
 5494   match(Set dst (MinD a b));
 5495   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5496 
 5497   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5498   ins_encode %{
 5499     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5500                     true /*min*/, false /*single*/);
 5501   %}
 5502   ins_pipe( pipe_slow );
 5503 %}
 5504 
 5505 // Load Effective Address
 5506 instruct leaP8(rRegP dst, indOffset8 mem)
 5507 %{
 5508   match(Set dst mem);
 5509 
 5510   ins_cost(110); // XXX
 5511   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5512   ins_encode %{
 5513     __ leaq($dst$$Register, $mem$$Address);
 5514   %}
 5515   ins_pipe(ialu_reg_reg_fat);
 5516 %}
 5517 
 5518 instruct leaP32(rRegP dst, indOffset32 mem)
 5519 %{
 5520   match(Set dst mem);
 5521 
 5522   ins_cost(110);
 5523   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5524   ins_encode %{
 5525     __ leaq($dst$$Register, $mem$$Address);
 5526   %}
 5527   ins_pipe(ialu_reg_reg_fat);
 5528 %}
 5529 
 5530 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5531 %{
 5532   match(Set dst mem);
 5533 
 5534   ins_cost(110);
 5535   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5536   ins_encode %{
 5537     __ leaq($dst$$Register, $mem$$Address);
 5538   %}
 5539   ins_pipe(ialu_reg_reg_fat);
 5540 %}
 5541 
 5542 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5543 %{
 5544   match(Set dst mem);
 5545 
 5546   ins_cost(110);
 5547   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5548   ins_encode %{
 5549     __ leaq($dst$$Register, $mem$$Address);
 5550   %}
 5551   ins_pipe(ialu_reg_reg_fat);
 5552 %}
 5553 
 5554 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5555 %{
 5556   match(Set dst mem);
 5557 
 5558   ins_cost(110);
 5559   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5560   ins_encode %{
 5561     __ leaq($dst$$Register, $mem$$Address);
 5562   %}
 5563   ins_pipe(ialu_reg_reg_fat);
 5564 %}
 5565 
 5566 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5567 %{
 5568   match(Set dst mem);
 5569 
 5570   ins_cost(110);
 5571   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5572   ins_encode %{
 5573     __ leaq($dst$$Register, $mem$$Address);
 5574   %}
 5575   ins_pipe(ialu_reg_reg_fat);
 5576 %}
 5577 
 5578 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5579 %{
 5580   match(Set dst mem);
 5581 
 5582   ins_cost(110);
 5583   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5584   ins_encode %{
 5585     __ leaq($dst$$Register, $mem$$Address);
 5586   %}
 5587   ins_pipe(ialu_reg_reg_fat);
 5588 %}
 5589 
 5590 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5591 %{
 5592   match(Set dst mem);
 5593 
 5594   ins_cost(110);
 5595   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5596   ins_encode %{
 5597     __ leaq($dst$$Register, $mem$$Address);
 5598   %}
 5599   ins_pipe(ialu_reg_reg_fat);
 5600 %}
 5601 
 5602 // Load Effective Address which uses Narrow (32-bits) oop
 5603 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5604 %{
 5605   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5606   match(Set dst mem);
 5607 
 5608   ins_cost(110);
 5609   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5610   ins_encode %{
 5611     __ leaq($dst$$Register, $mem$$Address);
 5612   %}
 5613   ins_pipe(ialu_reg_reg_fat);
 5614 %}
 5615 
 5616 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5617 %{
 5618   predicate(CompressedOops::shift() == 0);
 5619   match(Set dst mem);
 5620 
 5621   ins_cost(110); // XXX
 5622   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5623   ins_encode %{
 5624     __ leaq($dst$$Register, $mem$$Address);
 5625   %}
 5626   ins_pipe(ialu_reg_reg_fat);
 5627 %}
 5628 
 5629 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5630 %{
 5631   predicate(CompressedOops::shift() == 0);
 5632   match(Set dst mem);
 5633 
 5634   ins_cost(110);
 5635   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5636   ins_encode %{
 5637     __ leaq($dst$$Register, $mem$$Address);
 5638   %}
 5639   ins_pipe(ialu_reg_reg_fat);
 5640 %}
 5641 
 5642 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5643 %{
 5644   predicate(CompressedOops::shift() == 0);
 5645   match(Set dst mem);
 5646 
 5647   ins_cost(110);
 5648   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5649   ins_encode %{
 5650     __ leaq($dst$$Register, $mem$$Address);
 5651   %}
 5652   ins_pipe(ialu_reg_reg_fat);
 5653 %}
 5654 
 5655 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5656 %{
 5657   predicate(CompressedOops::shift() == 0);
 5658   match(Set dst mem);
 5659 
 5660   ins_cost(110);
 5661   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5662   ins_encode %{
 5663     __ leaq($dst$$Register, $mem$$Address);
 5664   %}
 5665   ins_pipe(ialu_reg_reg_fat);
 5666 %}
 5667 
 5668 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5669 %{
 5670   predicate(CompressedOops::shift() == 0);
 5671   match(Set dst mem);
 5672 
 5673   ins_cost(110);
 5674   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5675   ins_encode %{
 5676     __ leaq($dst$$Register, $mem$$Address);
 5677   %}
 5678   ins_pipe(ialu_reg_reg_fat);
 5679 %}
 5680 
 5681 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5682 %{
 5683   predicate(CompressedOops::shift() == 0);
 5684   match(Set dst mem);
 5685 
 5686   ins_cost(110);
 5687   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5688   ins_encode %{
 5689     __ leaq($dst$$Register, $mem$$Address);
 5690   %}
 5691   ins_pipe(ialu_reg_reg_fat);
 5692 %}
 5693 
 5694 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5695 %{
 5696   predicate(CompressedOops::shift() == 0);
 5697   match(Set dst mem);
 5698 
 5699   ins_cost(110);
 5700   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5701   ins_encode %{
 5702     __ leaq($dst$$Register, $mem$$Address);
 5703   %}
 5704   ins_pipe(ialu_reg_reg_fat);
 5705 %}
 5706 
 5707 instruct loadConI(rRegI dst, immI src)
 5708 %{
 5709   match(Set dst src);
 5710 
 5711   format %{ "movl    $dst, $src\t# int" %}
 5712   ins_encode %{
 5713     __ movl($dst$$Register, $src$$constant);
 5714   %}
 5715   ins_pipe(ialu_reg_fat); // XXX
 5716 %}
 5717 
 5718 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5719 %{
 5720   match(Set dst src);
 5721   effect(KILL cr);
 5722 
 5723   ins_cost(50);
 5724   format %{ "xorl    $dst, $dst\t# int" %}
 5725   ins_encode %{
 5726     __ xorl($dst$$Register, $dst$$Register);
 5727   %}
 5728   ins_pipe(ialu_reg);
 5729 %}
 5730 
 5731 instruct loadConL(rRegL dst, immL src)
 5732 %{
 5733   match(Set dst src);
 5734 
 5735   ins_cost(150);
 5736   format %{ "movq    $dst, $src\t# long" %}
 5737   ins_encode %{
 5738     __ mov64($dst$$Register, $src$$constant);
 5739   %}
 5740   ins_pipe(ialu_reg);
 5741 %}
 5742 
 5743 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5744 %{
 5745   match(Set dst src);
 5746   effect(KILL cr);
 5747 
 5748   ins_cost(50);
 5749   format %{ "xorl    $dst, $dst\t# long" %}
 5750   ins_encode %{
 5751     __ xorl($dst$$Register, $dst$$Register);
 5752   %}
 5753   ins_pipe(ialu_reg); // XXX
 5754 %}
 5755 
 5756 instruct loadConUL32(rRegL dst, immUL32 src)
 5757 %{
 5758   match(Set dst src);
 5759 
 5760   ins_cost(60);
 5761   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5762   ins_encode %{
 5763     __ movl($dst$$Register, $src$$constant);
 5764   %}
 5765   ins_pipe(ialu_reg);
 5766 %}
 5767 
 5768 instruct loadConL32(rRegL dst, immL32 src)
 5769 %{
 5770   match(Set dst src);
 5771 
 5772   ins_cost(70);
 5773   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5774   ins_encode %{
 5775     __ movq($dst$$Register, $src$$constant);
 5776   %}
 5777   ins_pipe(ialu_reg);
 5778 %}
 5779 
 5780 instruct loadConP(rRegP dst, immP con) %{
 5781   match(Set dst con);
 5782 
 5783   format %{ "movq    $dst, $con\t# ptr" %}
 5784   ins_encode %{
 5785     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5786   %}
 5787   ins_pipe(ialu_reg_fat); // XXX
 5788 %}
 5789 
 5790 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5791 %{
 5792   match(Set dst src);
 5793   effect(KILL cr);
 5794 
 5795   ins_cost(50);
 5796   format %{ "xorl    $dst, $dst\t# ptr" %}
 5797   ins_encode %{
 5798     __ xorl($dst$$Register, $dst$$Register);
 5799   %}
 5800   ins_pipe(ialu_reg);
 5801 %}
 5802 
 5803 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5804 %{
 5805   match(Set dst src);
 5806   effect(KILL cr);
 5807 
 5808   ins_cost(60);
 5809   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5810   ins_encode %{
 5811     __ movl($dst$$Register, $src$$constant);
 5812   %}
 5813   ins_pipe(ialu_reg);
 5814 %}
 5815 
 5816 instruct loadConF(regF dst, immF con) %{
 5817   match(Set dst con);
 5818   ins_cost(125);
 5819   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5820   ins_encode %{
 5821     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5822   %}
 5823   ins_pipe(pipe_slow);
 5824 %}
 5825 
 5826 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5827   match(Set dst src);
 5828   effect(KILL cr);
 5829   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5830   ins_encode %{
 5831     __ xorq($dst$$Register, $dst$$Register);
 5832   %}
 5833   ins_pipe(ialu_reg);
 5834 %}
 5835 
 5836 instruct loadConN(rRegN dst, immN src) %{
 5837   match(Set dst src);
 5838 
 5839   ins_cost(125);
 5840   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5841   ins_encode %{
 5842     address con = (address)$src$$constant;
 5843     if (con == NULL) {
 5844       ShouldNotReachHere();
 5845     } else {
 5846       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5847     }
 5848   %}
 5849   ins_pipe(ialu_reg_fat); // XXX
 5850 %}
 5851 
 5852 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5853   match(Set dst src);
 5854 
 5855   ins_cost(125);
 5856   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5857   ins_encode %{
 5858     address con = (address)$src$$constant;
 5859     if (con == NULL) {
 5860       ShouldNotReachHere();
 5861     } else {
 5862       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5863     }
 5864   %}
 5865   ins_pipe(ialu_reg_fat); // XXX
 5866 %}
 5867 
 5868 instruct loadConF0(regF dst, immF0 src)
 5869 %{
 5870   match(Set dst src);
 5871   ins_cost(100);
 5872 
 5873   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5874   ins_encode %{
 5875     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5876   %}
 5877   ins_pipe(pipe_slow);
 5878 %}
 5879 
 5880 // Use the same format since predicate() can not be used here.
 5881 instruct loadConD(regD dst, immD con) %{
 5882   match(Set dst con);
 5883   ins_cost(125);
 5884   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5885   ins_encode %{
 5886     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5887   %}
 5888   ins_pipe(pipe_slow);
 5889 %}
 5890 
 5891 instruct loadConD0(regD dst, immD0 src)
 5892 %{
 5893   match(Set dst src);
 5894   ins_cost(100);
 5895 
 5896   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5897   ins_encode %{
 5898     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5899   %}
 5900   ins_pipe(pipe_slow);
 5901 %}
 5902 
 5903 instruct loadSSI(rRegI dst, stackSlotI src)
 5904 %{
 5905   match(Set dst src);
 5906 
 5907   ins_cost(125);
 5908   format %{ "movl    $dst, $src\t# int stk" %}
 5909   opcode(0x8B);
 5910   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5911   ins_pipe(ialu_reg_mem);
 5912 %}
 5913 
 5914 instruct loadSSL(rRegL dst, stackSlotL src)
 5915 %{
 5916   match(Set dst src);
 5917 
 5918   ins_cost(125);
 5919   format %{ "movq    $dst, $src\t# long stk" %}
 5920   opcode(0x8B);
 5921   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5922   ins_pipe(ialu_reg_mem);
 5923 %}
 5924 
 5925 instruct loadSSP(rRegP dst, stackSlotP src)
 5926 %{
 5927   match(Set dst src);
 5928 
 5929   ins_cost(125);
 5930   format %{ "movq    $dst, $src\t# ptr stk" %}
 5931   opcode(0x8B);
 5932   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5933   ins_pipe(ialu_reg_mem);
 5934 %}
 5935 
 5936 instruct loadSSF(regF dst, stackSlotF src)
 5937 %{
 5938   match(Set dst src);
 5939 
 5940   ins_cost(125);
 5941   format %{ "movss   $dst, $src\t# float stk" %}
 5942   ins_encode %{
 5943     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5944   %}
 5945   ins_pipe(pipe_slow); // XXX
 5946 %}
 5947 
 5948 // Use the same format since predicate() can not be used here.
 5949 instruct loadSSD(regD dst, stackSlotD src)
 5950 %{
 5951   match(Set dst src);
 5952 
 5953   ins_cost(125);
 5954   format %{ "movsd   $dst, $src\t# double stk" %}
 5955   ins_encode  %{
 5956     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5957   %}
 5958   ins_pipe(pipe_slow); // XXX
 5959 %}
 5960 
 5961 // Prefetch instructions for allocation.
 5962 // Must be safe to execute with invalid address (cannot fault).
 5963 
 5964 instruct prefetchAlloc( memory mem ) %{
 5965   predicate(AllocatePrefetchInstr==3);
 5966   match(PrefetchAllocation mem);
 5967   ins_cost(125);
 5968 
 5969   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5970   ins_encode %{
 5971     __ prefetchw($mem$$Address);
 5972   %}
 5973   ins_pipe(ialu_mem);
 5974 %}
 5975 
 5976 instruct prefetchAllocNTA( memory mem ) %{
 5977   predicate(AllocatePrefetchInstr==0);
 5978   match(PrefetchAllocation mem);
 5979   ins_cost(125);
 5980 
 5981   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5982   ins_encode %{
 5983     __ prefetchnta($mem$$Address);
 5984   %}
 5985   ins_pipe(ialu_mem);
 5986 %}
 5987 
 5988 instruct prefetchAllocT0( memory mem ) %{
 5989   predicate(AllocatePrefetchInstr==1);
 5990   match(PrefetchAllocation mem);
 5991   ins_cost(125);
 5992 
 5993   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5994   ins_encode %{
 5995     __ prefetcht0($mem$$Address);
 5996   %}
 5997   ins_pipe(ialu_mem);
 5998 %}
 5999 
 6000 instruct prefetchAllocT2( memory mem ) %{
 6001   predicate(AllocatePrefetchInstr==2);
 6002   match(PrefetchAllocation mem);
 6003   ins_cost(125);
 6004 
 6005   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6006   ins_encode %{
 6007     __ prefetcht2($mem$$Address);
 6008   %}
 6009   ins_pipe(ialu_mem);
 6010 %}
 6011 
 6012 //----------Store Instructions-------------------------------------------------
 6013 
 6014 // Store Byte
 6015 instruct storeB(memory mem, rRegI src)
 6016 %{
 6017   match(Set mem (StoreB mem src));
 6018 
 6019   ins_cost(125); // XXX
 6020   format %{ "movb    $mem, $src\t# byte" %}
 6021   ins_encode %{
 6022     __ movb($mem$$Address, $src$$Register);
 6023   %}
 6024   ins_pipe(ialu_mem_reg);
 6025 %}
 6026 
 6027 // Store Char/Short
 6028 instruct storeC(memory mem, rRegI src)
 6029 %{
 6030   match(Set mem (StoreC mem src));
 6031 
 6032   ins_cost(125); // XXX
 6033   format %{ "movw    $mem, $src\t# char/short" %}
 6034   ins_encode %{
 6035     __ movw($mem$$Address, $src$$Register);
 6036   %}
 6037   ins_pipe(ialu_mem_reg);
 6038 %}
 6039 
 6040 // Store Integer
 6041 instruct storeI(memory mem, rRegI src)
 6042 %{
 6043   match(Set mem (StoreI mem src));
 6044 
 6045   ins_cost(125); // XXX
 6046   format %{ "movl    $mem, $src\t# int" %}
 6047   ins_encode %{
 6048     __ movl($mem$$Address, $src$$Register);
 6049   %}
 6050   ins_pipe(ialu_mem_reg);
 6051 %}
 6052 
 6053 // Store Long
 6054 instruct storeL(memory mem, rRegL src)
 6055 %{
 6056   match(Set mem (StoreL mem src));
 6057 
 6058   ins_cost(125); // XXX
 6059   format %{ "movq    $mem, $src\t# long" %}
 6060   ins_encode %{
 6061     __ movq($mem$$Address, $src$$Register);
 6062   %}
 6063   ins_pipe(ialu_mem_reg); // XXX
 6064 %}
 6065 
 6066 // Store Pointer
 6067 instruct storeP(memory mem, any_RegP src)
 6068 %{
 6069   match(Set mem (StoreP mem src));
 6070 
 6071   ins_cost(125); // XXX
 6072   format %{ "movq    $mem, $src\t# ptr" %}
 6073   ins_encode %{
 6074     __ movq($mem$$Address, $src$$Register);
 6075   %}
 6076   ins_pipe(ialu_mem_reg);
 6077 %}
 6078 
 6079 instruct storeImmP0(memory mem, immP0 zero)
 6080 %{
 6081   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6082   match(Set mem (StoreP mem zero));
 6083 
 6084   ins_cost(125); // XXX
 6085   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6086   ins_encode %{
 6087     __ movq($mem$$Address, r12);
 6088   %}
 6089   ins_pipe(ialu_mem_reg);
 6090 %}
 6091 
 6092 // Store NULL Pointer, mark word, or other simple pointer constant.
 6093 instruct storeImmP(memory mem, immP31 src)
 6094 %{
 6095   match(Set mem (StoreP mem src));
 6096 
 6097   ins_cost(150); // XXX
 6098   format %{ "movq    $mem, $src\t# ptr" %}
 6099   ins_encode %{
 6100     __ movq($mem$$Address, $src$$constant);
 6101   %}
 6102   ins_pipe(ialu_mem_imm);
 6103 %}
 6104 
 6105 // Store Compressed Pointer
 6106 instruct storeN(memory mem, rRegN src)
 6107 %{
 6108   match(Set mem (StoreN mem src));
 6109 
 6110   ins_cost(125); // XXX
 6111   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6112   ins_encode %{
 6113     __ movl($mem$$Address, $src$$Register);
 6114   %}
 6115   ins_pipe(ialu_mem_reg);
 6116 %}
 6117 
 6118 instruct storeNKlass(memory mem, rRegN src)
 6119 %{
 6120   match(Set mem (StoreNKlass mem src));
 6121 
 6122   ins_cost(125); // XXX
 6123   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6124   ins_encode %{
 6125     __ movl($mem$$Address, $src$$Register);
 6126   %}
 6127   ins_pipe(ialu_mem_reg);
 6128 %}
 6129 
 6130 instruct storeImmN0(memory mem, immN0 zero)
 6131 %{
 6132   predicate(CompressedOops::base() == NULL);
 6133   match(Set mem (StoreN mem zero));
 6134 
 6135   ins_cost(125); // XXX
 6136   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6137   ins_encode %{
 6138     __ movl($mem$$Address, r12);
 6139   %}
 6140   ins_pipe(ialu_mem_reg);
 6141 %}
 6142 
 6143 instruct storeImmN(memory mem, immN src)
 6144 %{
 6145   match(Set mem (StoreN mem src));
 6146 
 6147   ins_cost(150); // XXX
 6148   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6149   ins_encode %{
 6150     address con = (address)$src$$constant;
 6151     if (con == NULL) {
 6152       __ movl($mem$$Address, 0);
 6153     } else {
 6154       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6155     }
 6156   %}
 6157   ins_pipe(ialu_mem_imm);
 6158 %}
 6159 
 6160 instruct storeImmNKlass(memory mem, immNKlass src)
 6161 %{
 6162   match(Set mem (StoreNKlass mem src));
 6163 
 6164   ins_cost(150); // XXX
 6165   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6166   ins_encode %{
 6167     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6168   %}
 6169   ins_pipe(ialu_mem_imm);
 6170 %}
 6171 
 6172 // Store Integer Immediate
 6173 instruct storeImmI0(memory mem, immI_0 zero)
 6174 %{
 6175   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6176   match(Set mem (StoreI mem zero));
 6177 
 6178   ins_cost(125); // XXX
 6179   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6180   ins_encode %{
 6181     __ movl($mem$$Address, r12);
 6182   %}
 6183   ins_pipe(ialu_mem_reg);
 6184 %}
 6185 
 6186 instruct storeImmI(memory mem, immI src)
 6187 %{
 6188   match(Set mem (StoreI mem src));
 6189 
 6190   ins_cost(150);
 6191   format %{ "movl    $mem, $src\t# int" %}
 6192   ins_encode %{
 6193     __ movl($mem$$Address, $src$$constant);
 6194   %}
 6195   ins_pipe(ialu_mem_imm);
 6196 %}
 6197 
 6198 // Store Long Immediate
 6199 instruct storeImmL0(memory mem, immL0 zero)
 6200 %{
 6201   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6202   match(Set mem (StoreL mem zero));
 6203 
 6204   ins_cost(125); // XXX
 6205   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6206   ins_encode %{
 6207     __ movq($mem$$Address, r12);
 6208   %}
 6209   ins_pipe(ialu_mem_reg);
 6210 %}
 6211 
 6212 instruct storeImmL(memory mem, immL32 src)
 6213 %{
 6214   match(Set mem (StoreL mem src));
 6215 
 6216   ins_cost(150);
 6217   format %{ "movq    $mem, $src\t# long" %}
 6218   ins_encode %{
 6219     __ movq($mem$$Address, $src$$constant);
 6220   %}
 6221   ins_pipe(ialu_mem_imm);
 6222 %}
 6223 
 6224 // Store Short/Char Immediate
 6225 instruct storeImmC0(memory mem, immI_0 zero)
 6226 %{
 6227   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6228   match(Set mem (StoreC mem zero));
 6229 
 6230   ins_cost(125); // XXX
 6231   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6232   ins_encode %{
 6233     __ movw($mem$$Address, r12);
 6234   %}
 6235   ins_pipe(ialu_mem_reg);
 6236 %}
 6237 
 6238 instruct storeImmI16(memory mem, immI16 src)
 6239 %{
 6240   predicate(UseStoreImmI16);
 6241   match(Set mem (StoreC mem src));
 6242 
 6243   ins_cost(150);
 6244   format %{ "movw    $mem, $src\t# short/char" %}
 6245   ins_encode %{
 6246     __ movw($mem$$Address, $src$$constant);
 6247   %}
 6248   ins_pipe(ialu_mem_imm);
 6249 %}
 6250 
 6251 // Store Byte Immediate
 6252 instruct storeImmB0(memory mem, immI_0 zero)
 6253 %{
 6254   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6255   match(Set mem (StoreB mem zero));
 6256 
 6257   ins_cost(125); // XXX
 6258   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6259   ins_encode %{
 6260     __ movb($mem$$Address, r12);
 6261   %}
 6262   ins_pipe(ialu_mem_reg);
 6263 %}
 6264 
 6265 instruct storeImmB(memory mem, immI8 src)
 6266 %{
 6267   match(Set mem (StoreB mem src));
 6268 
 6269   ins_cost(150); // XXX
 6270   format %{ "movb    $mem, $src\t# byte" %}
 6271   ins_encode %{
 6272     __ movb($mem$$Address, $src$$constant);
 6273   %}
 6274   ins_pipe(ialu_mem_imm);
 6275 %}
 6276 
 6277 // Store CMS card-mark Immediate
 6278 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6279 %{
 6280   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6281   match(Set mem (StoreCM mem zero));
 6282 
 6283   ins_cost(125); // XXX
 6284   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6285   ins_encode %{
 6286     __ movb($mem$$Address, r12);
 6287   %}
 6288   ins_pipe(ialu_mem_reg);
 6289 %}
 6290 
 6291 instruct storeImmCM0(memory mem, immI_0 src)
 6292 %{
 6293   match(Set mem (StoreCM mem src));
 6294 
 6295   ins_cost(150); // XXX
 6296   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6297   ins_encode %{
 6298     __ movb($mem$$Address, $src$$constant);
 6299   %}
 6300   ins_pipe(ialu_mem_imm);
 6301 %}
 6302 
 6303 // Store Float
 6304 instruct storeF(memory mem, regF src)
 6305 %{
 6306   match(Set mem (StoreF mem src));
 6307 
 6308   ins_cost(95); // XXX
 6309   format %{ "movss   $mem, $src\t# float" %}
 6310   ins_encode %{
 6311     __ movflt($mem$$Address, $src$$XMMRegister);
 6312   %}
 6313   ins_pipe(pipe_slow); // XXX
 6314 %}
 6315 
 6316 // Store immediate Float value (it is faster than store from XMM register)
 6317 instruct storeF0(memory mem, immF0 zero)
 6318 %{
 6319   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6320   match(Set mem (StoreF mem zero));
 6321 
 6322   ins_cost(25); // XXX
 6323   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6324   ins_encode %{
 6325     __ movl($mem$$Address, r12);
 6326   %}
 6327   ins_pipe(ialu_mem_reg);
 6328 %}
 6329 
 6330 instruct storeF_imm(memory mem, immF src)
 6331 %{
 6332   match(Set mem (StoreF mem src));
 6333 
 6334   ins_cost(50);
 6335   format %{ "movl    $mem, $src\t# float" %}
 6336   ins_encode %{
 6337     __ movl($mem$$Address, jint_cast($src$$constant));
 6338   %}
 6339   ins_pipe(ialu_mem_imm);
 6340 %}
 6341 
 6342 // Store Double
 6343 instruct storeD(memory mem, regD src)
 6344 %{
 6345   match(Set mem (StoreD mem src));
 6346 
 6347   ins_cost(95); // XXX
 6348   format %{ "movsd   $mem, $src\t# double" %}
 6349   ins_encode %{
 6350     __ movdbl($mem$$Address, $src$$XMMRegister);
 6351   %}
 6352   ins_pipe(pipe_slow); // XXX
 6353 %}
 6354 
 6355 // Store immediate double 0.0 (it is faster than store from XMM register)
 6356 instruct storeD0_imm(memory mem, immD0 src)
 6357 %{
 6358   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6359   match(Set mem (StoreD mem src));
 6360 
 6361   ins_cost(50);
 6362   format %{ "movq    $mem, $src\t# double 0." %}
 6363   ins_encode %{
 6364     __ movq($mem$$Address, $src$$constant);
 6365   %}
 6366   ins_pipe(ialu_mem_imm);
 6367 %}
 6368 
 6369 instruct storeD0(memory mem, immD0 zero)
 6370 %{
 6371   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6372   match(Set mem (StoreD mem zero));
 6373 
 6374   ins_cost(25); // XXX
 6375   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6376   ins_encode %{
 6377     __ movq($mem$$Address, r12);
 6378   %}
 6379   ins_pipe(ialu_mem_reg);
 6380 %}
 6381 
 6382 instruct storeSSI(stackSlotI dst, rRegI src)
 6383 %{
 6384   match(Set dst src);
 6385 
 6386   ins_cost(100);
 6387   format %{ "movl    $dst, $src\t# int stk" %}
 6388   opcode(0x89);
 6389   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6390   ins_pipe( ialu_mem_reg );
 6391 %}
 6392 
 6393 instruct storeSSL(stackSlotL dst, rRegL src)
 6394 %{
 6395   match(Set dst src);
 6396 
 6397   ins_cost(100);
 6398   format %{ "movq    $dst, $src\t# long stk" %}
 6399   opcode(0x89);
 6400   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6401   ins_pipe(ialu_mem_reg);
 6402 %}
 6403 
 6404 instruct storeSSP(stackSlotP dst, rRegP src)
 6405 %{
 6406   match(Set dst src);
 6407 
 6408   ins_cost(100);
 6409   format %{ "movq    $dst, $src\t# ptr stk" %}
 6410   opcode(0x89);
 6411   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6412   ins_pipe(ialu_mem_reg);
 6413 %}
 6414 
 6415 instruct storeSSF(stackSlotF dst, regF src)
 6416 %{
 6417   match(Set dst src);
 6418 
 6419   ins_cost(95); // XXX
 6420   format %{ "movss   $dst, $src\t# float stk" %}
 6421   ins_encode %{
 6422     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6423   %}
 6424   ins_pipe(pipe_slow); // XXX
 6425 %}
 6426 
 6427 instruct storeSSD(stackSlotD dst, regD src)
 6428 %{
 6429   match(Set dst src);
 6430 
 6431   ins_cost(95); // XXX
 6432   format %{ "movsd   $dst, $src\t# double stk" %}
 6433   ins_encode %{
 6434     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6435   %}
 6436   ins_pipe(pipe_slow); // XXX
 6437 %}
 6438 
 6439 instruct cacheWB(indirect addr)
 6440 %{
 6441   predicate(VM_Version::supports_data_cache_line_flush());
 6442   match(CacheWB addr);
 6443 
 6444   ins_cost(100);
 6445   format %{"cache wb $addr" %}
 6446   ins_encode %{
 6447     assert($addr->index_position() < 0, "should be");
 6448     assert($addr$$disp == 0, "should be");
 6449     __ cache_wb(Address($addr$$base$$Register, 0));
 6450   %}
 6451   ins_pipe(pipe_slow); // XXX
 6452 %}
 6453 
 6454 instruct cacheWBPreSync()
 6455 %{
 6456   predicate(VM_Version::supports_data_cache_line_flush());
 6457   match(CacheWBPreSync);
 6458 
 6459   ins_cost(100);
 6460   format %{"cache wb presync" %}
 6461   ins_encode %{
 6462     __ cache_wbsync(true);
 6463   %}
 6464   ins_pipe(pipe_slow); // XXX
 6465 %}
 6466 
 6467 instruct cacheWBPostSync()
 6468 %{
 6469   predicate(VM_Version::supports_data_cache_line_flush());
 6470   match(CacheWBPostSync);
 6471 
 6472   ins_cost(100);
 6473   format %{"cache wb postsync" %}
 6474   ins_encode %{
 6475     __ cache_wbsync(false);
 6476   %}
 6477   ins_pipe(pipe_slow); // XXX
 6478 %}
 6479 
 6480 //----------BSWAP Instructions-------------------------------------------------
 6481 instruct bytes_reverse_int(rRegI dst) %{
 6482   match(Set dst (ReverseBytesI dst));
 6483 
 6484   format %{ "bswapl  $dst" %}
 6485   ins_encode %{
 6486     __ bswapl($dst$$Register);
 6487   %}
 6488   ins_pipe( ialu_reg );
 6489 %}
 6490 
 6491 instruct bytes_reverse_long(rRegL dst) %{
 6492   match(Set dst (ReverseBytesL dst));
 6493 
 6494   format %{ "bswapq  $dst" %}
 6495   ins_encode %{
 6496     __ bswapq($dst$$Register);
 6497   %}
 6498   ins_pipe( ialu_reg);
 6499 %}
 6500 
 6501 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6502   match(Set dst (ReverseBytesUS dst));
 6503   effect(KILL cr);
 6504 
 6505   format %{ "bswapl  $dst\n\t"
 6506             "shrl    $dst,16\n\t" %}
 6507   ins_encode %{
 6508     __ bswapl($dst$$Register);
 6509     __ shrl($dst$$Register, 16);
 6510   %}
 6511   ins_pipe( ialu_reg );
 6512 %}
 6513 
 6514 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6515   match(Set dst (ReverseBytesS dst));
 6516   effect(KILL cr);
 6517 
 6518   format %{ "bswapl  $dst\n\t"
 6519             "sar     $dst,16\n\t" %}
 6520   ins_encode %{
 6521     __ bswapl($dst$$Register);
 6522     __ sarl($dst$$Register, 16);
 6523   %}
 6524   ins_pipe( ialu_reg );
 6525 %}
 6526 
 6527 //---------- Zeros Count Instructions ------------------------------------------
 6528 
 6529 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6530   predicate(UseCountLeadingZerosInstruction);
 6531   match(Set dst (CountLeadingZerosI src));
 6532   effect(KILL cr);
 6533 
 6534   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6535   ins_encode %{
 6536     __ lzcntl($dst$$Register, $src$$Register);
 6537   %}
 6538   ins_pipe(ialu_reg);
 6539 %}
 6540 
 6541 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6542   predicate(UseCountLeadingZerosInstruction);
 6543   match(Set dst (CountLeadingZerosI (LoadI src)));
 6544   effect(KILL cr);
 6545   ins_cost(175);
 6546   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6547   ins_encode %{
 6548     __ lzcntl($dst$$Register, $src$$Address);
 6549   %}
 6550   ins_pipe(ialu_reg_mem);
 6551 %}
 6552 
 6553 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6554   predicate(!UseCountLeadingZerosInstruction);
 6555   match(Set dst (CountLeadingZerosI src));
 6556   effect(KILL cr);
 6557 
 6558   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6559             "jnz     skip\n\t"
 6560             "movl    $dst, -1\n"
 6561       "skip:\n\t"
 6562             "negl    $dst\n\t"
 6563             "addl    $dst, 31" %}
 6564   ins_encode %{
 6565     Register Rdst = $dst$$Register;
 6566     Register Rsrc = $src$$Register;
 6567     Label skip;
 6568     __ bsrl(Rdst, Rsrc);
 6569     __ jccb(Assembler::notZero, skip);
 6570     __ movl(Rdst, -1);
 6571     __ bind(skip);
 6572     __ negl(Rdst);
 6573     __ addl(Rdst, BitsPerInt - 1);
 6574   %}
 6575   ins_pipe(ialu_reg);
 6576 %}
 6577 
 6578 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6579   predicate(UseCountLeadingZerosInstruction);
 6580   match(Set dst (CountLeadingZerosL src));
 6581   effect(KILL cr);
 6582 
 6583   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6584   ins_encode %{
 6585     __ lzcntq($dst$$Register, $src$$Register);
 6586   %}
 6587   ins_pipe(ialu_reg);
 6588 %}
 6589 
 6590 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6591   predicate(UseCountLeadingZerosInstruction);
 6592   match(Set dst (CountLeadingZerosL (LoadL src)));
 6593   effect(KILL cr);
 6594   ins_cost(175);
 6595   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6596   ins_encode %{
 6597     __ lzcntq($dst$$Register, $src$$Address);
 6598   %}
 6599   ins_pipe(ialu_reg_mem);
 6600 %}
 6601 
 6602 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6603   predicate(!UseCountLeadingZerosInstruction);
 6604   match(Set dst (CountLeadingZerosL src));
 6605   effect(KILL cr);
 6606 
 6607   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6608             "jnz     skip\n\t"
 6609             "movl    $dst, -1\n"
 6610       "skip:\n\t"
 6611             "negl    $dst\n\t"
 6612             "addl    $dst, 63" %}
 6613   ins_encode %{
 6614     Register Rdst = $dst$$Register;
 6615     Register Rsrc = $src$$Register;
 6616     Label skip;
 6617     __ bsrq(Rdst, Rsrc);
 6618     __ jccb(Assembler::notZero, skip);
 6619     __ movl(Rdst, -1);
 6620     __ bind(skip);
 6621     __ negl(Rdst);
 6622     __ addl(Rdst, BitsPerLong - 1);
 6623   %}
 6624   ins_pipe(ialu_reg);
 6625 %}
 6626 
 6627 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6628   predicate(UseCountTrailingZerosInstruction);
 6629   match(Set dst (CountTrailingZerosI src));
 6630   effect(KILL cr);
 6631 
 6632   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6633   ins_encode %{
 6634     __ tzcntl($dst$$Register, $src$$Register);
 6635   %}
 6636   ins_pipe(ialu_reg);
 6637 %}
 6638 
 6639 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6640   predicate(UseCountTrailingZerosInstruction);
 6641   match(Set dst (CountTrailingZerosI (LoadI src)));
 6642   effect(KILL cr);
 6643   ins_cost(175);
 6644   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6645   ins_encode %{
 6646     __ tzcntl($dst$$Register, $src$$Address);
 6647   %}
 6648   ins_pipe(ialu_reg_mem);
 6649 %}
 6650 
 6651 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6652   predicate(!UseCountTrailingZerosInstruction);
 6653   match(Set dst (CountTrailingZerosI src));
 6654   effect(KILL cr);
 6655 
 6656   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6657             "jnz     done\n\t"
 6658             "movl    $dst, 32\n"
 6659       "done:" %}
 6660   ins_encode %{
 6661     Register Rdst = $dst$$Register;
 6662     Label done;
 6663     __ bsfl(Rdst, $src$$Register);
 6664     __ jccb(Assembler::notZero, done);
 6665     __ movl(Rdst, BitsPerInt);
 6666     __ bind(done);
 6667   %}
 6668   ins_pipe(ialu_reg);
 6669 %}
 6670 
 6671 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6672   predicate(UseCountTrailingZerosInstruction);
 6673   match(Set dst (CountTrailingZerosL src));
 6674   effect(KILL cr);
 6675 
 6676   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6677   ins_encode %{
 6678     __ tzcntq($dst$$Register, $src$$Register);
 6679   %}
 6680   ins_pipe(ialu_reg);
 6681 %}
 6682 
 6683 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 6684   predicate(UseCountTrailingZerosInstruction);
 6685   match(Set dst (CountTrailingZerosL (LoadL src)));
 6686   effect(KILL cr);
 6687   ins_cost(175);
 6688   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6689   ins_encode %{
 6690     __ tzcntq($dst$$Register, $src$$Address);
 6691   %}
 6692   ins_pipe(ialu_reg_mem);
 6693 %}
 6694 
 6695 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6696   predicate(!UseCountTrailingZerosInstruction);
 6697   match(Set dst (CountTrailingZerosL src));
 6698   effect(KILL cr);
 6699 
 6700   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6701             "jnz     done\n\t"
 6702             "movl    $dst, 64\n"
 6703       "done:" %}
 6704   ins_encode %{
 6705     Register Rdst = $dst$$Register;
 6706     Label done;
 6707     __ bsfq(Rdst, $src$$Register);
 6708     __ jccb(Assembler::notZero, done);
 6709     __ movl(Rdst, BitsPerLong);
 6710     __ bind(done);
 6711   %}
 6712   ins_pipe(ialu_reg);
 6713 %}
 6714 
 6715 //--------------- Reverse Operation Instructions ----------------
 6716 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 6717   predicate(!VM_Version::supports_gfni());
 6718   match(Set dst (ReverseI src));
 6719   effect(TEMP dst, TEMP rtmp, KILL cr);
 6720   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 6721   ins_encode %{
 6722     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 6723   %}
 6724   ins_pipe( ialu_reg );
 6725 %}
 6726 
 6727 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, regF xtmp1, regF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6728   predicate(VM_Version::supports_gfni());
 6729   match(Set dst (ReverseI src));
 6730   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6731   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6732   ins_encode %{
 6733     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 6734   %}
 6735   ins_pipe( ialu_reg );
 6736 %}
 6737 
 6738 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 6739   predicate(!VM_Version::supports_gfni());
 6740   match(Set dst (ReverseL src));
 6741   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 6742   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 6743   ins_encode %{
 6744     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 6745   %}
 6746   ins_pipe( ialu_reg );
 6747 %}
 6748 
 6749 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, regD xtmp1, regD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 6750   predicate(VM_Version::supports_gfni());
 6751   match(Set dst (ReverseL src));
 6752   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 6753   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 6754   ins_encode %{
 6755     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 6756   %}
 6757   ins_pipe( ialu_reg );
 6758 %}
 6759 
 6760 //---------- Population Count Instructions -------------------------------------
 6761 
 6762 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6763   predicate(UsePopCountInstruction);
 6764   match(Set dst (PopCountI src));
 6765   effect(KILL cr);
 6766 
 6767   format %{ "popcnt  $dst, $src" %}
 6768   ins_encode %{
 6769     __ popcntl($dst$$Register, $src$$Register);
 6770   %}
 6771   ins_pipe(ialu_reg);
 6772 %}
 6773 
 6774 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6775   predicate(UsePopCountInstruction);
 6776   match(Set dst (PopCountI (LoadI mem)));
 6777   effect(KILL cr);
 6778 
 6779   format %{ "popcnt  $dst, $mem" %}
 6780   ins_encode %{
 6781     __ popcntl($dst$$Register, $mem$$Address);
 6782   %}
 6783   ins_pipe(ialu_reg);
 6784 %}
 6785 
 6786 // Note: Long.bitCount(long) returns an int.
 6787 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6788   predicate(UsePopCountInstruction);
 6789   match(Set dst (PopCountL src));
 6790   effect(KILL cr);
 6791 
 6792   format %{ "popcnt  $dst, $src" %}
 6793   ins_encode %{
 6794     __ popcntq($dst$$Register, $src$$Register);
 6795   %}
 6796   ins_pipe(ialu_reg);
 6797 %}
 6798 
 6799 // Note: Long.bitCount(long) returns an int.
 6800 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6801   predicate(UsePopCountInstruction);
 6802   match(Set dst (PopCountL (LoadL mem)));
 6803   effect(KILL cr);
 6804 
 6805   format %{ "popcnt  $dst, $mem" %}
 6806   ins_encode %{
 6807     __ popcntq($dst$$Register, $mem$$Address);
 6808   %}
 6809   ins_pipe(ialu_reg);
 6810 %}
 6811 
 6812 
 6813 //----------MemBar Instructions-----------------------------------------------
 6814 // Memory barrier flavors
 6815 
 6816 instruct membar_acquire()
 6817 %{
 6818   match(MemBarAcquire);
 6819   match(LoadFence);
 6820   ins_cost(0);
 6821 
 6822   size(0);
 6823   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6824   ins_encode();
 6825   ins_pipe(empty);
 6826 %}
 6827 
 6828 instruct membar_acquire_lock()
 6829 %{
 6830   match(MemBarAcquireLock);
 6831   ins_cost(0);
 6832 
 6833   size(0);
 6834   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6835   ins_encode();
 6836   ins_pipe(empty);
 6837 %}
 6838 
 6839 instruct membar_release()
 6840 %{
 6841   match(MemBarRelease);
 6842   match(StoreFence);
 6843   ins_cost(0);
 6844 
 6845   size(0);
 6846   format %{ "MEMBAR-release ! (empty encoding)" %}
 6847   ins_encode();
 6848   ins_pipe(empty);
 6849 %}
 6850 
 6851 instruct membar_release_lock()
 6852 %{
 6853   match(MemBarReleaseLock);
 6854   ins_cost(0);
 6855 
 6856   size(0);
 6857   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6858   ins_encode();
 6859   ins_pipe(empty);
 6860 %}
 6861 
 6862 instruct membar_volatile(rFlagsReg cr) %{
 6863   match(MemBarVolatile);
 6864   effect(KILL cr);
 6865   ins_cost(400);
 6866 
 6867   format %{
 6868     $$template
 6869     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6870   %}
 6871   ins_encode %{
 6872     __ membar(Assembler::StoreLoad);
 6873   %}
 6874   ins_pipe(pipe_slow);
 6875 %}
 6876 
 6877 instruct unnecessary_membar_volatile()
 6878 %{
 6879   match(MemBarVolatile);
 6880   predicate(Matcher::post_store_load_barrier(n));
 6881   ins_cost(0);
 6882 
 6883   size(0);
 6884   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6885   ins_encode();
 6886   ins_pipe(empty);
 6887 %}
 6888 
 6889 instruct membar_storestore() %{
 6890   match(MemBarStoreStore);
 6891   match(StoreStoreFence);
 6892   ins_cost(0);
 6893 
 6894   size(0);
 6895   format %{ "MEMBAR-storestore (empty encoding)" %}
 6896   ins_encode( );
 6897   ins_pipe(empty);
 6898 %}
 6899 
 6900 //----------Move Instructions--------------------------------------------------
 6901 
 6902 instruct castX2P(rRegP dst, rRegL src)
 6903 %{
 6904   match(Set dst (CastX2P src));
 6905 
 6906   format %{ "movq    $dst, $src\t# long->ptr" %}
 6907   ins_encode %{
 6908     if ($dst$$reg != $src$$reg) {
 6909       __ movptr($dst$$Register, $src$$Register);
 6910     }
 6911   %}
 6912   ins_pipe(ialu_reg_reg); // XXX
 6913 %}
 6914 
 6915 instruct castP2X(rRegL dst, rRegP src)
 6916 %{
 6917   match(Set dst (CastP2X src));
 6918 
 6919   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6920   ins_encode %{
 6921     if ($dst$$reg != $src$$reg) {
 6922       __ movptr($dst$$Register, $src$$Register);
 6923     }
 6924   %}
 6925   ins_pipe(ialu_reg_reg); // XXX
 6926 %}
 6927 
 6928 // Convert oop into int for vectors alignment masking
 6929 instruct convP2I(rRegI dst, rRegP src)
 6930 %{
 6931   match(Set dst (ConvL2I (CastP2X src)));
 6932 
 6933   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6934   ins_encode %{
 6935     __ movl($dst$$Register, $src$$Register);
 6936   %}
 6937   ins_pipe(ialu_reg_reg); // XXX
 6938 %}
 6939 
 6940 // Convert compressed oop into int for vectors alignment masking
 6941 // in case of 32bit oops (heap < 4Gb).
 6942 instruct convN2I(rRegI dst, rRegN src)
 6943 %{
 6944   predicate(CompressedOops::shift() == 0);
 6945   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6946 
 6947   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6948   ins_encode %{
 6949     __ movl($dst$$Register, $src$$Register);
 6950   %}
 6951   ins_pipe(ialu_reg_reg); // XXX
 6952 %}
 6953 
 6954 // Convert oop pointer into compressed form
 6955 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6956   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6957   match(Set dst (EncodeP src));
 6958   effect(KILL cr);
 6959   format %{ "encode_heap_oop $dst,$src" %}
 6960   ins_encode %{
 6961     Register s = $src$$Register;
 6962     Register d = $dst$$Register;
 6963     if (s != d) {
 6964       __ movq(d, s);
 6965     }
 6966     __ encode_heap_oop(d);
 6967   %}
 6968   ins_pipe(ialu_reg_long);
 6969 %}
 6970 
 6971 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6972   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6973   match(Set dst (EncodeP src));
 6974   effect(KILL cr);
 6975   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6976   ins_encode %{
 6977     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6978   %}
 6979   ins_pipe(ialu_reg_long);
 6980 %}
 6981 
 6982 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6983   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6984             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6985   match(Set dst (DecodeN src));
 6986   effect(KILL cr);
 6987   format %{ "decode_heap_oop $dst,$src" %}
 6988   ins_encode %{
 6989     Register s = $src$$Register;
 6990     Register d = $dst$$Register;
 6991     if (s != d) {
 6992       __ movq(d, s);
 6993     }
 6994     __ decode_heap_oop(d);
 6995   %}
 6996   ins_pipe(ialu_reg_long);
 6997 %}
 6998 
 6999 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7000   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 7001             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 7002   match(Set dst (DecodeN src));
 7003   effect(KILL cr);
 7004   format %{ "decode_heap_oop_not_null $dst,$src" %}
 7005   ins_encode %{
 7006     Register s = $src$$Register;
 7007     Register d = $dst$$Register;
 7008     if (s != d) {
 7009       __ decode_heap_oop_not_null(d, s);
 7010     } else {
 7011       __ decode_heap_oop_not_null(d);
 7012     }
 7013   %}
 7014   ins_pipe(ialu_reg_long);
 7015 %}
 7016 
 7017 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 7018   match(Set dst (EncodePKlass src));
 7019   effect(TEMP dst, KILL cr);
 7020   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7021   ins_encode %{
 7022     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7023   %}
 7024   ins_pipe(ialu_reg_long);
 7025 %}
 7026 
 7027 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7028   match(Set dst (DecodeNKlass src));
 7029   effect(TEMP dst, KILL cr);
 7030   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7031   ins_encode %{
 7032     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7033   %}
 7034   ins_pipe(ialu_reg_long);
 7035 %}
 7036 
 7037 //----------Conditional Move---------------------------------------------------
 7038 // Jump
 7039 // dummy instruction for generating temp registers
 7040 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7041   match(Jump (LShiftL switch_val shift));
 7042   ins_cost(350);
 7043   predicate(false);
 7044   effect(TEMP dest);
 7045 
 7046   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7047             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7048   ins_encode %{
 7049     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7050     // to do that and the compiler is using that register as one it can allocate.
 7051     // So we build it all by hand.
 7052     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7053     // ArrayAddress dispatch(table, index);
 7054     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7055     __ lea($dest$$Register, $constantaddress);
 7056     __ jmp(dispatch);
 7057   %}
 7058   ins_pipe(pipe_jmp);
 7059 %}
 7060 
 7061 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7062   match(Jump (AddL (LShiftL switch_val shift) offset));
 7063   ins_cost(350);
 7064   effect(TEMP dest);
 7065 
 7066   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7067             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7068   ins_encode %{
 7069     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7070     // to do that and the compiler is using that register as one it can allocate.
 7071     // So we build it all by hand.
 7072     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7073     // ArrayAddress dispatch(table, index);
 7074     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7075     __ lea($dest$$Register, $constantaddress);
 7076     __ jmp(dispatch);
 7077   %}
 7078   ins_pipe(pipe_jmp);
 7079 %}
 7080 
 7081 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7082   match(Jump switch_val);
 7083   ins_cost(350);
 7084   effect(TEMP dest);
 7085 
 7086   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7087             "jmp     [$dest + $switch_val]\n\t" %}
 7088   ins_encode %{
 7089     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7090     // to do that and the compiler is using that register as one it can allocate.
 7091     // So we build it all by hand.
 7092     // Address index(noreg, switch_reg, Address::times_1);
 7093     // ArrayAddress dispatch(table, index);
 7094     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7095     __ lea($dest$$Register, $constantaddress);
 7096     __ jmp(dispatch);
 7097   %}
 7098   ins_pipe(pipe_jmp);
 7099 %}
 7100 
 7101 // Conditional move
 7102 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7103 %{
 7104   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7105   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7106 
 7107   ins_cost(100); // XXX
 7108   format %{ "setbn$cop $dst\t# signed, int" %}
 7109   ins_encode %{
 7110     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7111     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7112   %}
 7113   ins_pipe(ialu_reg);
 7114 %}
 7115 
 7116 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7117 %{
 7118   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7119 
 7120   ins_cost(200); // XXX
 7121   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7122   ins_encode %{
 7123     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7124   %}
 7125   ins_pipe(pipe_cmov_reg);
 7126 %}
 7127 
 7128 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7129 %{
 7130   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7131   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7132 
 7133   ins_cost(100); // XXX
 7134   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7135   ins_encode %{
 7136     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7137     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7138   %}
 7139   ins_pipe(ialu_reg);
 7140 %}
 7141 
 7142 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7143   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7144 
 7145   ins_cost(200); // XXX
 7146   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7147   ins_encode %{
 7148     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7149   %}
 7150   ins_pipe(pipe_cmov_reg);
 7151 %}
 7152 
 7153 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7154 %{
 7155   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 7156   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7157 
 7158   ins_cost(100); // XXX
 7159   format %{ "setbn$cop $dst\t# unsigned, int" %}
 7160   ins_encode %{
 7161     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7162     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7163   %}
 7164   ins_pipe(ialu_reg);
 7165 %}
 7166 
 7167 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7168   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7169   ins_cost(200);
 7170   expand %{
 7171     cmovI_regU(cop, cr, dst, src);
 7172   %}
 7173 %}
 7174 
 7175 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7176   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7177   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7178 
 7179   ins_cost(200); // XXX
 7180   format %{ "cmovpl  $dst, $src\n\t"
 7181             "cmovnel $dst, $src" %}
 7182   ins_encode %{
 7183     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7184     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7185   %}
 7186   ins_pipe(pipe_cmov_reg);
 7187 %}
 7188 
 7189 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7190 // inputs of the CMove
 7191 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7192   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7193   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 7194 
 7195   ins_cost(200); // XXX
 7196   format %{ "cmovpl  $dst, $src\n\t"
 7197             "cmovnel $dst, $src" %}
 7198   ins_encode %{
 7199     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7200     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7201   %}
 7202   ins_pipe(pipe_cmov_reg);
 7203 %}
 7204 
 7205 // Conditional move
 7206 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7207   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7208 
 7209   ins_cost(250); // XXX
 7210   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7211   ins_encode %{
 7212     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7213   %}
 7214   ins_pipe(pipe_cmov_mem);
 7215 %}
 7216 
 7217 // Conditional move
 7218 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7219 %{
 7220   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7221 
 7222   ins_cost(250); // XXX
 7223   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7224   ins_encode %{
 7225     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7226   %}
 7227   ins_pipe(pipe_cmov_mem);
 7228 %}
 7229 
 7230 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7231   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7232   ins_cost(250);
 7233   expand %{
 7234     cmovI_memU(cop, cr, dst, src);
 7235   %}
 7236 %}
 7237 
 7238 // Conditional move
 7239 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7240 %{
 7241   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7242 
 7243   ins_cost(200); // XXX
 7244   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7245   ins_encode %{
 7246     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7247   %}
 7248   ins_pipe(pipe_cmov_reg);
 7249 %}
 7250 
 7251 // Conditional move
 7252 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7253 %{
 7254   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7255 
 7256   ins_cost(200); // XXX
 7257   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7258   ins_encode %{
 7259     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7260   %}
 7261   ins_pipe(pipe_cmov_reg);
 7262 %}
 7263 
 7264 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7265   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7266   ins_cost(200);
 7267   expand %{
 7268     cmovN_regU(cop, cr, dst, src);
 7269   %}
 7270 %}
 7271 
 7272 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7273   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7274   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7275 
 7276   ins_cost(200); // XXX
 7277   format %{ "cmovpl  $dst, $src\n\t"
 7278             "cmovnel $dst, $src" %}
 7279   ins_encode %{
 7280     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7281     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7282   %}
 7283   ins_pipe(pipe_cmov_reg);
 7284 %}
 7285 
 7286 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7287 // inputs of the CMove
 7288 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7289   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7290   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 7291 
 7292   ins_cost(200); // XXX
 7293   format %{ "cmovpl  $dst, $src\n\t"
 7294             "cmovnel $dst, $src" %}
 7295   ins_encode %{
 7296     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 7297     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 7298   %}
 7299   ins_pipe(pipe_cmov_reg);
 7300 %}
 7301 
 7302 // Conditional move
 7303 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7304 %{
 7305   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7306 
 7307   ins_cost(200); // XXX
 7308   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7309   ins_encode %{
 7310     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7311   %}
 7312   ins_pipe(pipe_cmov_reg);  // XXX
 7313 %}
 7314 
 7315 // Conditional move
 7316 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7317 %{
 7318   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7319 
 7320   ins_cost(200); // XXX
 7321   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7322   ins_encode %{
 7323     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7324   %}
 7325   ins_pipe(pipe_cmov_reg); // XXX
 7326 %}
 7327 
 7328 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7329   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7330   ins_cost(200);
 7331   expand %{
 7332     cmovP_regU(cop, cr, dst, src);
 7333   %}
 7334 %}
 7335 
 7336 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7337   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7338   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7339 
 7340   ins_cost(200); // XXX
 7341   format %{ "cmovpq  $dst, $src\n\t"
 7342             "cmovneq $dst, $src" %}
 7343   ins_encode %{
 7344     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7345     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7346   %}
 7347   ins_pipe(pipe_cmov_reg);
 7348 %}
 7349 
 7350 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7351 // inputs of the CMove
 7352 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7353   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7354   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 7355 
 7356   ins_cost(200); // XXX
 7357   format %{ "cmovpq  $dst, $src\n\t"
 7358             "cmovneq $dst, $src" %}
 7359   ins_encode %{
 7360     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7361     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7362   %}
 7363   ins_pipe(pipe_cmov_reg);
 7364 %}
 7365 
 7366 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7367 // correctly meets the two pointer arguments; one is an incoming
 7368 // register but the other is a memory operand.  ALSO appears to
 7369 // be buggy with implicit null checks.
 7370 //
 7371 //// Conditional move
 7372 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7373 //%{
 7374 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7375 //  ins_cost(250);
 7376 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7377 //  opcode(0x0F,0x40);
 7378 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7379 //  ins_pipe( pipe_cmov_mem );
 7380 //%}
 7381 //
 7382 //// Conditional move
 7383 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7384 //%{
 7385 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7386 //  ins_cost(250);
 7387 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7388 //  opcode(0x0F,0x40);
 7389 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7390 //  ins_pipe( pipe_cmov_mem );
 7391 //%}
 7392 
 7393 instruct cmovL_imm_01(rRegL dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 7394 %{
 7395   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7396   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7397 
 7398   ins_cost(100); // XXX
 7399   format %{ "setbn$cop $dst\t# signed, long" %}
 7400   ins_encode %{
 7401     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7402     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7403   %}
 7404   ins_pipe(ialu_reg);
 7405 %}
 7406 
 7407 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7408 %{
 7409   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7410 
 7411   ins_cost(200); // XXX
 7412   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7413   ins_encode %{
 7414     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7415   %}
 7416   ins_pipe(pipe_cmov_reg);  // XXX
 7417 %}
 7418 
 7419 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7420 %{
 7421   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7422 
 7423   ins_cost(200); // XXX
 7424   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7425   ins_encode %{
 7426     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7427   %}
 7428   ins_pipe(pipe_cmov_mem);  // XXX
 7429 %}
 7430 
 7431 instruct cmovL_imm_01U(rRegL dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 7432 %{
 7433   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7434   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7435 
 7436   ins_cost(100); // XXX
 7437   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7438   ins_encode %{
 7439     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7440     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7441   %}
 7442   ins_pipe(ialu_reg);
 7443 %}
 7444 
 7445 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7446 %{
 7447   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7448 
 7449   ins_cost(200); // XXX
 7450   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7451   ins_encode %{
 7452     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7453   %}
 7454   ins_pipe(pipe_cmov_reg); // XXX
 7455 %}
 7456 
 7457 instruct cmovL_imm_01UCF(rRegL dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 7458 %{
 7459   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 7460   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7461 
 7462   ins_cost(100); // XXX
 7463   format %{ "setbn$cop $dst\t# unsigned, long" %}
 7464   ins_encode %{
 7465     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 7466     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 7467   %}
 7468   ins_pipe(ialu_reg);
 7469 %}
 7470 
 7471 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7472   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7473   ins_cost(200);
 7474   expand %{
 7475     cmovL_regU(cop, cr, dst, src);
 7476   %}
 7477 %}
 7478 
 7479 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7480   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 7481   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7482 
 7483   ins_cost(200); // XXX
 7484   format %{ "cmovpq  $dst, $src\n\t"
 7485             "cmovneq $dst, $src" %}
 7486   ins_encode %{
 7487     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7488     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7489   %}
 7490   ins_pipe(pipe_cmov_reg);
 7491 %}
 7492 
 7493 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 7494 // inputs of the CMove
 7495 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7496   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 7497   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 7498 
 7499   ins_cost(200); // XXX
 7500   format %{ "cmovpq  $dst, $src\n\t"
 7501             "cmovneq $dst, $src" %}
 7502   ins_encode %{
 7503     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 7504     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 7505   %}
 7506   ins_pipe(pipe_cmov_reg);
 7507 %}
 7508 
 7509 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7510 %{
 7511   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7512 
 7513   ins_cost(200); // XXX
 7514   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7515   ins_encode %{
 7516     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7517   %}
 7518   ins_pipe(pipe_cmov_mem); // XXX
 7519 %}
 7520 
 7521 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7522   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7523   ins_cost(200);
 7524   expand %{
 7525     cmovL_memU(cop, cr, dst, src);
 7526   %}
 7527 %}
 7528 
 7529 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7530 %{
 7531   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7532 
 7533   ins_cost(200); // XXX
 7534   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7535             "movss     $dst, $src\n"
 7536     "skip:" %}
 7537   ins_encode %{
 7538     Label Lskip;
 7539     // Invert sense of branch from sense of CMOV
 7540     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7541     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7542     __ bind(Lskip);
 7543   %}
 7544   ins_pipe(pipe_slow);
 7545 %}
 7546 
 7547 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7548 // %{
 7549 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7550 
 7551 //   ins_cost(200); // XXX
 7552 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7553 //             "movss     $dst, $src\n"
 7554 //     "skip:" %}
 7555 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7556 //   ins_pipe(pipe_slow);
 7557 // %}
 7558 
 7559 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7560 %{
 7561   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7562 
 7563   ins_cost(200); // XXX
 7564   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7565             "movss     $dst, $src\n"
 7566     "skip:" %}
 7567   ins_encode %{
 7568     Label Lskip;
 7569     // Invert sense of branch from sense of CMOV
 7570     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7571     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7572     __ bind(Lskip);
 7573   %}
 7574   ins_pipe(pipe_slow);
 7575 %}
 7576 
 7577 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7578   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7579   ins_cost(200);
 7580   expand %{
 7581     cmovF_regU(cop, cr, dst, src);
 7582   %}
 7583 %}
 7584 
 7585 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7586 %{
 7587   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7588 
 7589   ins_cost(200); // XXX
 7590   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7591             "movsd     $dst, $src\n"
 7592     "skip:" %}
 7593   ins_encode %{
 7594     Label Lskip;
 7595     // Invert sense of branch from sense of CMOV
 7596     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7597     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7598     __ bind(Lskip);
 7599   %}
 7600   ins_pipe(pipe_slow);
 7601 %}
 7602 
 7603 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7604 %{
 7605   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7606 
 7607   ins_cost(200); // XXX
 7608   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7609             "movsd     $dst, $src\n"
 7610     "skip:" %}
 7611   ins_encode %{
 7612     Label Lskip;
 7613     // Invert sense of branch from sense of CMOV
 7614     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7615     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7616     __ bind(Lskip);
 7617   %}
 7618   ins_pipe(pipe_slow);
 7619 %}
 7620 
 7621 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7622   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7623   ins_cost(200);
 7624   expand %{
 7625     cmovD_regU(cop, cr, dst, src);
 7626   %}
 7627 %}
 7628 
 7629 //----------Arithmetic Instructions--------------------------------------------
 7630 //----------Addition Instructions----------------------------------------------
 7631 
 7632 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7633 %{
 7634   match(Set dst (AddI dst src));
 7635   effect(KILL cr);
 7636 
 7637   format %{ "addl    $dst, $src\t# int" %}
 7638   ins_encode %{
 7639     __ addl($dst$$Register, $src$$Register);
 7640   %}
 7641   ins_pipe(ialu_reg_reg);
 7642 %}
 7643 
 7644 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7645 %{
 7646   match(Set dst (AddI dst src));
 7647   effect(KILL cr);
 7648 
 7649   format %{ "addl    $dst, $src\t# int" %}
 7650   ins_encode %{
 7651     __ addl($dst$$Register, $src$$constant);
 7652   %}
 7653   ins_pipe( ialu_reg );
 7654 %}
 7655 
 7656 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7657 %{
 7658   match(Set dst (AddI dst (LoadI src)));
 7659   effect(KILL cr);
 7660 
 7661   ins_cost(150); // XXX
 7662   format %{ "addl    $dst, $src\t# int" %}
 7663   ins_encode %{
 7664     __ addl($dst$$Register, $src$$Address);
 7665   %}
 7666   ins_pipe(ialu_reg_mem);
 7667 %}
 7668 
 7669 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7670 %{
 7671   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7672   effect(KILL cr);
 7673 
 7674   ins_cost(150); // XXX
 7675   format %{ "addl    $dst, $src\t# int" %}
 7676   ins_encode %{
 7677     __ addl($dst$$Address, $src$$Register);
 7678   %}
 7679   ins_pipe(ialu_mem_reg);
 7680 %}
 7681 
 7682 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7683 %{
 7684   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7685   effect(KILL cr);
 7686 
 7687   ins_cost(125); // XXX
 7688   format %{ "addl    $dst, $src\t# int" %}
 7689   ins_encode %{
 7690     __ addl($dst$$Address, $src$$constant);
 7691   %}
 7692   ins_pipe(ialu_mem_imm);
 7693 %}
 7694 
 7695 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7696 %{
 7697   predicate(UseIncDec);
 7698   match(Set dst (AddI dst src));
 7699   effect(KILL cr);
 7700 
 7701   format %{ "incl    $dst\t# int" %}
 7702   ins_encode %{
 7703     __ incrementl($dst$$Register);
 7704   %}
 7705   ins_pipe(ialu_reg);
 7706 %}
 7707 
 7708 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7709 %{
 7710   predicate(UseIncDec);
 7711   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7712   effect(KILL cr);
 7713 
 7714   ins_cost(125); // XXX
 7715   format %{ "incl    $dst\t# int" %}
 7716   ins_encode %{
 7717     __ incrementl($dst$$Address);
 7718   %}
 7719   ins_pipe(ialu_mem_imm);
 7720 %}
 7721 
 7722 // XXX why does that use AddI
 7723 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7724 %{
 7725   predicate(UseIncDec);
 7726   match(Set dst (AddI dst src));
 7727   effect(KILL cr);
 7728 
 7729   format %{ "decl    $dst\t# int" %}
 7730   ins_encode %{
 7731     __ decrementl($dst$$Register);
 7732   %}
 7733   ins_pipe(ialu_reg);
 7734 %}
 7735 
 7736 // XXX why does that use AddI
 7737 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7738 %{
 7739   predicate(UseIncDec);
 7740   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7741   effect(KILL cr);
 7742 
 7743   ins_cost(125); // XXX
 7744   format %{ "decl    $dst\t# int" %}
 7745   ins_encode %{
 7746     __ decrementl($dst$$Address);
 7747   %}
 7748   ins_pipe(ialu_mem_imm);
 7749 %}
 7750 
 7751 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7752 %{
 7753   predicate(VM_Version::supports_fast_2op_lea());
 7754   match(Set dst (AddI (LShiftI index scale) disp));
 7755 
 7756   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7757   ins_encode %{
 7758     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7759     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7760   %}
 7761   ins_pipe(ialu_reg_reg);
 7762 %}
 7763 
 7764 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7765 %{
 7766   predicate(VM_Version::supports_fast_3op_lea());
 7767   match(Set dst (AddI (AddI base index) disp));
 7768 
 7769   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7770   ins_encode %{
 7771     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7772   %}
 7773   ins_pipe(ialu_reg_reg);
 7774 %}
 7775 
 7776 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7777 %{
 7778   predicate(VM_Version::supports_fast_2op_lea());
 7779   match(Set dst (AddI base (LShiftI index scale)));
 7780 
 7781   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7782   ins_encode %{
 7783     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7784     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7785   %}
 7786   ins_pipe(ialu_reg_reg);
 7787 %}
 7788 
 7789 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7790 %{
 7791   predicate(VM_Version::supports_fast_3op_lea());
 7792   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7793 
 7794   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7795   ins_encode %{
 7796     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7797     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7798   %}
 7799   ins_pipe(ialu_reg_reg);
 7800 %}
 7801 
 7802 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7803 %{
 7804   match(Set dst (AddL dst src));
 7805   effect(KILL cr);
 7806 
 7807   format %{ "addq    $dst, $src\t# long" %}
 7808   ins_encode %{
 7809     __ addq($dst$$Register, $src$$Register);
 7810   %}
 7811   ins_pipe(ialu_reg_reg);
 7812 %}
 7813 
 7814 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7815 %{
 7816   match(Set dst (AddL dst src));
 7817   effect(KILL cr);
 7818 
 7819   format %{ "addq    $dst, $src\t# long" %}
 7820   ins_encode %{
 7821     __ addq($dst$$Register, $src$$constant);
 7822   %}
 7823   ins_pipe( ialu_reg );
 7824 %}
 7825 
 7826 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7827 %{
 7828   match(Set dst (AddL dst (LoadL src)));
 7829   effect(KILL cr);
 7830 
 7831   ins_cost(150); // XXX
 7832   format %{ "addq    $dst, $src\t# long" %}
 7833   ins_encode %{
 7834     __ addq($dst$$Register, $src$$Address);
 7835   %}
 7836   ins_pipe(ialu_reg_mem);
 7837 %}
 7838 
 7839 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7840 %{
 7841   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7842   effect(KILL cr);
 7843 
 7844   ins_cost(150); // XXX
 7845   format %{ "addq    $dst, $src\t# long" %}
 7846   ins_encode %{
 7847     __ addq($dst$$Address, $src$$Register);
 7848   %}
 7849   ins_pipe(ialu_mem_reg);
 7850 %}
 7851 
 7852 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7853 %{
 7854   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7855   effect(KILL cr);
 7856 
 7857   ins_cost(125); // XXX
 7858   format %{ "addq    $dst, $src\t# long" %}
 7859   ins_encode %{
 7860     __ addq($dst$$Address, $src$$constant);
 7861   %}
 7862   ins_pipe(ialu_mem_imm);
 7863 %}
 7864 
 7865 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7866 %{
 7867   predicate(UseIncDec);
 7868   match(Set dst (AddL dst src));
 7869   effect(KILL cr);
 7870 
 7871   format %{ "incq    $dst\t# long" %}
 7872   ins_encode %{
 7873     __ incrementq($dst$$Register);
 7874   %}
 7875   ins_pipe(ialu_reg);
 7876 %}
 7877 
 7878 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7879 %{
 7880   predicate(UseIncDec);
 7881   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7882   effect(KILL cr);
 7883 
 7884   ins_cost(125); // XXX
 7885   format %{ "incq    $dst\t# long" %}
 7886   ins_encode %{
 7887     __ incrementq($dst$$Address);
 7888   %}
 7889   ins_pipe(ialu_mem_imm);
 7890 %}
 7891 
 7892 // XXX why does that use AddL
 7893 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7894 %{
 7895   predicate(UseIncDec);
 7896   match(Set dst (AddL dst src));
 7897   effect(KILL cr);
 7898 
 7899   format %{ "decq    $dst\t# long" %}
 7900   ins_encode %{
 7901     __ decrementq($dst$$Register);
 7902   %}
 7903   ins_pipe(ialu_reg);
 7904 %}
 7905 
 7906 // XXX why does that use AddL
 7907 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7908 %{
 7909   predicate(UseIncDec);
 7910   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7911   effect(KILL cr);
 7912 
 7913   ins_cost(125); // XXX
 7914   format %{ "decq    $dst\t# long" %}
 7915   ins_encode %{
 7916     __ decrementq($dst$$Address);
 7917   %}
 7918   ins_pipe(ialu_mem_imm);
 7919 %}
 7920 
 7921 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7922 %{
 7923   predicate(VM_Version::supports_fast_2op_lea());
 7924   match(Set dst (AddL (LShiftL index scale) disp));
 7925 
 7926   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7927   ins_encode %{
 7928     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7929     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7930   %}
 7931   ins_pipe(ialu_reg_reg);
 7932 %}
 7933 
 7934 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7935 %{
 7936   predicate(VM_Version::supports_fast_3op_lea());
 7937   match(Set dst (AddL (AddL base index) disp));
 7938 
 7939   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7940   ins_encode %{
 7941     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7942   %}
 7943   ins_pipe(ialu_reg_reg);
 7944 %}
 7945 
 7946 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7947 %{
 7948   predicate(VM_Version::supports_fast_2op_lea());
 7949   match(Set dst (AddL base (LShiftL index scale)));
 7950 
 7951   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7952   ins_encode %{
 7953     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7954     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7955   %}
 7956   ins_pipe(ialu_reg_reg);
 7957 %}
 7958 
 7959 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7960 %{
 7961   predicate(VM_Version::supports_fast_3op_lea());
 7962   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7963 
 7964   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7965   ins_encode %{
 7966     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7967     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7968   %}
 7969   ins_pipe(ialu_reg_reg);
 7970 %}
 7971 
 7972 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7973 %{
 7974   match(Set dst (AddP dst src));
 7975   effect(KILL cr);
 7976 
 7977   format %{ "addq    $dst, $src\t# ptr" %}
 7978   ins_encode %{
 7979     __ addq($dst$$Register, $src$$Register);
 7980   %}
 7981   ins_pipe(ialu_reg_reg);
 7982 %}
 7983 
 7984 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7985 %{
 7986   match(Set dst (AddP dst src));
 7987   effect(KILL cr);
 7988 
 7989   format %{ "addq    $dst, $src\t# ptr" %}
 7990   ins_encode %{
 7991     __ addq($dst$$Register, $src$$constant);
 7992   %}
 7993   ins_pipe( ialu_reg );
 7994 %}
 7995 
 7996 // XXX addP mem ops ????
 7997 
 7998 instruct checkCastPP(rRegP dst)
 7999 %{
 8000   match(Set dst (CheckCastPP dst));
 8001 
 8002   size(0);
 8003   format %{ "# checkcastPP of $dst" %}
 8004   ins_encode(/* empty encoding */);
 8005   ins_pipe(empty);
 8006 %}
 8007 
 8008 instruct castPP(rRegP dst)
 8009 %{
 8010   match(Set dst (CastPP dst));
 8011 
 8012   size(0);
 8013   format %{ "# castPP of $dst" %}
 8014   ins_encode(/* empty encoding */);
 8015   ins_pipe(empty);
 8016 %}
 8017 
 8018 instruct castII(rRegI dst)
 8019 %{
 8020   match(Set dst (CastII dst));
 8021 
 8022   size(0);
 8023   format %{ "# castII of $dst" %}
 8024   ins_encode(/* empty encoding */);
 8025   ins_cost(0);
 8026   ins_pipe(empty);
 8027 %}
 8028 
 8029 instruct castLL(rRegL dst)
 8030 %{
 8031   match(Set dst (CastLL dst));
 8032 
 8033   size(0);
 8034   format %{ "# castLL of $dst" %}
 8035   ins_encode(/* empty encoding */);
 8036   ins_cost(0);
 8037   ins_pipe(empty);
 8038 %}
 8039 
 8040 instruct castFF(regF dst)
 8041 %{
 8042   match(Set dst (CastFF dst));
 8043 
 8044   size(0);
 8045   format %{ "# castFF of $dst" %}
 8046   ins_encode(/* empty encoding */);
 8047   ins_cost(0);
 8048   ins_pipe(empty);
 8049 %}
 8050 
 8051 instruct castDD(regD dst)
 8052 %{
 8053   match(Set dst (CastDD dst));
 8054 
 8055   size(0);
 8056   format %{ "# castDD of $dst" %}
 8057   ins_encode(/* empty encoding */);
 8058   ins_cost(0);
 8059   ins_pipe(empty);
 8060 %}
 8061 
 8062 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 8063 instruct compareAndSwapP(rRegI res,
 8064                          memory mem_ptr,
 8065                          rax_RegP oldval, rRegP newval,
 8066                          rFlagsReg cr)
 8067 %{
 8068   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8069   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 8070   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 8071   effect(KILL cr, KILL oldval);
 8072 
 8073   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8074             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8075             "sete    $res\n\t"
 8076             "movzbl  $res, $res" %}
 8077   ins_encode %{
 8078     __ lock();
 8079     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8080     __ sete($res$$Register);
 8081     __ movzbl($res$$Register, $res$$Register);
 8082   %}
 8083   ins_pipe( pipe_cmpxchg );
 8084 %}
 8085 
 8086 instruct compareAndSwapL(rRegI res,
 8087                          memory mem_ptr,
 8088                          rax_RegL oldval, rRegL newval,
 8089                          rFlagsReg cr)
 8090 %{
 8091   predicate(VM_Version::supports_cx8());
 8092   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 8093   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 8094   effect(KILL cr, KILL oldval);
 8095 
 8096   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8097             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8098             "sete    $res\n\t"
 8099             "movzbl  $res, $res" %}
 8100   ins_encode %{
 8101     __ lock();
 8102     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8103     __ sete($res$$Register);
 8104     __ movzbl($res$$Register, $res$$Register);
 8105   %}
 8106   ins_pipe( pipe_cmpxchg );
 8107 %}
 8108 
 8109 instruct compareAndSwapI(rRegI res,
 8110                          memory mem_ptr,
 8111                          rax_RegI oldval, rRegI newval,
 8112                          rFlagsReg cr)
 8113 %{
 8114   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 8115   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 8116   effect(KILL cr, KILL oldval);
 8117 
 8118   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8119             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8120             "sete    $res\n\t"
 8121             "movzbl  $res, $res" %}
 8122   ins_encode %{
 8123     __ lock();
 8124     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8125     __ sete($res$$Register);
 8126     __ movzbl($res$$Register, $res$$Register);
 8127   %}
 8128   ins_pipe( pipe_cmpxchg );
 8129 %}
 8130 
 8131 instruct compareAndSwapB(rRegI res,
 8132                          memory mem_ptr,
 8133                          rax_RegI oldval, rRegI newval,
 8134                          rFlagsReg cr)
 8135 %{
 8136   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 8137   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 8138   effect(KILL cr, KILL oldval);
 8139 
 8140   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8141             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8142             "sete    $res\n\t"
 8143             "movzbl  $res, $res" %}
 8144   ins_encode %{
 8145     __ lock();
 8146     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8147     __ sete($res$$Register);
 8148     __ movzbl($res$$Register, $res$$Register);
 8149   %}
 8150   ins_pipe( pipe_cmpxchg );
 8151 %}
 8152 
 8153 instruct compareAndSwapS(rRegI res,
 8154                          memory mem_ptr,
 8155                          rax_RegI oldval, rRegI newval,
 8156                          rFlagsReg cr)
 8157 %{
 8158   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 8159   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8160   effect(KILL cr, KILL oldval);
 8161 
 8162   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8163             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8164             "sete    $res\n\t"
 8165             "movzbl  $res, $res" %}
 8166   ins_encode %{
 8167     __ lock();
 8168     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8169     __ sete($res$$Register);
 8170     __ movzbl($res$$Register, $res$$Register);
 8171   %}
 8172   ins_pipe( pipe_cmpxchg );
 8173 %}
 8174 
 8175 instruct compareAndSwapN(rRegI res,
 8176                           memory mem_ptr,
 8177                           rax_RegN oldval, rRegN newval,
 8178                           rFlagsReg cr) %{
 8179   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8180   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8181   effect(KILL cr, KILL oldval);
 8182 
 8183   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8184             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8185             "sete    $res\n\t"
 8186             "movzbl  $res, $res" %}
 8187   ins_encode %{
 8188     __ lock();
 8189     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8190     __ sete($res$$Register);
 8191     __ movzbl($res$$Register, $res$$Register);
 8192   %}
 8193   ins_pipe( pipe_cmpxchg );
 8194 %}
 8195 
 8196 instruct compareAndExchangeB(
 8197                          memory mem_ptr,
 8198                          rax_RegI oldval, rRegI newval,
 8199                          rFlagsReg cr)
 8200 %{
 8201   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8202   effect(KILL cr);
 8203 
 8204   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8205             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8206   ins_encode %{
 8207     __ lock();
 8208     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8209   %}
 8210   ins_pipe( pipe_cmpxchg );
 8211 %}
 8212 
 8213 instruct compareAndExchangeS(
 8214                          memory mem_ptr,
 8215                          rax_RegI oldval, rRegI newval,
 8216                          rFlagsReg cr)
 8217 %{
 8218   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8219   effect(KILL cr);
 8220 
 8221   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8222             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8223   ins_encode %{
 8224     __ lock();
 8225     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8226   %}
 8227   ins_pipe( pipe_cmpxchg );
 8228 %}
 8229 
 8230 instruct compareAndExchangeI(
 8231                          memory mem_ptr,
 8232                          rax_RegI oldval, rRegI newval,
 8233                          rFlagsReg cr)
 8234 %{
 8235   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8236   effect(KILL cr);
 8237 
 8238   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8239             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8240   ins_encode %{
 8241     __ lock();
 8242     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8243   %}
 8244   ins_pipe( pipe_cmpxchg );
 8245 %}
 8246 
 8247 instruct compareAndExchangeL(
 8248                          memory mem_ptr,
 8249                          rax_RegL oldval, rRegL newval,
 8250                          rFlagsReg cr)
 8251 %{
 8252   predicate(VM_Version::supports_cx8());
 8253   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8254   effect(KILL cr);
 8255 
 8256   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8257             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8258   ins_encode %{
 8259     __ lock();
 8260     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8261   %}
 8262   ins_pipe( pipe_cmpxchg );
 8263 %}
 8264 
 8265 instruct compareAndExchangeN(
 8266                           memory mem_ptr,
 8267                           rax_RegN oldval, rRegN newval,
 8268                           rFlagsReg cr) %{
 8269   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8270   effect(KILL cr);
 8271 
 8272   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8273             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8274   ins_encode %{
 8275     __ lock();
 8276     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8277   %}
 8278   ins_pipe( pipe_cmpxchg );
 8279 %}
 8280 
 8281 instruct compareAndExchangeP(
 8282                          memory mem_ptr,
 8283                          rax_RegP oldval, rRegP newval,
 8284                          rFlagsReg cr)
 8285 %{
 8286   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8287   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8288   effect(KILL cr);
 8289 
 8290   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8291             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8292   ins_encode %{
 8293     __ lock();
 8294     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8295   %}
 8296   ins_pipe( pipe_cmpxchg );
 8297 %}
 8298 
 8299 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8300   predicate(n->as_LoadStore()->result_not_used());
 8301   match(Set dummy (GetAndAddB mem add));
 8302   effect(KILL cr);
 8303   format %{ "ADDB  [$mem],$add" %}
 8304   ins_encode %{
 8305     __ lock();
 8306     __ addb($mem$$Address, $add$$constant);
 8307   %}
 8308   ins_pipe( pipe_cmpxchg );
 8309 %}
 8310 
 8311 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8312   match(Set newval (GetAndAddB mem newval));
 8313   effect(KILL cr);
 8314   format %{ "XADDB  [$mem],$newval" %}
 8315   ins_encode %{
 8316     __ lock();
 8317     __ xaddb($mem$$Address, $newval$$Register);
 8318   %}
 8319   ins_pipe( pipe_cmpxchg );
 8320 %}
 8321 
 8322 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8323   predicate(n->as_LoadStore()->result_not_used());
 8324   match(Set dummy (GetAndAddS mem add));
 8325   effect(KILL cr);
 8326   format %{ "ADDW  [$mem],$add" %}
 8327   ins_encode %{
 8328     __ lock();
 8329     __ addw($mem$$Address, $add$$constant);
 8330   %}
 8331   ins_pipe( pipe_cmpxchg );
 8332 %}
 8333 
 8334 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8335   match(Set newval (GetAndAddS mem newval));
 8336   effect(KILL cr);
 8337   format %{ "XADDW  [$mem],$newval" %}
 8338   ins_encode %{
 8339     __ lock();
 8340     __ xaddw($mem$$Address, $newval$$Register);
 8341   %}
 8342   ins_pipe( pipe_cmpxchg );
 8343 %}
 8344 
 8345 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8346   predicate(n->as_LoadStore()->result_not_used());
 8347   match(Set dummy (GetAndAddI mem add));
 8348   effect(KILL cr);
 8349   format %{ "ADDL  [$mem],$add" %}
 8350   ins_encode %{
 8351     __ lock();
 8352     __ addl($mem$$Address, $add$$constant);
 8353   %}
 8354   ins_pipe( pipe_cmpxchg );
 8355 %}
 8356 
 8357 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8358   match(Set newval (GetAndAddI mem newval));
 8359   effect(KILL cr);
 8360   format %{ "XADDL  [$mem],$newval" %}
 8361   ins_encode %{
 8362     __ lock();
 8363     __ xaddl($mem$$Address, $newval$$Register);
 8364   %}
 8365   ins_pipe( pipe_cmpxchg );
 8366 %}
 8367 
 8368 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8369   predicate(n->as_LoadStore()->result_not_used());
 8370   match(Set dummy (GetAndAddL mem add));
 8371   effect(KILL cr);
 8372   format %{ "ADDQ  [$mem],$add" %}
 8373   ins_encode %{
 8374     __ lock();
 8375     __ addq($mem$$Address, $add$$constant);
 8376   %}
 8377   ins_pipe( pipe_cmpxchg );
 8378 %}
 8379 
 8380 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8381   match(Set newval (GetAndAddL mem newval));
 8382   effect(KILL cr);
 8383   format %{ "XADDQ  [$mem],$newval" %}
 8384   ins_encode %{
 8385     __ lock();
 8386     __ xaddq($mem$$Address, $newval$$Register);
 8387   %}
 8388   ins_pipe( pipe_cmpxchg );
 8389 %}
 8390 
 8391 instruct xchgB( memory mem, rRegI newval) %{
 8392   match(Set newval (GetAndSetB mem newval));
 8393   format %{ "XCHGB  $newval,[$mem]" %}
 8394   ins_encode %{
 8395     __ xchgb($newval$$Register, $mem$$Address);
 8396   %}
 8397   ins_pipe( pipe_cmpxchg );
 8398 %}
 8399 
 8400 instruct xchgS( memory mem, rRegI newval) %{
 8401   match(Set newval (GetAndSetS mem newval));
 8402   format %{ "XCHGW  $newval,[$mem]" %}
 8403   ins_encode %{
 8404     __ xchgw($newval$$Register, $mem$$Address);
 8405   %}
 8406   ins_pipe( pipe_cmpxchg );
 8407 %}
 8408 
 8409 instruct xchgI( memory mem, rRegI newval) %{
 8410   match(Set newval (GetAndSetI mem newval));
 8411   format %{ "XCHGL  $newval,[$mem]" %}
 8412   ins_encode %{
 8413     __ xchgl($newval$$Register, $mem$$Address);
 8414   %}
 8415   ins_pipe( pipe_cmpxchg );
 8416 %}
 8417 
 8418 instruct xchgL( memory mem, rRegL newval) %{
 8419   match(Set newval (GetAndSetL mem newval));
 8420   format %{ "XCHGL  $newval,[$mem]" %}
 8421   ins_encode %{
 8422     __ xchgq($newval$$Register, $mem$$Address);
 8423   %}
 8424   ins_pipe( pipe_cmpxchg );
 8425 %}
 8426 
 8427 instruct xchgP( memory mem, rRegP newval) %{
 8428   match(Set newval (GetAndSetP mem newval));
 8429   predicate(n->as_LoadStore()->barrier_data() == 0);
 8430   format %{ "XCHGQ  $newval,[$mem]" %}
 8431   ins_encode %{
 8432     __ xchgq($newval$$Register, $mem$$Address);
 8433   %}
 8434   ins_pipe( pipe_cmpxchg );
 8435 %}
 8436 
 8437 instruct xchgN( memory mem, rRegN newval) %{
 8438   match(Set newval (GetAndSetN mem newval));
 8439   format %{ "XCHGL  $newval,$mem]" %}
 8440   ins_encode %{
 8441     __ xchgl($newval$$Register, $mem$$Address);
 8442   %}
 8443   ins_pipe( pipe_cmpxchg );
 8444 %}
 8445 
 8446 //----------Abs Instructions-------------------------------------------
 8447 
 8448 // Integer Absolute Instructions
 8449 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8450 %{
 8451   match(Set dst (AbsI src));
 8452   effect(TEMP dst, TEMP tmp, KILL cr);
 8453   format %{ "movl $tmp, $src\n\t"
 8454             "sarl $tmp, 31\n\t"
 8455             "movl $dst, $src\n\t"
 8456             "xorl $dst, $tmp\n\t"
 8457             "subl $dst, $tmp\n"
 8458           %}
 8459   ins_encode %{
 8460     __ movl($tmp$$Register, $src$$Register);
 8461     __ sarl($tmp$$Register, 31);
 8462     __ movl($dst$$Register, $src$$Register);
 8463     __ xorl($dst$$Register, $tmp$$Register);
 8464     __ subl($dst$$Register, $tmp$$Register);
 8465   %}
 8466 
 8467   ins_pipe(ialu_reg_reg);
 8468 %}
 8469 
 8470 // Long Absolute Instructions
 8471 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8472 %{
 8473   match(Set dst (AbsL src));
 8474   effect(TEMP dst, TEMP tmp, KILL cr);
 8475   format %{ "movq $tmp, $src\n\t"
 8476             "sarq $tmp, 63\n\t"
 8477             "movq $dst, $src\n\t"
 8478             "xorq $dst, $tmp\n\t"
 8479             "subq $dst, $tmp\n"
 8480           %}
 8481   ins_encode %{
 8482     __ movq($tmp$$Register, $src$$Register);
 8483     __ sarq($tmp$$Register, 63);
 8484     __ movq($dst$$Register, $src$$Register);
 8485     __ xorq($dst$$Register, $tmp$$Register);
 8486     __ subq($dst$$Register, $tmp$$Register);
 8487   %}
 8488 
 8489   ins_pipe(ialu_reg_reg);
 8490 %}
 8491 
 8492 //----------Subtraction Instructions-------------------------------------------
 8493 
 8494 // Integer Subtraction Instructions
 8495 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8496 %{
 8497   match(Set dst (SubI dst src));
 8498   effect(KILL cr);
 8499 
 8500   format %{ "subl    $dst, $src\t# int" %}
 8501   ins_encode %{
 8502     __ subl($dst$$Register, $src$$Register);
 8503   %}
 8504   ins_pipe(ialu_reg_reg);
 8505 %}
 8506 
 8507 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8508 %{
 8509   match(Set dst (SubI dst (LoadI src)));
 8510   effect(KILL cr);
 8511 
 8512   ins_cost(150);
 8513   format %{ "subl    $dst, $src\t# int" %}
 8514   ins_encode %{
 8515     __ subl($dst$$Register, $src$$Address);
 8516   %}
 8517   ins_pipe(ialu_reg_mem);
 8518 %}
 8519 
 8520 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8521 %{
 8522   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8523   effect(KILL cr);
 8524 
 8525   ins_cost(150);
 8526   format %{ "subl    $dst, $src\t# int" %}
 8527   ins_encode %{
 8528     __ subl($dst$$Address, $src$$Register);
 8529   %}
 8530   ins_pipe(ialu_mem_reg);
 8531 %}
 8532 
 8533 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8534 %{
 8535   match(Set dst (SubL dst src));
 8536   effect(KILL cr);
 8537 
 8538   format %{ "subq    $dst, $src\t# long" %}
 8539   ins_encode %{
 8540     __ subq($dst$$Register, $src$$Register);
 8541   %}
 8542   ins_pipe(ialu_reg_reg);
 8543 %}
 8544 
 8545 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8546 %{
 8547   match(Set dst (SubL dst (LoadL src)));
 8548   effect(KILL cr);
 8549 
 8550   ins_cost(150);
 8551   format %{ "subq    $dst, $src\t# long" %}
 8552   ins_encode %{
 8553     __ subq($dst$$Register, $src$$Address);
 8554   %}
 8555   ins_pipe(ialu_reg_mem);
 8556 %}
 8557 
 8558 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8559 %{
 8560   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8561   effect(KILL cr);
 8562 
 8563   ins_cost(150);
 8564   format %{ "subq    $dst, $src\t# long" %}
 8565   ins_encode %{
 8566     __ subq($dst$$Address, $src$$Register);
 8567   %}
 8568   ins_pipe(ialu_mem_reg);
 8569 %}
 8570 
 8571 // Subtract from a pointer
 8572 // XXX hmpf???
 8573 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8574 %{
 8575   match(Set dst (AddP dst (SubI zero src)));
 8576   effect(KILL cr);
 8577 
 8578   format %{ "subq    $dst, $src\t# ptr - int" %}
 8579   opcode(0x2B);
 8580   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8581   ins_pipe(ialu_reg_reg);
 8582 %}
 8583 
 8584 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8585 %{
 8586   match(Set dst (SubI zero dst));
 8587   effect(KILL cr);
 8588 
 8589   format %{ "negl    $dst\t# int" %}
 8590   ins_encode %{
 8591     __ negl($dst$$Register);
 8592   %}
 8593   ins_pipe(ialu_reg);
 8594 %}
 8595 
 8596 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8597 %{
 8598   match(Set dst (NegI dst));
 8599   effect(KILL cr);
 8600 
 8601   format %{ "negl    $dst\t# int" %}
 8602   ins_encode %{
 8603     __ negl($dst$$Register);
 8604   %}
 8605   ins_pipe(ialu_reg);
 8606 %}
 8607 
 8608 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8609 %{
 8610   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8611   effect(KILL cr);
 8612 
 8613   format %{ "negl    $dst\t# int" %}
 8614   ins_encode %{
 8615     __ negl($dst$$Address);
 8616   %}
 8617   ins_pipe(ialu_reg);
 8618 %}
 8619 
 8620 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8621 %{
 8622   match(Set dst (SubL zero dst));
 8623   effect(KILL cr);
 8624 
 8625   format %{ "negq    $dst\t# long" %}
 8626   ins_encode %{
 8627     __ negq($dst$$Register);
 8628   %}
 8629   ins_pipe(ialu_reg);
 8630 %}
 8631 
 8632 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8633 %{
 8634   match(Set dst (NegL dst));
 8635   effect(KILL cr);
 8636 
 8637   format %{ "negq    $dst\t# int" %}
 8638   ins_encode %{
 8639     __ negq($dst$$Register);
 8640   %}
 8641   ins_pipe(ialu_reg);
 8642 %}
 8643 
 8644 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8645 %{
 8646   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8647   effect(KILL cr);
 8648 
 8649   format %{ "negq    $dst\t# long" %}
 8650   ins_encode %{
 8651     __ negq($dst$$Address);
 8652   %}
 8653   ins_pipe(ialu_reg);
 8654 %}
 8655 
 8656 //----------Multiplication/Division Instructions-------------------------------
 8657 // Integer Multiplication Instructions
 8658 // Multiply Register
 8659 
 8660 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8661 %{
 8662   match(Set dst (MulI dst src));
 8663   effect(KILL cr);
 8664 
 8665   ins_cost(300);
 8666   format %{ "imull   $dst, $src\t# int" %}
 8667   ins_encode %{
 8668     __ imull($dst$$Register, $src$$Register);
 8669   %}
 8670   ins_pipe(ialu_reg_reg_alu0);
 8671 %}
 8672 
 8673 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8674 %{
 8675   match(Set dst (MulI src imm));
 8676   effect(KILL cr);
 8677 
 8678   ins_cost(300);
 8679   format %{ "imull   $dst, $src, $imm\t# int" %}
 8680   ins_encode %{
 8681     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8682   %}
 8683   ins_pipe(ialu_reg_reg_alu0);
 8684 %}
 8685 
 8686 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8687 %{
 8688   match(Set dst (MulI dst (LoadI src)));
 8689   effect(KILL cr);
 8690 
 8691   ins_cost(350);
 8692   format %{ "imull   $dst, $src\t# int" %}
 8693   ins_encode %{
 8694     __ imull($dst$$Register, $src$$Address);
 8695   %}
 8696   ins_pipe(ialu_reg_mem_alu0);
 8697 %}
 8698 
 8699 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8700 %{
 8701   match(Set dst (MulI (LoadI src) imm));
 8702   effect(KILL cr);
 8703 
 8704   ins_cost(300);
 8705   format %{ "imull   $dst, $src, $imm\t# int" %}
 8706   ins_encode %{
 8707     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8708   %}
 8709   ins_pipe(ialu_reg_mem_alu0);
 8710 %}
 8711 
 8712 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8713 %{
 8714   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8715   effect(KILL cr, KILL src2);
 8716 
 8717   expand %{ mulI_rReg(dst, src1, cr);
 8718            mulI_rReg(src2, src3, cr);
 8719            addI_rReg(dst, src2, cr); %}
 8720 %}
 8721 
 8722 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8723 %{
 8724   match(Set dst (MulL dst src));
 8725   effect(KILL cr);
 8726 
 8727   ins_cost(300);
 8728   format %{ "imulq   $dst, $src\t# long" %}
 8729   ins_encode %{
 8730     __ imulq($dst$$Register, $src$$Register);
 8731   %}
 8732   ins_pipe(ialu_reg_reg_alu0);
 8733 %}
 8734 
 8735 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8736 %{
 8737   match(Set dst (MulL src imm));
 8738   effect(KILL cr);
 8739 
 8740   ins_cost(300);
 8741   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8742   ins_encode %{
 8743     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8744   %}
 8745   ins_pipe(ialu_reg_reg_alu0);
 8746 %}
 8747 
 8748 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8749 %{
 8750   match(Set dst (MulL dst (LoadL src)));
 8751   effect(KILL cr);
 8752 
 8753   ins_cost(350);
 8754   format %{ "imulq   $dst, $src\t# long" %}
 8755   ins_encode %{
 8756     __ imulq($dst$$Register, $src$$Address);
 8757   %}
 8758   ins_pipe(ialu_reg_mem_alu0);
 8759 %}
 8760 
 8761 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8762 %{
 8763   match(Set dst (MulL (LoadL src) imm));
 8764   effect(KILL cr);
 8765 
 8766   ins_cost(300);
 8767   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8768   ins_encode %{
 8769     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8770   %}
 8771   ins_pipe(ialu_reg_mem_alu0);
 8772 %}
 8773 
 8774 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8775 %{
 8776   match(Set dst (MulHiL src rax));
 8777   effect(USE_KILL rax, KILL cr);
 8778 
 8779   ins_cost(300);
 8780   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8781   ins_encode %{
 8782     __ imulq($src$$Register);
 8783   %}
 8784   ins_pipe(ialu_reg_reg_alu0);
 8785 %}
 8786 
 8787 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8788 %{
 8789   match(Set dst (UMulHiL src rax));
 8790   effect(USE_KILL rax, KILL cr);
 8791 
 8792   ins_cost(300);
 8793   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8794   ins_encode %{
 8795     __ mulq($src$$Register);
 8796   %}
 8797   ins_pipe(ialu_reg_reg_alu0);
 8798 %}
 8799 
 8800 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8801                    rFlagsReg cr)
 8802 %{
 8803   match(Set rax (DivI rax div));
 8804   effect(KILL rdx, KILL cr);
 8805 
 8806   ins_cost(30*100+10*100); // XXX
 8807   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8808             "jne,s   normal\n\t"
 8809             "xorl    rdx, rdx\n\t"
 8810             "cmpl    $div, -1\n\t"
 8811             "je,s    done\n"
 8812     "normal: cdql\n\t"
 8813             "idivl   $div\n"
 8814     "done:"        %}
 8815   ins_encode(cdql_enc(div));
 8816   ins_pipe(ialu_reg_reg_alu0);
 8817 %}
 8818 
 8819 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8820                    rFlagsReg cr)
 8821 %{
 8822   match(Set rax (DivL rax div));
 8823   effect(KILL rdx, KILL cr);
 8824 
 8825   ins_cost(30*100+10*100); // XXX
 8826   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8827             "cmpq    rax, rdx\n\t"
 8828             "jne,s   normal\n\t"
 8829             "xorl    rdx, rdx\n\t"
 8830             "cmpq    $div, -1\n\t"
 8831             "je,s    done\n"
 8832     "normal: cdqq\n\t"
 8833             "idivq   $div\n"
 8834     "done:"        %}
 8835   ins_encode(cdqq_enc(div));
 8836   ins_pipe(ialu_reg_reg_alu0);
 8837 %}
 8838 
 8839 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8840 %{
 8841   match(Set rax (UDivI rax div));
 8842   effect(KILL rdx, KILL cr);
 8843 
 8844   ins_cost(300);
 8845   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8846   ins_encode %{
 8847     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8848   %}
 8849   ins_pipe(ialu_reg_reg_alu0);
 8850 %}
 8851 
 8852 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8853 %{
 8854   match(Set rax (UDivL rax div));
 8855   effect(KILL rdx, KILL cr);
 8856 
 8857   ins_cost(300);
 8858   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8859   ins_encode %{
 8860      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8861   %}
 8862   ins_pipe(ialu_reg_reg_alu0);
 8863 %}
 8864 
 8865 // Integer DIVMOD with Register, both quotient and mod results
 8866 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8867                              rFlagsReg cr)
 8868 %{
 8869   match(DivModI rax div);
 8870   effect(KILL cr);
 8871 
 8872   ins_cost(30*100+10*100); // XXX
 8873   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8874             "jne,s   normal\n\t"
 8875             "xorl    rdx, rdx\n\t"
 8876             "cmpl    $div, -1\n\t"
 8877             "je,s    done\n"
 8878     "normal: cdql\n\t"
 8879             "idivl   $div\n"
 8880     "done:"        %}
 8881   ins_encode(cdql_enc(div));
 8882   ins_pipe(pipe_slow);
 8883 %}
 8884 
 8885 // Long DIVMOD with Register, both quotient and mod results
 8886 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8887                              rFlagsReg cr)
 8888 %{
 8889   match(DivModL rax div);
 8890   effect(KILL cr);
 8891 
 8892   ins_cost(30*100+10*100); // XXX
 8893   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8894             "cmpq    rax, rdx\n\t"
 8895             "jne,s   normal\n\t"
 8896             "xorl    rdx, rdx\n\t"
 8897             "cmpq    $div, -1\n\t"
 8898             "je,s    done\n"
 8899     "normal: cdqq\n\t"
 8900             "idivq   $div\n"
 8901     "done:"        %}
 8902   ins_encode(cdqq_enc(div));
 8903   ins_pipe(pipe_slow);
 8904 %}
 8905 
 8906 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8907 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8908                               no_rax_rdx_RegI div, rFlagsReg cr)
 8909 %{
 8910   match(UDivModI rax div);
 8911   effect(TEMP tmp, KILL cr);
 8912 
 8913   ins_cost(300);
 8914   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8915             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8916           %}
 8917   ins_encode %{
 8918     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8919   %}
 8920   ins_pipe(pipe_slow);
 8921 %}
 8922 
 8923 // Unsigned long DIVMOD with Register, both quotient and mod results
 8924 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8925                               no_rax_rdx_RegL div, rFlagsReg cr)
 8926 %{
 8927   match(UDivModL rax div);
 8928   effect(TEMP tmp, KILL cr);
 8929 
 8930   ins_cost(300);
 8931   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8932             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8933           %}
 8934   ins_encode %{
 8935     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8936   %}
 8937   ins_pipe(pipe_slow);
 8938 %}
 8939 
 8940 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8941                    rFlagsReg cr)
 8942 %{
 8943   match(Set rdx (ModI rax div));
 8944   effect(KILL rax, KILL cr);
 8945 
 8946   ins_cost(300); // XXX
 8947   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8948             "jne,s   normal\n\t"
 8949             "xorl    rdx, rdx\n\t"
 8950             "cmpl    $div, -1\n\t"
 8951             "je,s    done\n"
 8952     "normal: cdql\n\t"
 8953             "idivl   $div\n"
 8954     "done:"        %}
 8955   ins_encode(cdql_enc(div));
 8956   ins_pipe(ialu_reg_reg_alu0);
 8957 %}
 8958 
 8959 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8960                    rFlagsReg cr)
 8961 %{
 8962   match(Set rdx (ModL rax div));
 8963   effect(KILL rax, KILL cr);
 8964 
 8965   ins_cost(300); // XXX
 8966   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8967             "cmpq    rax, rdx\n\t"
 8968             "jne,s   normal\n\t"
 8969             "xorl    rdx, rdx\n\t"
 8970             "cmpq    $div, -1\n\t"
 8971             "je,s    done\n"
 8972     "normal: cdqq\n\t"
 8973             "idivq   $div\n"
 8974     "done:"        %}
 8975   ins_encode(cdqq_enc(div));
 8976   ins_pipe(ialu_reg_reg_alu0);
 8977 %}
 8978 
 8979 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8980 %{
 8981   match(Set rdx (UModI rax div));
 8982   effect(KILL rax, KILL cr);
 8983 
 8984   ins_cost(300);
 8985   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8986   ins_encode %{
 8987     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8988   %}
 8989   ins_pipe(ialu_reg_reg_alu0);
 8990 %}
 8991 
 8992 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8993 %{
 8994   match(Set rdx (UModL rax div));
 8995   effect(KILL rax, KILL cr);
 8996 
 8997   ins_cost(300);
 8998   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8999   ins_encode %{
 9000     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9001   %}
 9002   ins_pipe(ialu_reg_reg_alu0);
 9003 %}
 9004 
 9005 // Integer Shift Instructions
 9006 // Shift Left by 8-bit immediate
 9007 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9008 %{
 9009   match(Set dst (LShiftI dst shift));
 9010   effect(KILL cr);
 9011 
 9012   format %{ "sall    $dst, $shift" %}
 9013   ins_encode %{
 9014     __ sall($dst$$Register, $shift$$constant);
 9015   %}
 9016   ins_pipe(ialu_reg);
 9017 %}
 9018 
 9019 // Shift Left by 8-bit immediate
 9020 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9021 %{
 9022   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9023   effect(KILL cr);
 9024 
 9025   format %{ "sall    $dst, $shift" %}
 9026   ins_encode %{
 9027     __ sall($dst$$Address, $shift$$constant);
 9028   %}
 9029   ins_pipe(ialu_mem_imm);
 9030 %}
 9031 
 9032 // Shift Left by variable
 9033 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9034 %{
 9035   predicate(!VM_Version::supports_bmi2());
 9036   match(Set dst (LShiftI dst shift));
 9037   effect(KILL cr);
 9038 
 9039   format %{ "sall    $dst, $shift" %}
 9040   ins_encode %{
 9041     __ sall($dst$$Register);
 9042   %}
 9043   ins_pipe(ialu_reg_reg);
 9044 %}
 9045 
 9046 // Shift Left by variable
 9047 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9048 %{
 9049   predicate(!VM_Version::supports_bmi2());
 9050   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9051   effect(KILL cr);
 9052 
 9053   format %{ "sall    $dst, $shift" %}
 9054   ins_encode %{
 9055     __ sall($dst$$Address);
 9056   %}
 9057   ins_pipe(ialu_mem_reg);
 9058 %}
 9059 
 9060 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9061 %{
 9062   predicate(VM_Version::supports_bmi2());
 9063   match(Set dst (LShiftI src shift));
 9064 
 9065   format %{ "shlxl   $dst, $src, $shift" %}
 9066   ins_encode %{
 9067     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9068   %}
 9069   ins_pipe(ialu_reg_reg);
 9070 %}
 9071 
 9072 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9073 %{
 9074   predicate(VM_Version::supports_bmi2());
 9075   match(Set dst (LShiftI (LoadI src) shift));
 9076   ins_cost(175);
 9077   format %{ "shlxl   $dst, $src, $shift" %}
 9078   ins_encode %{
 9079     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9080   %}
 9081   ins_pipe(ialu_reg_mem);
 9082 %}
 9083 
 9084 // Arithmetic Shift Right by 8-bit immediate
 9085 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9086 %{
 9087   match(Set dst (RShiftI dst shift));
 9088   effect(KILL cr);
 9089 
 9090   format %{ "sarl    $dst, $shift" %}
 9091   ins_encode %{
 9092     __ sarl($dst$$Register, $shift$$constant);
 9093   %}
 9094   ins_pipe(ialu_mem_imm);
 9095 %}
 9096 
 9097 // Arithmetic Shift Right by 8-bit immediate
 9098 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9099 %{
 9100   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9101   effect(KILL cr);
 9102 
 9103   format %{ "sarl    $dst, $shift" %}
 9104   ins_encode %{
 9105     __ sarl($dst$$Address, $shift$$constant);
 9106   %}
 9107   ins_pipe(ialu_mem_imm);
 9108 %}
 9109 
 9110 // Arithmetic Shift Right by variable
 9111 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9112 %{
 9113   predicate(!VM_Version::supports_bmi2());
 9114   match(Set dst (RShiftI dst shift));
 9115   effect(KILL cr);
 9116   format %{ "sarl    $dst, $shift" %}
 9117   ins_encode %{
 9118     __ sarl($dst$$Register);
 9119   %}
 9120   ins_pipe(ialu_reg_reg);
 9121 %}
 9122 
 9123 // Arithmetic Shift Right by variable
 9124 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9125 %{
 9126   predicate(!VM_Version::supports_bmi2());
 9127   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9128   effect(KILL cr);
 9129 
 9130   format %{ "sarl    $dst, $shift" %}
 9131   ins_encode %{
 9132     __ sarl($dst$$Address);
 9133   %}
 9134   ins_pipe(ialu_mem_reg);
 9135 %}
 9136 
 9137 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9138 %{
 9139   predicate(VM_Version::supports_bmi2());
 9140   match(Set dst (RShiftI src shift));
 9141 
 9142   format %{ "sarxl   $dst, $src, $shift" %}
 9143   ins_encode %{
 9144     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9145   %}
 9146   ins_pipe(ialu_reg_reg);
 9147 %}
 9148 
 9149 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9150 %{
 9151   predicate(VM_Version::supports_bmi2());
 9152   match(Set dst (RShiftI (LoadI src) shift));
 9153   ins_cost(175);
 9154   format %{ "sarxl   $dst, $src, $shift" %}
 9155   ins_encode %{
 9156     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9157   %}
 9158   ins_pipe(ialu_reg_mem);
 9159 %}
 9160 
 9161 // Logical Shift Right by 8-bit immediate
 9162 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9163 %{
 9164   match(Set dst (URShiftI dst shift));
 9165   effect(KILL cr);
 9166 
 9167   format %{ "shrl    $dst, $shift" %}
 9168   ins_encode %{
 9169     __ shrl($dst$$Register, $shift$$constant);
 9170   %}
 9171   ins_pipe(ialu_reg);
 9172 %}
 9173 
 9174 // Logical Shift Right by 8-bit immediate
 9175 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9176 %{
 9177   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9178   effect(KILL cr);
 9179 
 9180   format %{ "shrl    $dst, $shift" %}
 9181   ins_encode %{
 9182     __ shrl($dst$$Address, $shift$$constant);
 9183   %}
 9184   ins_pipe(ialu_mem_imm);
 9185 %}
 9186 
 9187 // Logical Shift Right by variable
 9188 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9189 %{
 9190   predicate(!VM_Version::supports_bmi2());
 9191   match(Set dst (URShiftI dst shift));
 9192   effect(KILL cr);
 9193 
 9194   format %{ "shrl    $dst, $shift" %}
 9195   ins_encode %{
 9196     __ shrl($dst$$Register);
 9197   %}
 9198   ins_pipe(ialu_reg_reg);
 9199 %}
 9200 
 9201 // Logical Shift Right by variable
 9202 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9203 %{
 9204   predicate(!VM_Version::supports_bmi2());
 9205   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9206   effect(KILL cr);
 9207 
 9208   format %{ "shrl    $dst, $shift" %}
 9209   ins_encode %{
 9210     __ shrl($dst$$Address);
 9211   %}
 9212   ins_pipe(ialu_mem_reg);
 9213 %}
 9214 
 9215 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9216 %{
 9217   predicate(VM_Version::supports_bmi2());
 9218   match(Set dst (URShiftI src shift));
 9219 
 9220   format %{ "shrxl   $dst, $src, $shift" %}
 9221   ins_encode %{
 9222     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9223   %}
 9224   ins_pipe(ialu_reg_reg);
 9225 %}
 9226 
 9227 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9228 %{
 9229   predicate(VM_Version::supports_bmi2());
 9230   match(Set dst (URShiftI (LoadI src) shift));
 9231   ins_cost(175);
 9232   format %{ "shrxl   $dst, $src, $shift" %}
 9233   ins_encode %{
 9234     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9235   %}
 9236   ins_pipe(ialu_reg_mem);
 9237 %}
 9238 
 9239 // Long Shift Instructions
 9240 // Shift Left by 8-bit immediate
 9241 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9242 %{
 9243   match(Set dst (LShiftL dst shift));
 9244   effect(KILL cr);
 9245 
 9246   format %{ "salq    $dst, $shift" %}
 9247   ins_encode %{
 9248     __ salq($dst$$Register, $shift$$constant);
 9249   %}
 9250   ins_pipe(ialu_reg);
 9251 %}
 9252 
 9253 // Shift Left by 8-bit immediate
 9254 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9255 %{
 9256   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9257   effect(KILL cr);
 9258 
 9259   format %{ "salq    $dst, $shift" %}
 9260   ins_encode %{
 9261     __ salq($dst$$Address, $shift$$constant);
 9262   %}
 9263   ins_pipe(ialu_mem_imm);
 9264 %}
 9265 
 9266 // Shift Left by variable
 9267 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9268 %{
 9269   predicate(!VM_Version::supports_bmi2());
 9270   match(Set dst (LShiftL dst shift));
 9271   effect(KILL cr);
 9272 
 9273   format %{ "salq    $dst, $shift" %}
 9274   ins_encode %{
 9275     __ salq($dst$$Register);
 9276   %}
 9277   ins_pipe(ialu_reg_reg);
 9278 %}
 9279 
 9280 // Shift Left by variable
 9281 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9282 %{
 9283   predicate(!VM_Version::supports_bmi2());
 9284   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9285   effect(KILL cr);
 9286 
 9287   format %{ "salq    $dst, $shift" %}
 9288   ins_encode %{
 9289     __ salq($dst$$Address);
 9290   %}
 9291   ins_pipe(ialu_mem_reg);
 9292 %}
 9293 
 9294 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9295 %{
 9296   predicate(VM_Version::supports_bmi2());
 9297   match(Set dst (LShiftL src shift));
 9298 
 9299   format %{ "shlxq   $dst, $src, $shift" %}
 9300   ins_encode %{
 9301     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9302   %}
 9303   ins_pipe(ialu_reg_reg);
 9304 %}
 9305 
 9306 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9307 %{
 9308   predicate(VM_Version::supports_bmi2());
 9309   match(Set dst (LShiftL (LoadL src) shift));
 9310   ins_cost(175);
 9311   format %{ "shlxq   $dst, $src, $shift" %}
 9312   ins_encode %{
 9313     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9314   %}
 9315   ins_pipe(ialu_reg_mem);
 9316 %}
 9317 
 9318 // Arithmetic Shift Right by 8-bit immediate
 9319 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9320 %{
 9321   match(Set dst (RShiftL dst shift));
 9322   effect(KILL cr);
 9323 
 9324   format %{ "sarq    $dst, $shift" %}
 9325   ins_encode %{
 9326     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9327   %}
 9328   ins_pipe(ialu_mem_imm);
 9329 %}
 9330 
 9331 // Arithmetic Shift Right by 8-bit immediate
 9332 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9333 %{
 9334   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9335   effect(KILL cr);
 9336 
 9337   format %{ "sarq    $dst, $shift" %}
 9338   ins_encode %{
 9339     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9340   %}
 9341   ins_pipe(ialu_mem_imm);
 9342 %}
 9343 
 9344 // Arithmetic Shift Right by variable
 9345 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9346 %{
 9347   predicate(!VM_Version::supports_bmi2());
 9348   match(Set dst (RShiftL dst shift));
 9349   effect(KILL cr);
 9350 
 9351   format %{ "sarq    $dst, $shift" %}
 9352   ins_encode %{
 9353     __ sarq($dst$$Register);
 9354   %}
 9355   ins_pipe(ialu_reg_reg);
 9356 %}
 9357 
 9358 // Arithmetic Shift Right by variable
 9359 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9360 %{
 9361   predicate(!VM_Version::supports_bmi2());
 9362   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9363   effect(KILL cr);
 9364 
 9365   format %{ "sarq    $dst, $shift" %}
 9366   ins_encode %{
 9367     __ sarq($dst$$Address);
 9368   %}
 9369   ins_pipe(ialu_mem_reg);
 9370 %}
 9371 
 9372 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9373 %{
 9374   predicate(VM_Version::supports_bmi2());
 9375   match(Set dst (RShiftL src shift));
 9376 
 9377   format %{ "sarxq   $dst, $src, $shift" %}
 9378   ins_encode %{
 9379     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9380   %}
 9381   ins_pipe(ialu_reg_reg);
 9382 %}
 9383 
 9384 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9385 %{
 9386   predicate(VM_Version::supports_bmi2());
 9387   match(Set dst (RShiftL (LoadL src) shift));
 9388   ins_cost(175);
 9389   format %{ "sarxq   $dst, $src, $shift" %}
 9390   ins_encode %{
 9391     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9392   %}
 9393   ins_pipe(ialu_reg_mem);
 9394 %}
 9395 
 9396 // Logical Shift Right by 8-bit immediate
 9397 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9398 %{
 9399   match(Set dst (URShiftL dst shift));
 9400   effect(KILL cr);
 9401 
 9402   format %{ "shrq    $dst, $shift" %}
 9403   ins_encode %{
 9404     __ shrq($dst$$Register, $shift$$constant);
 9405   %}
 9406   ins_pipe(ialu_reg);
 9407 %}
 9408 
 9409 // Logical Shift Right by 8-bit immediate
 9410 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9411 %{
 9412   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9413   effect(KILL cr);
 9414 
 9415   format %{ "shrq    $dst, $shift" %}
 9416   ins_encode %{
 9417     __ shrq($dst$$Address, $shift$$constant);
 9418   %}
 9419   ins_pipe(ialu_mem_imm);
 9420 %}
 9421 
 9422 // Logical Shift Right by variable
 9423 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9424 %{
 9425   predicate(!VM_Version::supports_bmi2());
 9426   match(Set dst (URShiftL dst shift));
 9427   effect(KILL cr);
 9428 
 9429   format %{ "shrq    $dst, $shift" %}
 9430   ins_encode %{
 9431     __ shrq($dst$$Register);
 9432   %}
 9433   ins_pipe(ialu_reg_reg);
 9434 %}
 9435 
 9436 // Logical Shift Right by variable
 9437 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9438 %{
 9439   predicate(!VM_Version::supports_bmi2());
 9440   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9441   effect(KILL cr);
 9442 
 9443   format %{ "shrq    $dst, $shift" %}
 9444   ins_encode %{
 9445     __ shrq($dst$$Address);
 9446   %}
 9447   ins_pipe(ialu_mem_reg);
 9448 %}
 9449 
 9450 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9451 %{
 9452   predicate(VM_Version::supports_bmi2());
 9453   match(Set dst (URShiftL src shift));
 9454 
 9455   format %{ "shrxq   $dst, $src, $shift" %}
 9456   ins_encode %{
 9457     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9458   %}
 9459   ins_pipe(ialu_reg_reg);
 9460 %}
 9461 
 9462 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9463 %{
 9464   predicate(VM_Version::supports_bmi2());
 9465   match(Set dst (URShiftL (LoadL src) shift));
 9466   ins_cost(175);
 9467   format %{ "shrxq   $dst, $src, $shift" %}
 9468   ins_encode %{
 9469     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9470   %}
 9471   ins_pipe(ialu_reg_mem);
 9472 %}
 9473 
 9474 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9475 // This idiom is used by the compiler for the i2b bytecode.
 9476 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9477 %{
 9478   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9479 
 9480   format %{ "movsbl  $dst, $src\t# i2b" %}
 9481   ins_encode %{
 9482     __ movsbl($dst$$Register, $src$$Register);
 9483   %}
 9484   ins_pipe(ialu_reg_reg);
 9485 %}
 9486 
 9487 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9488 // This idiom is used by the compiler the i2s bytecode.
 9489 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9490 %{
 9491   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9492 
 9493   format %{ "movswl  $dst, $src\t# i2s" %}
 9494   ins_encode %{
 9495     __ movswl($dst$$Register, $src$$Register);
 9496   %}
 9497   ins_pipe(ialu_reg_reg);
 9498 %}
 9499 
 9500 // ROL/ROR instructions
 9501 
 9502 // Rotate left by constant.
 9503 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9504 %{
 9505   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9506   match(Set dst (RotateLeft dst shift));
 9507   effect(KILL cr);
 9508   format %{ "roll    $dst, $shift" %}
 9509   ins_encode %{
 9510     __ roll($dst$$Register, $shift$$constant);
 9511   %}
 9512   ins_pipe(ialu_reg);
 9513 %}
 9514 
 9515 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9516 %{
 9517   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9518   match(Set dst (RotateLeft src shift));
 9519   format %{ "rolxl   $dst, $src, $shift" %}
 9520   ins_encode %{
 9521     int shift = 32 - ($shift$$constant & 31);
 9522     __ rorxl($dst$$Register, $src$$Register, shift);
 9523   %}
 9524   ins_pipe(ialu_reg_reg);
 9525 %}
 9526 
 9527 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9528 %{
 9529   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9530   match(Set dst (RotateLeft (LoadI src) shift));
 9531   ins_cost(175);
 9532   format %{ "rolxl   $dst, $src, $shift" %}
 9533   ins_encode %{
 9534     int shift = 32 - ($shift$$constant & 31);
 9535     __ rorxl($dst$$Register, $src$$Address, shift);
 9536   %}
 9537   ins_pipe(ialu_reg_mem);
 9538 %}
 9539 
 9540 // Rotate Left by variable
 9541 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9542 %{
 9543   predicate(n->bottom_type()->basic_type() == T_INT);
 9544   match(Set dst (RotateLeft dst shift));
 9545   effect(KILL cr);
 9546   format %{ "roll    $dst, $shift" %}
 9547   ins_encode %{
 9548     __ roll($dst$$Register);
 9549   %}
 9550   ins_pipe(ialu_reg_reg);
 9551 %}
 9552 
 9553 // Rotate Right by constant.
 9554 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9555 %{
 9556   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9557   match(Set dst (RotateRight dst shift));
 9558   effect(KILL cr);
 9559   format %{ "rorl    $dst, $shift" %}
 9560   ins_encode %{
 9561     __ rorl($dst$$Register, $shift$$constant);
 9562   %}
 9563   ins_pipe(ialu_reg);
 9564 %}
 9565 
 9566 // Rotate Right by constant.
 9567 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9568 %{
 9569   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9570   match(Set dst (RotateRight src shift));
 9571   format %{ "rorxl   $dst, $src, $shift" %}
 9572   ins_encode %{
 9573     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9574   %}
 9575   ins_pipe(ialu_reg_reg);
 9576 %}
 9577 
 9578 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9579 %{
 9580   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9581   match(Set dst (RotateRight (LoadI src) shift));
 9582   ins_cost(175);
 9583   format %{ "rorxl   $dst, $src, $shift" %}
 9584   ins_encode %{
 9585     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9586   %}
 9587   ins_pipe(ialu_reg_mem);
 9588 %}
 9589 
 9590 // Rotate Right by variable
 9591 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9592 %{
 9593   predicate(n->bottom_type()->basic_type() == T_INT);
 9594   match(Set dst (RotateRight dst shift));
 9595   effect(KILL cr);
 9596   format %{ "rorl    $dst, $shift" %}
 9597   ins_encode %{
 9598     __ rorl($dst$$Register);
 9599   %}
 9600   ins_pipe(ialu_reg_reg);
 9601 %}
 9602 
 9603 // Rotate Left by constant.
 9604 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9605 %{
 9606   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9607   match(Set dst (RotateLeft dst shift));
 9608   effect(KILL cr);
 9609   format %{ "rolq    $dst, $shift" %}
 9610   ins_encode %{
 9611     __ rolq($dst$$Register, $shift$$constant);
 9612   %}
 9613   ins_pipe(ialu_reg);
 9614 %}
 9615 
 9616 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9617 %{
 9618   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9619   match(Set dst (RotateLeft src shift));
 9620   format %{ "rolxq   $dst, $src, $shift" %}
 9621   ins_encode %{
 9622     int shift = 64 - ($shift$$constant & 63);
 9623     __ rorxq($dst$$Register, $src$$Register, shift);
 9624   %}
 9625   ins_pipe(ialu_reg_reg);
 9626 %}
 9627 
 9628 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9629 %{
 9630   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9631   match(Set dst (RotateLeft (LoadL src) shift));
 9632   ins_cost(175);
 9633   format %{ "rolxq   $dst, $src, $shift" %}
 9634   ins_encode %{
 9635     int shift = 64 - ($shift$$constant & 63);
 9636     __ rorxq($dst$$Register, $src$$Address, shift);
 9637   %}
 9638   ins_pipe(ialu_reg_mem);
 9639 %}
 9640 
 9641 // Rotate Left by variable
 9642 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9643 %{
 9644   predicate(n->bottom_type()->basic_type() == T_LONG);
 9645   match(Set dst (RotateLeft dst shift));
 9646   effect(KILL cr);
 9647   format %{ "rolq    $dst, $shift" %}
 9648   ins_encode %{
 9649     __ rolq($dst$$Register);
 9650   %}
 9651   ins_pipe(ialu_reg_reg);
 9652 %}
 9653 
 9654 // Rotate Right by constant.
 9655 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9656 %{
 9657   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9658   match(Set dst (RotateRight dst shift));
 9659   effect(KILL cr);
 9660   format %{ "rorq    $dst, $shift" %}
 9661   ins_encode %{
 9662     __ rorq($dst$$Register, $shift$$constant);
 9663   %}
 9664   ins_pipe(ialu_reg);
 9665 %}
 9666 
 9667 // Rotate Right by constant
 9668 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9669 %{
 9670   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9671   match(Set dst (RotateRight src shift));
 9672   format %{ "rorxq   $dst, $src, $shift" %}
 9673   ins_encode %{
 9674     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9675   %}
 9676   ins_pipe(ialu_reg_reg);
 9677 %}
 9678 
 9679 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9680 %{
 9681   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9682   match(Set dst (RotateRight (LoadL src) shift));
 9683   ins_cost(175);
 9684   format %{ "rorxq   $dst, $src, $shift" %}
 9685   ins_encode %{
 9686     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9687   %}
 9688   ins_pipe(ialu_reg_mem);
 9689 %}
 9690 
 9691 // Rotate Right by variable
 9692 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9693 %{
 9694   predicate(n->bottom_type()->basic_type() == T_LONG);
 9695   match(Set dst (RotateRight dst shift));
 9696   effect(KILL cr);
 9697   format %{ "rorq    $dst, $shift" %}
 9698   ins_encode %{
 9699     __ rorq($dst$$Register);
 9700   %}
 9701   ins_pipe(ialu_reg_reg);
 9702 %}
 9703 
 9704 //----------------------------- CompressBits/ExpandBits ------------------------
 9705 
 9706 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9707   predicate(n->bottom_type()->isa_long());
 9708   match(Set dst (CompressBits src mask));
 9709   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9710   ins_encode %{
 9711     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9712   %}
 9713   ins_pipe( pipe_slow );
 9714 %}
 9715 
 9716 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9717   predicate(n->bottom_type()->isa_long());
 9718   match(Set dst (ExpandBits src mask));
 9719   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9720   ins_encode %{
 9721     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9722   %}
 9723   ins_pipe( pipe_slow );
 9724 %}
 9725 
 9726 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9727   predicate(n->bottom_type()->isa_long());
 9728   match(Set dst (CompressBits src (LoadL mask)));
 9729   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9730   ins_encode %{
 9731     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 9732   %}
 9733   ins_pipe( pipe_slow );
 9734 %}
 9735 
 9736 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9737   predicate(n->bottom_type()->isa_long());
 9738   match(Set dst (ExpandBits src (LoadL mask)));
 9739   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9740   ins_encode %{
 9741     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 9742   %}
 9743   ins_pipe( pipe_slow );
 9744 %}
 9745 
 9746 
 9747 // Logical Instructions
 9748 
 9749 // Integer Logical Instructions
 9750 
 9751 // And Instructions
 9752 // And Register with Register
 9753 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9754 %{
 9755   match(Set dst (AndI dst src));
 9756   effect(KILL cr);
 9757 
 9758   format %{ "andl    $dst, $src\t# int" %}
 9759   ins_encode %{
 9760     __ andl($dst$$Register, $src$$Register);
 9761   %}
 9762   ins_pipe(ialu_reg_reg);
 9763 %}
 9764 
 9765 // And Register with Immediate 255
 9766 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 9767 %{
 9768   match(Set dst (AndI src mask));
 9769 
 9770   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 9771   ins_encode %{
 9772     __ movzbl($dst$$Register, $src$$Register);
 9773   %}
 9774   ins_pipe(ialu_reg);
 9775 %}
 9776 
 9777 // And Register with Immediate 255 and promote to long
 9778 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9779 %{
 9780   match(Set dst (ConvI2L (AndI src mask)));
 9781 
 9782   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9783   ins_encode %{
 9784     __ movzbl($dst$$Register, $src$$Register);
 9785   %}
 9786   ins_pipe(ialu_reg);
 9787 %}
 9788 
 9789 // And Register with Immediate 65535
 9790 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 9791 %{
 9792   match(Set dst (AndI src mask));
 9793 
 9794   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 9795   ins_encode %{
 9796     __ movzwl($dst$$Register, $src$$Register);
 9797   %}
 9798   ins_pipe(ialu_reg);
 9799 %}
 9800 
 9801 // And Register with Immediate 65535 and promote to long
 9802 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9803 %{
 9804   match(Set dst (ConvI2L (AndI src mask)));
 9805 
 9806   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9807   ins_encode %{
 9808     __ movzwl($dst$$Register, $src$$Register);
 9809   %}
 9810   ins_pipe(ialu_reg);
 9811 %}
 9812 
 9813 // Can skip int2long conversions after AND with small bitmask
 9814 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9815 %{
 9816   predicate(VM_Version::supports_bmi2());
 9817   ins_cost(125);
 9818   effect(TEMP tmp, KILL cr);
 9819   match(Set dst (ConvI2L (AndI src mask)));
 9820   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9821   ins_encode %{
 9822     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9823     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9824   %}
 9825   ins_pipe(ialu_reg_reg);
 9826 %}
 9827 
 9828 // And Register with Immediate
 9829 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9830 %{
 9831   match(Set dst (AndI dst src));
 9832   effect(KILL cr);
 9833 
 9834   format %{ "andl    $dst, $src\t# int" %}
 9835   ins_encode %{
 9836     __ andl($dst$$Register, $src$$constant);
 9837   %}
 9838   ins_pipe(ialu_reg);
 9839 %}
 9840 
 9841 // And Register with Memory
 9842 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9843 %{
 9844   match(Set dst (AndI dst (LoadI src)));
 9845   effect(KILL cr);
 9846 
 9847   ins_cost(150);
 9848   format %{ "andl    $dst, $src\t# int" %}
 9849   ins_encode %{
 9850     __ andl($dst$$Register, $src$$Address);
 9851   %}
 9852   ins_pipe(ialu_reg_mem);
 9853 %}
 9854 
 9855 // And Memory with Register
 9856 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9857 %{
 9858   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9859   effect(KILL cr);
 9860 
 9861   ins_cost(150);
 9862   format %{ "andb    $dst, $src\t# byte" %}
 9863   ins_encode %{
 9864     __ andb($dst$$Address, $src$$Register);
 9865   %}
 9866   ins_pipe(ialu_mem_reg);
 9867 %}
 9868 
 9869 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9870 %{
 9871   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9872   effect(KILL cr);
 9873 
 9874   ins_cost(150);
 9875   format %{ "andl    $dst, $src\t# int" %}
 9876   ins_encode %{
 9877     __ andl($dst$$Address, $src$$Register);
 9878   %}
 9879   ins_pipe(ialu_mem_reg);
 9880 %}
 9881 
 9882 // And Memory with Immediate
 9883 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9884 %{
 9885   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9886   effect(KILL cr);
 9887 
 9888   ins_cost(125);
 9889   format %{ "andl    $dst, $src\t# int" %}
 9890   ins_encode %{
 9891     __ andl($dst$$Address, $src$$constant);
 9892   %}
 9893   ins_pipe(ialu_mem_imm);
 9894 %}
 9895 
 9896 // BMI1 instructions
 9897 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9898   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9899   predicate(UseBMI1Instructions);
 9900   effect(KILL cr);
 9901 
 9902   ins_cost(125);
 9903   format %{ "andnl  $dst, $src1, $src2" %}
 9904 
 9905   ins_encode %{
 9906     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9907   %}
 9908   ins_pipe(ialu_reg_mem);
 9909 %}
 9910 
 9911 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9912   match(Set dst (AndI (XorI src1 minus_1) src2));
 9913   predicate(UseBMI1Instructions);
 9914   effect(KILL cr);
 9915 
 9916   format %{ "andnl  $dst, $src1, $src2" %}
 9917 
 9918   ins_encode %{
 9919     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9920   %}
 9921   ins_pipe(ialu_reg);
 9922 %}
 9923 
 9924 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9925   match(Set dst (AndI (SubI imm_zero src) src));
 9926   predicate(UseBMI1Instructions);
 9927   effect(KILL cr);
 9928 
 9929   format %{ "blsil  $dst, $src" %}
 9930 
 9931   ins_encode %{
 9932     __ blsil($dst$$Register, $src$$Register);
 9933   %}
 9934   ins_pipe(ialu_reg);
 9935 %}
 9936 
 9937 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9938   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9939   predicate(UseBMI1Instructions);
 9940   effect(KILL cr);
 9941 
 9942   ins_cost(125);
 9943   format %{ "blsil  $dst, $src" %}
 9944 
 9945   ins_encode %{
 9946     __ blsil($dst$$Register, $src$$Address);
 9947   %}
 9948   ins_pipe(ialu_reg_mem);
 9949 %}
 9950 
 9951 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9952 %{
 9953   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9954   predicate(UseBMI1Instructions);
 9955   effect(KILL cr);
 9956 
 9957   ins_cost(125);
 9958   format %{ "blsmskl $dst, $src" %}
 9959 
 9960   ins_encode %{
 9961     __ blsmskl($dst$$Register, $src$$Address);
 9962   %}
 9963   ins_pipe(ialu_reg_mem);
 9964 %}
 9965 
 9966 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9967 %{
 9968   match(Set dst (XorI (AddI src minus_1) src));
 9969   predicate(UseBMI1Instructions);
 9970   effect(KILL cr);
 9971 
 9972   format %{ "blsmskl $dst, $src" %}
 9973 
 9974   ins_encode %{
 9975     __ blsmskl($dst$$Register, $src$$Register);
 9976   %}
 9977 
 9978   ins_pipe(ialu_reg);
 9979 %}
 9980 
 9981 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9982 %{
 9983   match(Set dst (AndI (AddI src minus_1) src) );
 9984   predicate(UseBMI1Instructions);
 9985   effect(KILL cr);
 9986 
 9987   format %{ "blsrl  $dst, $src" %}
 9988 
 9989   ins_encode %{
 9990     __ blsrl($dst$$Register, $src$$Register);
 9991   %}
 9992 
 9993   ins_pipe(ialu_reg_mem);
 9994 %}
 9995 
 9996 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9997 %{
 9998   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9999   predicate(UseBMI1Instructions);
10000   effect(KILL cr);
10001 
10002   ins_cost(125);
10003   format %{ "blsrl  $dst, $src" %}
10004 
10005   ins_encode %{
10006     __ blsrl($dst$$Register, $src$$Address);
10007   %}
10008 
10009   ins_pipe(ialu_reg);
10010 %}
10011 
10012 // Or Instructions
10013 // Or Register with Register
10014 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10015 %{
10016   match(Set dst (OrI dst src));
10017   effect(KILL cr);
10018 
10019   format %{ "orl     $dst, $src\t# int" %}
10020   ins_encode %{
10021     __ orl($dst$$Register, $src$$Register);
10022   %}
10023   ins_pipe(ialu_reg_reg);
10024 %}
10025 
10026 // Or Register with Immediate
10027 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10028 %{
10029   match(Set dst (OrI dst src));
10030   effect(KILL cr);
10031 
10032   format %{ "orl     $dst, $src\t# int" %}
10033   ins_encode %{
10034     __ orl($dst$$Register, $src$$constant);
10035   %}
10036   ins_pipe(ialu_reg);
10037 %}
10038 
10039 // Or Register with Memory
10040 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10041 %{
10042   match(Set dst (OrI dst (LoadI src)));
10043   effect(KILL cr);
10044 
10045   ins_cost(150);
10046   format %{ "orl     $dst, $src\t# int" %}
10047   ins_encode %{
10048     __ orl($dst$$Register, $src$$Address);
10049   %}
10050   ins_pipe(ialu_reg_mem);
10051 %}
10052 
10053 // Or Memory with Register
10054 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10055 %{
10056   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10057   effect(KILL cr);
10058 
10059   ins_cost(150);
10060   format %{ "orb    $dst, $src\t# byte" %}
10061   ins_encode %{
10062     __ orb($dst$$Address, $src$$Register);
10063   %}
10064   ins_pipe(ialu_mem_reg);
10065 %}
10066 
10067 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10068 %{
10069   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10070   effect(KILL cr);
10071 
10072   ins_cost(150);
10073   format %{ "orl     $dst, $src\t# int" %}
10074   ins_encode %{
10075     __ orl($dst$$Address, $src$$Register);
10076   %}
10077   ins_pipe(ialu_mem_reg);
10078 %}
10079 
10080 // Or Memory with Immediate
10081 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10082 %{
10083   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10084   effect(KILL cr);
10085 
10086   ins_cost(125);
10087   format %{ "orl     $dst, $src\t# int" %}
10088   ins_encode %{
10089     __ orl($dst$$Address, $src$$constant);
10090   %}
10091   ins_pipe(ialu_mem_imm);
10092 %}
10093 
10094 // Xor Instructions
10095 // Xor Register with Register
10096 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10097 %{
10098   match(Set dst (XorI dst src));
10099   effect(KILL cr);
10100 
10101   format %{ "xorl    $dst, $src\t# int" %}
10102   ins_encode %{
10103     __ xorl($dst$$Register, $src$$Register);
10104   %}
10105   ins_pipe(ialu_reg_reg);
10106 %}
10107 
10108 // Xor Register with Immediate -1
10109 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
10110   match(Set dst (XorI dst imm));
10111 
10112   format %{ "not    $dst" %}
10113   ins_encode %{
10114      __ notl($dst$$Register);
10115   %}
10116   ins_pipe(ialu_reg);
10117 %}
10118 
10119 // Xor Register with Immediate
10120 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10121 %{
10122   match(Set dst (XorI dst src));
10123   effect(KILL cr);
10124 
10125   format %{ "xorl    $dst, $src\t# int" %}
10126   ins_encode %{
10127     __ xorl($dst$$Register, $src$$constant);
10128   %}
10129   ins_pipe(ialu_reg);
10130 %}
10131 
10132 // Xor Register with Memory
10133 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10134 %{
10135   match(Set dst (XorI dst (LoadI src)));
10136   effect(KILL cr);
10137 
10138   ins_cost(150);
10139   format %{ "xorl    $dst, $src\t# int" %}
10140   ins_encode %{
10141     __ xorl($dst$$Register, $src$$Address);
10142   %}
10143   ins_pipe(ialu_reg_mem);
10144 %}
10145 
10146 // Xor Memory with Register
10147 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10148 %{
10149   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10150   effect(KILL cr);
10151 
10152   ins_cost(150);
10153   format %{ "xorb    $dst, $src\t# byte" %}
10154   ins_encode %{
10155     __ xorb($dst$$Address, $src$$Register);
10156   %}
10157   ins_pipe(ialu_mem_reg);
10158 %}
10159 
10160 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10161 %{
10162   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10163   effect(KILL cr);
10164 
10165   ins_cost(150);
10166   format %{ "xorl    $dst, $src\t# int" %}
10167   ins_encode %{
10168     __ xorl($dst$$Address, $src$$Register);
10169   %}
10170   ins_pipe(ialu_mem_reg);
10171 %}
10172 
10173 // Xor Memory with Immediate
10174 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10175 %{
10176   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10177   effect(KILL cr);
10178 
10179   ins_cost(125);
10180   format %{ "xorl    $dst, $src\t# int" %}
10181   ins_encode %{
10182     __ xorl($dst$$Address, $src$$constant);
10183   %}
10184   ins_pipe(ialu_mem_imm);
10185 %}
10186 
10187 
10188 // Long Logical Instructions
10189 
10190 // And Instructions
10191 // And Register with Register
10192 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10193 %{
10194   match(Set dst (AndL dst src));
10195   effect(KILL cr);
10196 
10197   format %{ "andq    $dst, $src\t# long" %}
10198   ins_encode %{
10199     __ andq($dst$$Register, $src$$Register);
10200   %}
10201   ins_pipe(ialu_reg_reg);
10202 %}
10203 
10204 // And Register with Immediate 255
10205 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10206 %{
10207   match(Set dst (AndL src mask));
10208 
10209   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10210   ins_encode %{
10211     // movzbl zeroes out the upper 32-bit and does not need REX.W
10212     __ movzbl($dst$$Register, $src$$Register);
10213   %}
10214   ins_pipe(ialu_reg);
10215 %}
10216 
10217 // And Register with Immediate 65535
10218 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10219 %{
10220   match(Set dst (AndL src mask));
10221 
10222   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10223   ins_encode %{
10224     // movzwl zeroes out the upper 32-bit and does not need REX.W
10225     __ movzwl($dst$$Register, $src$$Register);
10226   %}
10227   ins_pipe(ialu_reg);
10228 %}
10229 
10230 // And Register with Immediate
10231 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10232 %{
10233   match(Set dst (AndL dst src));
10234   effect(KILL cr);
10235 
10236   format %{ "andq    $dst, $src\t# long" %}
10237   ins_encode %{
10238     __ andq($dst$$Register, $src$$constant);
10239   %}
10240   ins_pipe(ialu_reg);
10241 %}
10242 
10243 // And Register with Memory
10244 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10245 %{
10246   match(Set dst (AndL dst (LoadL src)));
10247   effect(KILL cr);
10248 
10249   ins_cost(150);
10250   format %{ "andq    $dst, $src\t# long" %}
10251   ins_encode %{
10252     __ andq($dst$$Register, $src$$Address);
10253   %}
10254   ins_pipe(ialu_reg_mem);
10255 %}
10256 
10257 // And Memory with Register
10258 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10259 %{
10260   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10261   effect(KILL cr);
10262 
10263   ins_cost(150);
10264   format %{ "andq    $dst, $src\t# long" %}
10265   ins_encode %{
10266     __ andq($dst$$Address, $src$$Register);
10267   %}
10268   ins_pipe(ialu_mem_reg);
10269 %}
10270 
10271 // And Memory with Immediate
10272 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10273 %{
10274   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10275   effect(KILL cr);
10276 
10277   ins_cost(125);
10278   format %{ "andq    $dst, $src\t# long" %}
10279   ins_encode %{
10280     __ andq($dst$$Address, $src$$constant);
10281   %}
10282   ins_pipe(ialu_mem_imm);
10283 %}
10284 
10285 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10286 %{
10287   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10288   // because AND/OR works well enough for 8/32-bit values.
10289   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10290 
10291   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10292   effect(KILL cr);
10293 
10294   ins_cost(125);
10295   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10296   ins_encode %{
10297     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10298   %}
10299   ins_pipe(ialu_mem_imm);
10300 %}
10301 
10302 // BMI1 instructions
10303 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10304   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10305   predicate(UseBMI1Instructions);
10306   effect(KILL cr);
10307 
10308   ins_cost(125);
10309   format %{ "andnq  $dst, $src1, $src2" %}
10310 
10311   ins_encode %{
10312     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10313   %}
10314   ins_pipe(ialu_reg_mem);
10315 %}
10316 
10317 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10318   match(Set dst (AndL (XorL src1 minus_1) src2));
10319   predicate(UseBMI1Instructions);
10320   effect(KILL cr);
10321 
10322   format %{ "andnq  $dst, $src1, $src2" %}
10323 
10324   ins_encode %{
10325   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10326   %}
10327   ins_pipe(ialu_reg_mem);
10328 %}
10329 
10330 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10331   match(Set dst (AndL (SubL imm_zero src) src));
10332   predicate(UseBMI1Instructions);
10333   effect(KILL cr);
10334 
10335   format %{ "blsiq  $dst, $src" %}
10336 
10337   ins_encode %{
10338     __ blsiq($dst$$Register, $src$$Register);
10339   %}
10340   ins_pipe(ialu_reg);
10341 %}
10342 
10343 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10344   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10345   predicate(UseBMI1Instructions);
10346   effect(KILL cr);
10347 
10348   ins_cost(125);
10349   format %{ "blsiq  $dst, $src" %}
10350 
10351   ins_encode %{
10352     __ blsiq($dst$$Register, $src$$Address);
10353   %}
10354   ins_pipe(ialu_reg_mem);
10355 %}
10356 
10357 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10358 %{
10359   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10360   predicate(UseBMI1Instructions);
10361   effect(KILL cr);
10362 
10363   ins_cost(125);
10364   format %{ "blsmskq $dst, $src" %}
10365 
10366   ins_encode %{
10367     __ blsmskq($dst$$Register, $src$$Address);
10368   %}
10369   ins_pipe(ialu_reg_mem);
10370 %}
10371 
10372 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10373 %{
10374   match(Set dst (XorL (AddL src minus_1) src));
10375   predicate(UseBMI1Instructions);
10376   effect(KILL cr);
10377 
10378   format %{ "blsmskq $dst, $src" %}
10379 
10380   ins_encode %{
10381     __ blsmskq($dst$$Register, $src$$Register);
10382   %}
10383 
10384   ins_pipe(ialu_reg);
10385 %}
10386 
10387 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10388 %{
10389   match(Set dst (AndL (AddL src minus_1) src) );
10390   predicate(UseBMI1Instructions);
10391   effect(KILL cr);
10392 
10393   format %{ "blsrq  $dst, $src" %}
10394 
10395   ins_encode %{
10396     __ blsrq($dst$$Register, $src$$Register);
10397   %}
10398 
10399   ins_pipe(ialu_reg);
10400 %}
10401 
10402 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10403 %{
10404   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10405   predicate(UseBMI1Instructions);
10406   effect(KILL cr);
10407 
10408   ins_cost(125);
10409   format %{ "blsrq  $dst, $src" %}
10410 
10411   ins_encode %{
10412     __ blsrq($dst$$Register, $src$$Address);
10413   %}
10414 
10415   ins_pipe(ialu_reg);
10416 %}
10417 
10418 // Or Instructions
10419 // Or Register with Register
10420 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10421 %{
10422   match(Set dst (OrL dst src));
10423   effect(KILL cr);
10424 
10425   format %{ "orq     $dst, $src\t# long" %}
10426   ins_encode %{
10427     __ orq($dst$$Register, $src$$Register);
10428   %}
10429   ins_pipe(ialu_reg_reg);
10430 %}
10431 
10432 // Use any_RegP to match R15 (TLS register) without spilling.
10433 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10434   match(Set dst (OrL dst (CastP2X src)));
10435   effect(KILL cr);
10436 
10437   format %{ "orq     $dst, $src\t# long" %}
10438   ins_encode %{
10439     __ orq($dst$$Register, $src$$Register);
10440   %}
10441   ins_pipe(ialu_reg_reg);
10442 %}
10443 
10444 
10445 // Or Register with Immediate
10446 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10447 %{
10448   match(Set dst (OrL dst src));
10449   effect(KILL cr);
10450 
10451   format %{ "orq     $dst, $src\t# long" %}
10452   ins_encode %{
10453     __ orq($dst$$Register, $src$$constant);
10454   %}
10455   ins_pipe(ialu_reg);
10456 %}
10457 
10458 // Or Register with Memory
10459 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10460 %{
10461   match(Set dst (OrL dst (LoadL src)));
10462   effect(KILL cr);
10463 
10464   ins_cost(150);
10465   format %{ "orq     $dst, $src\t# long" %}
10466   ins_encode %{
10467     __ orq($dst$$Register, $src$$Address);
10468   %}
10469   ins_pipe(ialu_reg_mem);
10470 %}
10471 
10472 // Or Memory with Register
10473 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10474 %{
10475   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10476   effect(KILL cr);
10477 
10478   ins_cost(150);
10479   format %{ "orq     $dst, $src\t# long" %}
10480   ins_encode %{
10481     __ orq($dst$$Address, $src$$Register);
10482   %}
10483   ins_pipe(ialu_mem_reg);
10484 %}
10485 
10486 // Or Memory with Immediate
10487 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10488 %{
10489   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10490   effect(KILL cr);
10491 
10492   ins_cost(125);
10493   format %{ "orq     $dst, $src\t# long" %}
10494   ins_encode %{
10495     __ orq($dst$$Address, $src$$constant);
10496   %}
10497   ins_pipe(ialu_mem_imm);
10498 %}
10499 
10500 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10501 %{
10502   // con should be a pure 64-bit power of 2 immediate
10503   // because AND/OR works well enough for 8/32-bit values.
10504   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10505 
10506   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10507   effect(KILL cr);
10508 
10509   ins_cost(125);
10510   format %{ "btsq    $dst, log2($con)\t# long" %}
10511   ins_encode %{
10512     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10513   %}
10514   ins_pipe(ialu_mem_imm);
10515 %}
10516 
10517 // Xor Instructions
10518 // Xor Register with Register
10519 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10520 %{
10521   match(Set dst (XorL dst src));
10522   effect(KILL cr);
10523 
10524   format %{ "xorq    $dst, $src\t# long" %}
10525   ins_encode %{
10526     __ xorq($dst$$Register, $src$$Register);
10527   %}
10528   ins_pipe(ialu_reg_reg);
10529 %}
10530 
10531 // Xor Register with Immediate -1
10532 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10533   match(Set dst (XorL dst imm));
10534 
10535   format %{ "notq   $dst" %}
10536   ins_encode %{
10537      __ notq($dst$$Register);
10538   %}
10539   ins_pipe(ialu_reg);
10540 %}
10541 
10542 // Xor Register with Immediate
10543 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10544 %{
10545   match(Set dst (XorL dst src));
10546   effect(KILL cr);
10547 
10548   format %{ "xorq    $dst, $src\t# long" %}
10549   ins_encode %{
10550     __ xorq($dst$$Register, $src$$constant);
10551   %}
10552   ins_pipe(ialu_reg);
10553 %}
10554 
10555 // Xor Register with Memory
10556 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10557 %{
10558   match(Set dst (XorL dst (LoadL src)));
10559   effect(KILL cr);
10560 
10561   ins_cost(150);
10562   format %{ "xorq    $dst, $src\t# long" %}
10563   ins_encode %{
10564     __ xorq($dst$$Register, $src$$Address);
10565   %}
10566   ins_pipe(ialu_reg_mem);
10567 %}
10568 
10569 // Xor Memory with Register
10570 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10571 %{
10572   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10573   effect(KILL cr);
10574 
10575   ins_cost(150);
10576   format %{ "xorq    $dst, $src\t# long" %}
10577   ins_encode %{
10578     __ xorq($dst$$Address, $src$$Register);
10579   %}
10580   ins_pipe(ialu_mem_reg);
10581 %}
10582 
10583 // Xor Memory with Immediate
10584 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10585 %{
10586   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10587   effect(KILL cr);
10588 
10589   ins_cost(125);
10590   format %{ "xorq    $dst, $src\t# long" %}
10591   ins_encode %{
10592     __ xorq($dst$$Address, $src$$constant);
10593   %}
10594   ins_pipe(ialu_mem_imm);
10595 %}
10596 
10597 // Convert Int to Boolean
10598 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10599 %{
10600   match(Set dst (Conv2B src));
10601   effect(KILL cr);
10602 
10603   format %{ "testl   $src, $src\t# ci2b\n\t"
10604             "setnz   $dst\n\t"
10605             "movzbl  $dst, $dst" %}
10606   ins_encode %{
10607     __ testl($src$$Register, $src$$Register);
10608     __ set_byte_if_not_zero($dst$$Register);
10609     __ movzbl($dst$$Register, $dst$$Register);
10610   %}
10611   ins_pipe(pipe_slow); // XXX
10612 %}
10613 
10614 // Convert Pointer to Boolean
10615 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10616 %{
10617   match(Set dst (Conv2B src));
10618   effect(KILL cr);
10619 
10620   format %{ "testq   $src, $src\t# cp2b\n\t"
10621             "setnz   $dst\n\t"
10622             "movzbl  $dst, $dst" %}
10623   ins_encode %{
10624     __ testq($src$$Register, $src$$Register);
10625     __ set_byte_if_not_zero($dst$$Register);
10626     __ movzbl($dst$$Register, $dst$$Register);
10627   %}
10628   ins_pipe(pipe_slow); // XXX
10629 %}
10630 
10631 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10632 %{
10633   match(Set dst (CmpLTMask p q));
10634   effect(KILL cr);
10635 
10636   ins_cost(400);
10637   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10638             "setlt   $dst\n\t"
10639             "movzbl  $dst, $dst\n\t"
10640             "negl    $dst" %}
10641   ins_encode %{
10642     __ cmpl($p$$Register, $q$$Register);
10643     __ setl($dst$$Register);
10644     __ movzbl($dst$$Register, $dst$$Register);
10645     __ negl($dst$$Register);
10646   %}
10647   ins_pipe(pipe_slow);
10648 %}
10649 
10650 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10651 %{
10652   match(Set dst (CmpLTMask dst zero));
10653   effect(KILL cr);
10654 
10655   ins_cost(100);
10656   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10657   ins_encode %{
10658     __ sarl($dst$$Register, 31);
10659   %}
10660   ins_pipe(ialu_reg);
10661 %}
10662 
10663 /* Better to save a register than avoid a branch */
10664 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10665 %{
10666   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10667   effect(KILL cr);
10668   ins_cost(300);
10669   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10670             "jge     done\n\t"
10671             "addl    $p,$y\n"
10672             "done:   " %}
10673   ins_encode %{
10674     Register Rp = $p$$Register;
10675     Register Rq = $q$$Register;
10676     Register Ry = $y$$Register;
10677     Label done;
10678     __ subl(Rp, Rq);
10679     __ jccb(Assembler::greaterEqual, done);
10680     __ addl(Rp, Ry);
10681     __ bind(done);
10682   %}
10683   ins_pipe(pipe_cmplt);
10684 %}
10685 
10686 /* Better to save a register than avoid a branch */
10687 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10688 %{
10689   match(Set y (AndI (CmpLTMask p q) y));
10690   effect(KILL cr);
10691 
10692   ins_cost(300);
10693 
10694   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10695             "jlt     done\n\t"
10696             "xorl    $y, $y\n"
10697             "done:   " %}
10698   ins_encode %{
10699     Register Rp = $p$$Register;
10700     Register Rq = $q$$Register;
10701     Register Ry = $y$$Register;
10702     Label done;
10703     __ cmpl(Rp, Rq);
10704     __ jccb(Assembler::less, done);
10705     __ xorl(Ry, Ry);
10706     __ bind(done);
10707   %}
10708   ins_pipe(pipe_cmplt);
10709 %}
10710 
10711 
10712 //---------- FP Instructions------------------------------------------------
10713 
10714 // Really expensive, avoid
10715 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10716 %{
10717   match(Set cr (CmpF src1 src2));
10718 
10719   ins_cost(500);
10720   format %{ "ucomiss $src1, $src2\n\t"
10721             "jnp,s   exit\n\t"
10722             "pushfq\t# saw NaN, set CF\n\t"
10723             "andq    [rsp], #0xffffff2b\n\t"
10724             "popfq\n"
10725     "exit:" %}
10726   ins_encode %{
10727     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10728     emit_cmpfp_fixup(_masm);
10729   %}
10730   ins_pipe(pipe_slow);
10731 %}
10732 
10733 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10734   match(Set cr (CmpF src1 src2));
10735 
10736   ins_cost(100);
10737   format %{ "ucomiss $src1, $src2" %}
10738   ins_encode %{
10739     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10740   %}
10741   ins_pipe(pipe_slow);
10742 %}
10743 
10744 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10745   match(Set cr (CmpF src1 (LoadF src2)));
10746 
10747   ins_cost(100);
10748   format %{ "ucomiss $src1, $src2" %}
10749   ins_encode %{
10750     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10751   %}
10752   ins_pipe(pipe_slow);
10753 %}
10754 
10755 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10756   match(Set cr (CmpF src con));
10757   ins_cost(100);
10758   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10759   ins_encode %{
10760     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10761   %}
10762   ins_pipe(pipe_slow);
10763 %}
10764 
10765 // Really expensive, avoid
10766 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10767 %{
10768   match(Set cr (CmpD src1 src2));
10769 
10770   ins_cost(500);
10771   format %{ "ucomisd $src1, $src2\n\t"
10772             "jnp,s   exit\n\t"
10773             "pushfq\t# saw NaN, set CF\n\t"
10774             "andq    [rsp], #0xffffff2b\n\t"
10775             "popfq\n"
10776     "exit:" %}
10777   ins_encode %{
10778     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10779     emit_cmpfp_fixup(_masm);
10780   %}
10781   ins_pipe(pipe_slow);
10782 %}
10783 
10784 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10785   match(Set cr (CmpD src1 src2));
10786 
10787   ins_cost(100);
10788   format %{ "ucomisd $src1, $src2 test" %}
10789   ins_encode %{
10790     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10791   %}
10792   ins_pipe(pipe_slow);
10793 %}
10794 
10795 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10796   match(Set cr (CmpD src1 (LoadD src2)));
10797 
10798   ins_cost(100);
10799   format %{ "ucomisd $src1, $src2" %}
10800   ins_encode %{
10801     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10802   %}
10803   ins_pipe(pipe_slow);
10804 %}
10805 
10806 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10807   match(Set cr (CmpD src con));
10808   ins_cost(100);
10809   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10810   ins_encode %{
10811     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10812   %}
10813   ins_pipe(pipe_slow);
10814 %}
10815 
10816 // Compare into -1,0,1
10817 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10818 %{
10819   match(Set dst (CmpF3 src1 src2));
10820   effect(KILL cr);
10821 
10822   ins_cost(275);
10823   format %{ "ucomiss $src1, $src2\n\t"
10824             "movl    $dst, #-1\n\t"
10825             "jp,s    done\n\t"
10826             "jb,s    done\n\t"
10827             "setne   $dst\n\t"
10828             "movzbl  $dst, $dst\n"
10829     "done:" %}
10830   ins_encode %{
10831     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10832     emit_cmpfp3(_masm, $dst$$Register);
10833   %}
10834   ins_pipe(pipe_slow);
10835 %}
10836 
10837 // Compare into -1,0,1
10838 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10839 %{
10840   match(Set dst (CmpF3 src1 (LoadF src2)));
10841   effect(KILL cr);
10842 
10843   ins_cost(275);
10844   format %{ "ucomiss $src1, $src2\n\t"
10845             "movl    $dst, #-1\n\t"
10846             "jp,s    done\n\t"
10847             "jb,s    done\n\t"
10848             "setne   $dst\n\t"
10849             "movzbl  $dst, $dst\n"
10850     "done:" %}
10851   ins_encode %{
10852     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10853     emit_cmpfp3(_masm, $dst$$Register);
10854   %}
10855   ins_pipe(pipe_slow);
10856 %}
10857 
10858 // Compare into -1,0,1
10859 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10860   match(Set dst (CmpF3 src con));
10861   effect(KILL cr);
10862 
10863   ins_cost(275);
10864   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10865             "movl    $dst, #-1\n\t"
10866             "jp,s    done\n\t"
10867             "jb,s    done\n\t"
10868             "setne   $dst\n\t"
10869             "movzbl  $dst, $dst\n"
10870     "done:" %}
10871   ins_encode %{
10872     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10873     emit_cmpfp3(_masm, $dst$$Register);
10874   %}
10875   ins_pipe(pipe_slow);
10876 %}
10877 
10878 // Compare into -1,0,1
10879 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10880 %{
10881   match(Set dst (CmpD3 src1 src2));
10882   effect(KILL cr);
10883 
10884   ins_cost(275);
10885   format %{ "ucomisd $src1, $src2\n\t"
10886             "movl    $dst, #-1\n\t"
10887             "jp,s    done\n\t"
10888             "jb,s    done\n\t"
10889             "setne   $dst\n\t"
10890             "movzbl  $dst, $dst\n"
10891     "done:" %}
10892   ins_encode %{
10893     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10894     emit_cmpfp3(_masm, $dst$$Register);
10895   %}
10896   ins_pipe(pipe_slow);
10897 %}
10898 
10899 // Compare into -1,0,1
10900 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10901 %{
10902   match(Set dst (CmpD3 src1 (LoadD src2)));
10903   effect(KILL cr);
10904 
10905   ins_cost(275);
10906   format %{ "ucomisd $src1, $src2\n\t"
10907             "movl    $dst, #-1\n\t"
10908             "jp,s    done\n\t"
10909             "jb,s    done\n\t"
10910             "setne   $dst\n\t"
10911             "movzbl  $dst, $dst\n"
10912     "done:" %}
10913   ins_encode %{
10914     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10915     emit_cmpfp3(_masm, $dst$$Register);
10916   %}
10917   ins_pipe(pipe_slow);
10918 %}
10919 
10920 // Compare into -1,0,1
10921 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10922   match(Set dst (CmpD3 src con));
10923   effect(KILL cr);
10924 
10925   ins_cost(275);
10926   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10927             "movl    $dst, #-1\n\t"
10928             "jp,s    done\n\t"
10929             "jb,s    done\n\t"
10930             "setne   $dst\n\t"
10931             "movzbl  $dst, $dst\n"
10932     "done:" %}
10933   ins_encode %{
10934     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10935     emit_cmpfp3(_masm, $dst$$Register);
10936   %}
10937   ins_pipe(pipe_slow);
10938 %}
10939 
10940 //----------Arithmetic Conversion Instructions---------------------------------
10941 
10942 instruct convF2D_reg_reg(regD dst, regF src)
10943 %{
10944   match(Set dst (ConvF2D src));
10945 
10946   format %{ "cvtss2sd $dst, $src" %}
10947   ins_encode %{
10948     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10949   %}
10950   ins_pipe(pipe_slow); // XXX
10951 %}
10952 
10953 instruct convF2D_reg_mem(regD dst, memory src)
10954 %{
10955   match(Set dst (ConvF2D (LoadF src)));
10956 
10957   format %{ "cvtss2sd $dst, $src" %}
10958   ins_encode %{
10959     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10960   %}
10961   ins_pipe(pipe_slow); // XXX
10962 %}
10963 
10964 instruct convD2F_reg_reg(regF dst, regD src)
10965 %{
10966   match(Set dst (ConvD2F src));
10967 
10968   format %{ "cvtsd2ss $dst, $src" %}
10969   ins_encode %{
10970     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10971   %}
10972   ins_pipe(pipe_slow); // XXX
10973 %}
10974 
10975 instruct convD2F_reg_mem(regF dst, memory src)
10976 %{
10977   match(Set dst (ConvD2F (LoadD src)));
10978 
10979   format %{ "cvtsd2ss $dst, $src" %}
10980   ins_encode %{
10981     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10982   %}
10983   ins_pipe(pipe_slow); // XXX
10984 %}
10985 
10986 // XXX do mem variants
10987 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10988 %{
10989   match(Set dst (ConvF2I src));
10990   effect(KILL cr);
10991   format %{ "convert_f2i $dst,$src" %}
10992   ins_encode %{
10993     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10994   %}
10995   ins_pipe(pipe_slow);
10996 %}
10997 
10998 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10999 %{
11000   match(Set dst (ConvF2L src));
11001   effect(KILL cr);
11002   format %{ "convert_f2l $dst,$src"%}
11003   ins_encode %{
11004     __ convert_f2l($dst$$Register, $src$$XMMRegister);
11005   %}
11006   ins_pipe(pipe_slow);
11007 %}
11008 
11009 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11010 %{
11011   match(Set dst (ConvD2I src));
11012   effect(KILL cr);
11013   format %{ "convert_d2i $dst,$src"%}
11014   ins_encode %{
11015     __ convert_d2i($dst$$Register, $src$$XMMRegister);
11016   %}
11017   ins_pipe(pipe_slow);
11018 %}
11019 
11020 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11021 %{
11022   match(Set dst (ConvD2L src));
11023   effect(KILL cr);
11024   format %{ "convert_d2l $dst,$src"%}
11025   ins_encode %{
11026     __ convert_d2l($dst$$Register, $src$$XMMRegister);
11027   %}
11028   ins_pipe(pipe_slow);
11029 %}
11030 
11031 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11032 %{
11033   match(Set dst (RoundD src));
11034   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11035   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11036   ins_encode %{
11037     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11038   %}
11039   ins_pipe(pipe_slow);
11040 %}
11041 
11042 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11043 %{
11044   match(Set dst (RoundF src));
11045   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11046   format %{ "round_float $dst,$src" %}
11047   ins_encode %{
11048     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11049   %}
11050   ins_pipe(pipe_slow);
11051 %}
11052 
11053 instruct convI2F_reg_reg(regF dst, rRegI src)
11054 %{
11055   predicate(!UseXmmI2F);
11056   match(Set dst (ConvI2F src));
11057 
11058   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11059   ins_encode %{
11060     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11061   %}
11062   ins_pipe(pipe_slow); // XXX
11063 %}
11064 
11065 instruct convI2F_reg_mem(regF dst, memory src)
11066 %{
11067   match(Set dst (ConvI2F (LoadI src)));
11068 
11069   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11070   ins_encode %{
11071     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11072   %}
11073   ins_pipe(pipe_slow); // XXX
11074 %}
11075 
11076 instruct convI2D_reg_reg(regD dst, rRegI src)
11077 %{
11078   predicate(!UseXmmI2D);
11079   match(Set dst (ConvI2D src));
11080 
11081   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11082   ins_encode %{
11083     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11084   %}
11085   ins_pipe(pipe_slow); // XXX
11086 %}
11087 
11088 instruct convI2D_reg_mem(regD dst, memory src)
11089 %{
11090   match(Set dst (ConvI2D (LoadI src)));
11091 
11092   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11093   ins_encode %{
11094     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11095   %}
11096   ins_pipe(pipe_slow); // XXX
11097 %}
11098 
11099 instruct convXI2F_reg(regF dst, rRegI src)
11100 %{
11101   predicate(UseXmmI2F);
11102   match(Set dst (ConvI2F src));
11103 
11104   format %{ "movdl $dst, $src\n\t"
11105             "cvtdq2psl $dst, $dst\t# i2f" %}
11106   ins_encode %{
11107     __ movdl($dst$$XMMRegister, $src$$Register);
11108     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11109   %}
11110   ins_pipe(pipe_slow); // XXX
11111 %}
11112 
11113 instruct convXI2D_reg(regD dst, rRegI src)
11114 %{
11115   predicate(UseXmmI2D);
11116   match(Set dst (ConvI2D src));
11117 
11118   format %{ "movdl $dst, $src\n\t"
11119             "cvtdq2pdl $dst, $dst\t# i2d" %}
11120   ins_encode %{
11121     __ movdl($dst$$XMMRegister, $src$$Register);
11122     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11123   %}
11124   ins_pipe(pipe_slow); // XXX
11125 %}
11126 
11127 instruct convL2F_reg_reg(regF dst, rRegL src)
11128 %{
11129   match(Set dst (ConvL2F src));
11130 
11131   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11132   ins_encode %{
11133     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11134   %}
11135   ins_pipe(pipe_slow); // XXX
11136 %}
11137 
11138 instruct convL2F_reg_mem(regF dst, memory src)
11139 %{
11140   match(Set dst (ConvL2F (LoadL src)));
11141 
11142   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11143   ins_encode %{
11144     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11145   %}
11146   ins_pipe(pipe_slow); // XXX
11147 %}
11148 
11149 instruct convL2D_reg_reg(regD dst, rRegL src)
11150 %{
11151   match(Set dst (ConvL2D src));
11152 
11153   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11154   ins_encode %{
11155     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11156   %}
11157   ins_pipe(pipe_slow); // XXX
11158 %}
11159 
11160 instruct convL2D_reg_mem(regD dst, memory src)
11161 %{
11162   match(Set dst (ConvL2D (LoadL src)));
11163 
11164   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11165   ins_encode %{
11166     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11167   %}
11168   ins_pipe(pipe_slow); // XXX
11169 %}
11170 
11171 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11172 %{
11173   match(Set dst (ConvI2L src));
11174 
11175   ins_cost(125);
11176   format %{ "movslq  $dst, $src\t# i2l" %}
11177   ins_encode %{
11178     __ movslq($dst$$Register, $src$$Register);
11179   %}
11180   ins_pipe(ialu_reg_reg);
11181 %}
11182 
11183 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11184 // %{
11185 //   match(Set dst (ConvI2L src));
11186 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11187 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11188 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11189 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11190 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11191 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11192 
11193 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11194 //   ins_encode(enc_copy(dst, src));
11195 // //   opcode(0x63); // needs REX.W
11196 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11197 //   ins_pipe(ialu_reg_reg);
11198 // %}
11199 
11200 // Zero-extend convert int to long
11201 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11202 %{
11203   match(Set dst (AndL (ConvI2L src) mask));
11204 
11205   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11206   ins_encode %{
11207     if ($dst$$reg != $src$$reg) {
11208       __ movl($dst$$Register, $src$$Register);
11209     }
11210   %}
11211   ins_pipe(ialu_reg_reg);
11212 %}
11213 
11214 // Zero-extend convert int to long
11215 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11216 %{
11217   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11218 
11219   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11220   ins_encode %{
11221     __ movl($dst$$Register, $src$$Address);
11222   %}
11223   ins_pipe(ialu_reg_mem);
11224 %}
11225 
11226 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11227 %{
11228   match(Set dst (AndL src mask));
11229 
11230   format %{ "movl    $dst, $src\t# zero-extend long" %}
11231   ins_encode %{
11232     __ movl($dst$$Register, $src$$Register);
11233   %}
11234   ins_pipe(ialu_reg_reg);
11235 %}
11236 
11237 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11238 %{
11239   match(Set dst (ConvL2I src));
11240 
11241   format %{ "movl    $dst, $src\t# l2i" %}
11242   ins_encode %{
11243     __ movl($dst$$Register, $src$$Register);
11244   %}
11245   ins_pipe(ialu_reg_reg);
11246 %}
11247 
11248 
11249 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11250   match(Set dst (MoveF2I src));
11251   effect(DEF dst, USE src);
11252 
11253   ins_cost(125);
11254   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11255   ins_encode %{
11256     __ movl($dst$$Register, Address(rsp, $src$$disp));
11257   %}
11258   ins_pipe(ialu_reg_mem);
11259 %}
11260 
11261 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11262   match(Set dst (MoveI2F src));
11263   effect(DEF dst, USE src);
11264 
11265   ins_cost(125);
11266   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11267   ins_encode %{
11268     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11269   %}
11270   ins_pipe(pipe_slow);
11271 %}
11272 
11273 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11274   match(Set dst (MoveD2L src));
11275   effect(DEF dst, USE src);
11276 
11277   ins_cost(125);
11278   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11279   ins_encode %{
11280     __ movq($dst$$Register, Address(rsp, $src$$disp));
11281   %}
11282   ins_pipe(ialu_reg_mem);
11283 %}
11284 
11285 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11286   predicate(!UseXmmLoadAndClearUpper);
11287   match(Set dst (MoveL2D src));
11288   effect(DEF dst, USE src);
11289 
11290   ins_cost(125);
11291   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11292   ins_encode %{
11293     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11294   %}
11295   ins_pipe(pipe_slow);
11296 %}
11297 
11298 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11299   predicate(UseXmmLoadAndClearUpper);
11300   match(Set dst (MoveL2D src));
11301   effect(DEF dst, USE src);
11302 
11303   ins_cost(125);
11304   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11305   ins_encode %{
11306     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11307   %}
11308   ins_pipe(pipe_slow);
11309 %}
11310 
11311 
11312 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11313   match(Set dst (MoveF2I src));
11314   effect(DEF dst, USE src);
11315 
11316   ins_cost(95); // XXX
11317   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11318   ins_encode %{
11319     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11320   %}
11321   ins_pipe(pipe_slow);
11322 %}
11323 
11324 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11325   match(Set dst (MoveI2F src));
11326   effect(DEF dst, USE src);
11327 
11328   ins_cost(100);
11329   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11330   ins_encode %{
11331     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11332   %}
11333   ins_pipe( ialu_mem_reg );
11334 %}
11335 
11336 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11337   match(Set dst (MoveD2L src));
11338   effect(DEF dst, USE src);
11339 
11340   ins_cost(95); // XXX
11341   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11342   ins_encode %{
11343     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11344   %}
11345   ins_pipe(pipe_slow);
11346 %}
11347 
11348 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11349   match(Set dst (MoveL2D src));
11350   effect(DEF dst, USE src);
11351 
11352   ins_cost(100);
11353   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11354   ins_encode %{
11355     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11356   %}
11357   ins_pipe(ialu_mem_reg);
11358 %}
11359 
11360 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11361   match(Set dst (MoveF2I src));
11362   effect(DEF dst, USE src);
11363   ins_cost(85);
11364   format %{ "movd    $dst,$src\t# MoveF2I" %}
11365   ins_encode %{
11366     __ movdl($dst$$Register, $src$$XMMRegister);
11367   %}
11368   ins_pipe( pipe_slow );
11369 %}
11370 
11371 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11372   match(Set dst (MoveD2L src));
11373   effect(DEF dst, USE src);
11374   ins_cost(85);
11375   format %{ "movd    $dst,$src\t# MoveD2L" %}
11376   ins_encode %{
11377     __ movdq($dst$$Register, $src$$XMMRegister);
11378   %}
11379   ins_pipe( pipe_slow );
11380 %}
11381 
11382 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11383   match(Set dst (MoveI2F src));
11384   effect(DEF dst, USE src);
11385   ins_cost(100);
11386   format %{ "movd    $dst,$src\t# MoveI2F" %}
11387   ins_encode %{
11388     __ movdl($dst$$XMMRegister, $src$$Register);
11389   %}
11390   ins_pipe( pipe_slow );
11391 %}
11392 
11393 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11394   match(Set dst (MoveL2D src));
11395   effect(DEF dst, USE src);
11396   ins_cost(100);
11397   format %{ "movd    $dst,$src\t# MoveL2D" %}
11398   ins_encode %{
11399      __ movdq($dst$$XMMRegister, $src$$Register);
11400   %}
11401   ins_pipe( pipe_slow );
11402 %}
11403 
11404 // Fast clearing of an array
11405 // Small ClearArray non-AVX512.
11406 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11407                   Universe dummy, rFlagsReg cr)
11408 %{
11409   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
11410   match(Set dummy (ClearArray cnt base));
11411   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11412 
11413   format %{ $$template
11414     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11415     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11416     $$emit$$"jg      LARGE\n\t"
11417     $$emit$$"dec     rcx\n\t"
11418     $$emit$$"js      DONE\t# Zero length\n\t"
11419     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11420     $$emit$$"dec     rcx\n\t"
11421     $$emit$$"jge     LOOP\n\t"
11422     $$emit$$"jmp     DONE\n\t"
11423     $$emit$$"# LARGE:\n\t"
11424     if (UseFastStosb) {
11425        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11426        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11427     } else if (UseXMMForObjInit) {
11428        $$emit$$"mov     rdi,rax\n\t"
11429        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11430        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11431        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11432        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11433        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11434        $$emit$$"add     0x40,rax\n\t"
11435        $$emit$$"# L_zero_64_bytes:\n\t"
11436        $$emit$$"sub     0x8,rcx\n\t"
11437        $$emit$$"jge     L_loop\n\t"
11438        $$emit$$"add     0x4,rcx\n\t"
11439        $$emit$$"jl      L_tail\n\t"
11440        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11441        $$emit$$"add     0x20,rax\n\t"
11442        $$emit$$"sub     0x4,rcx\n\t"
11443        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11444        $$emit$$"add     0x4,rcx\n\t"
11445        $$emit$$"jle     L_end\n\t"
11446        $$emit$$"dec     rcx\n\t"
11447        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11448        $$emit$$"vmovq   xmm0,(rax)\n\t"
11449        $$emit$$"add     0x8,rax\n\t"
11450        $$emit$$"dec     rcx\n\t"
11451        $$emit$$"jge     L_sloop\n\t"
11452        $$emit$$"# L_end:\n\t"
11453     } else {
11454        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11455     }
11456     $$emit$$"# DONE"
11457   %}
11458   ins_encode %{
11459     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11460                  $tmp$$XMMRegister, false, knoreg);
11461   %}
11462   ins_pipe(pipe_slow);
11463 %}
11464 
11465 // Small ClearArray AVX512 non-constant length.
11466 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11467                        Universe dummy, rFlagsReg cr)
11468 %{
11469   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
11470   match(Set dummy (ClearArray cnt base));
11471   ins_cost(125);
11472   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11473 
11474   format %{ $$template
11475     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11476     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11477     $$emit$$"jg      LARGE\n\t"
11478     $$emit$$"dec     rcx\n\t"
11479     $$emit$$"js      DONE\t# Zero length\n\t"
11480     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11481     $$emit$$"dec     rcx\n\t"
11482     $$emit$$"jge     LOOP\n\t"
11483     $$emit$$"jmp     DONE\n\t"
11484     $$emit$$"# LARGE:\n\t"
11485     if (UseFastStosb) {
11486        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11487        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11488     } else if (UseXMMForObjInit) {
11489        $$emit$$"mov     rdi,rax\n\t"
11490        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11491        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11492        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11493        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11494        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11495        $$emit$$"add     0x40,rax\n\t"
11496        $$emit$$"# L_zero_64_bytes:\n\t"
11497        $$emit$$"sub     0x8,rcx\n\t"
11498        $$emit$$"jge     L_loop\n\t"
11499        $$emit$$"add     0x4,rcx\n\t"
11500        $$emit$$"jl      L_tail\n\t"
11501        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11502        $$emit$$"add     0x20,rax\n\t"
11503        $$emit$$"sub     0x4,rcx\n\t"
11504        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11505        $$emit$$"add     0x4,rcx\n\t"
11506        $$emit$$"jle     L_end\n\t"
11507        $$emit$$"dec     rcx\n\t"
11508        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11509        $$emit$$"vmovq   xmm0,(rax)\n\t"
11510        $$emit$$"add     0x8,rax\n\t"
11511        $$emit$$"dec     rcx\n\t"
11512        $$emit$$"jge     L_sloop\n\t"
11513        $$emit$$"# L_end:\n\t"
11514     } else {
11515        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11516     }
11517     $$emit$$"# DONE"
11518   %}
11519   ins_encode %{
11520     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11521                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
11522   %}
11523   ins_pipe(pipe_slow);
11524 %}
11525 
11526 // Large ClearArray non-AVX512.
11527 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11528                         Universe dummy, rFlagsReg cr)
11529 %{
11530   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
11531   match(Set dummy (ClearArray cnt base));
11532   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11533 
11534   format %{ $$template
11535     if (UseFastStosb) {
11536        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11537        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11538        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11539     } else if (UseXMMForObjInit) {
11540        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11541        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11542        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11543        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11544        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11545        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11546        $$emit$$"add     0x40,rax\n\t"
11547        $$emit$$"# L_zero_64_bytes:\n\t"
11548        $$emit$$"sub     0x8,rcx\n\t"
11549        $$emit$$"jge     L_loop\n\t"
11550        $$emit$$"add     0x4,rcx\n\t"
11551        $$emit$$"jl      L_tail\n\t"
11552        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11553        $$emit$$"add     0x20,rax\n\t"
11554        $$emit$$"sub     0x4,rcx\n\t"
11555        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11556        $$emit$$"add     0x4,rcx\n\t"
11557        $$emit$$"jle     L_end\n\t"
11558        $$emit$$"dec     rcx\n\t"
11559        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11560        $$emit$$"vmovq   xmm0,(rax)\n\t"
11561        $$emit$$"add     0x8,rax\n\t"
11562        $$emit$$"dec     rcx\n\t"
11563        $$emit$$"jge     L_sloop\n\t"
11564        $$emit$$"# L_end:\n\t"
11565     } else {
11566        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11567        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11568     }
11569   %}
11570   ins_encode %{
11571     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11572                  $tmp$$XMMRegister, true, knoreg);
11573   %}
11574   ins_pipe(pipe_slow);
11575 %}
11576 
11577 // Large ClearArray AVX512.
11578 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
11579                              Universe dummy, rFlagsReg cr)
11580 %{
11581   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
11582   match(Set dummy (ClearArray cnt base));
11583   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
11584 
11585   format %{ $$template
11586     if (UseFastStosb) {
11587        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11588        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11589        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11590     } else if (UseXMMForObjInit) {
11591        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11592        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11593        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11594        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11595        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11596        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11597        $$emit$$"add     0x40,rax\n\t"
11598        $$emit$$"# L_zero_64_bytes:\n\t"
11599        $$emit$$"sub     0x8,rcx\n\t"
11600        $$emit$$"jge     L_loop\n\t"
11601        $$emit$$"add     0x4,rcx\n\t"
11602        $$emit$$"jl      L_tail\n\t"
11603        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11604        $$emit$$"add     0x20,rax\n\t"
11605        $$emit$$"sub     0x4,rcx\n\t"
11606        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11607        $$emit$$"add     0x4,rcx\n\t"
11608        $$emit$$"jle     L_end\n\t"
11609        $$emit$$"dec     rcx\n\t"
11610        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11611        $$emit$$"vmovq   xmm0,(rax)\n\t"
11612        $$emit$$"add     0x8,rax\n\t"
11613        $$emit$$"dec     rcx\n\t"
11614        $$emit$$"jge     L_sloop\n\t"
11615        $$emit$$"# L_end:\n\t"
11616     } else {
11617        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11618        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11619     }
11620   %}
11621   ins_encode %{
11622     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11623                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
11624   %}
11625   ins_pipe(pipe_slow);
11626 %}
11627 
11628 // Small ClearArray AVX512 constant length.
11629 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
11630 %{
11631   predicate(!((ClearArrayNode*)n)->is_large() &&
11632               ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11633   match(Set dummy (ClearArray cnt base));
11634   ins_cost(100);
11635   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
11636   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11637   ins_encode %{
11638    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11639   %}
11640   ins_pipe(pipe_slow);
11641 %}
11642 
11643 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11644                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11645 %{
11646   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11647   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11648   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11649 
11650   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11651   ins_encode %{
11652     __ string_compare($str1$$Register, $str2$$Register,
11653                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11654                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11655   %}
11656   ins_pipe( pipe_slow );
11657 %}
11658 
11659 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11660                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11661 %{
11662   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11663   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11664   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11665 
11666   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11667   ins_encode %{
11668     __ string_compare($str1$$Register, $str2$$Register,
11669                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11670                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11671   %}
11672   ins_pipe( pipe_slow );
11673 %}
11674 
11675 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11676                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11677 %{
11678   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11679   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11680   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11681 
11682   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11683   ins_encode %{
11684     __ string_compare($str1$$Register, $str2$$Register,
11685                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11686                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11687   %}
11688   ins_pipe( pipe_slow );
11689 %}
11690 
11691 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11692                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11693 %{
11694   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11695   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11696   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11697 
11698   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11699   ins_encode %{
11700     __ string_compare($str1$$Register, $str2$$Register,
11701                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11702                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11703   %}
11704   ins_pipe( pipe_slow );
11705 %}
11706 
11707 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11708                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11709 %{
11710   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11711   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11712   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11713 
11714   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11715   ins_encode %{
11716     __ string_compare($str1$$Register, $str2$$Register,
11717                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11718                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11719   %}
11720   ins_pipe( pipe_slow );
11721 %}
11722 
11723 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11724                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11725 %{
11726   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11727   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11728   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11729 
11730   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11731   ins_encode %{
11732     __ string_compare($str1$$Register, $str2$$Register,
11733                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11734                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11735   %}
11736   ins_pipe( pipe_slow );
11737 %}
11738 
11739 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11740                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11741 %{
11742   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11743   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11744   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11745 
11746   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11747   ins_encode %{
11748     __ string_compare($str2$$Register, $str1$$Register,
11749                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11750                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11751   %}
11752   ins_pipe( pipe_slow );
11753 %}
11754 
11755 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11756                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11757 %{
11758   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11759   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11760   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11761 
11762   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11763   ins_encode %{
11764     __ string_compare($str2$$Register, $str1$$Register,
11765                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11766                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11767   %}
11768   ins_pipe( pipe_slow );
11769 %}
11770 
11771 // fast search of substring with known size.
11772 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11773                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11774 %{
11775   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11776   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11777   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11778 
11779   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11780   ins_encode %{
11781     int icnt2 = (int)$int_cnt2$$constant;
11782     if (icnt2 >= 16) {
11783       // IndexOf for constant substrings with size >= 16 elements
11784       // which don't need to be loaded through stack.
11785       __ string_indexofC8($str1$$Register, $str2$$Register,
11786                           $cnt1$$Register, $cnt2$$Register,
11787                           icnt2, $result$$Register,
11788                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11789     } else {
11790       // Small strings are loaded through stack if they cross page boundary.
11791       __ string_indexof($str1$$Register, $str2$$Register,
11792                         $cnt1$$Register, $cnt2$$Register,
11793                         icnt2, $result$$Register,
11794                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11795     }
11796   %}
11797   ins_pipe( pipe_slow );
11798 %}
11799 
11800 // fast search of substring with known size.
11801 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11802                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11803 %{
11804   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11805   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11806   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11807 
11808   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11809   ins_encode %{
11810     int icnt2 = (int)$int_cnt2$$constant;
11811     if (icnt2 >= 8) {
11812       // IndexOf for constant substrings with size >= 8 elements
11813       // which don't need to be loaded through stack.
11814       __ string_indexofC8($str1$$Register, $str2$$Register,
11815                           $cnt1$$Register, $cnt2$$Register,
11816                           icnt2, $result$$Register,
11817                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11818     } else {
11819       // Small strings are loaded through stack if they cross page boundary.
11820       __ string_indexof($str1$$Register, $str2$$Register,
11821                         $cnt1$$Register, $cnt2$$Register,
11822                         icnt2, $result$$Register,
11823                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11824     }
11825   %}
11826   ins_pipe( pipe_slow );
11827 %}
11828 
11829 // fast search of substring with known size.
11830 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11831                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11832 %{
11833   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11834   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11835   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11836 
11837   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11838   ins_encode %{
11839     int icnt2 = (int)$int_cnt2$$constant;
11840     if (icnt2 >= 8) {
11841       // IndexOf for constant substrings with size >= 8 elements
11842       // which don't need to be loaded through stack.
11843       __ string_indexofC8($str1$$Register, $str2$$Register,
11844                           $cnt1$$Register, $cnt2$$Register,
11845                           icnt2, $result$$Register,
11846                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11847     } else {
11848       // Small strings are loaded through stack if they cross page boundary.
11849       __ string_indexof($str1$$Register, $str2$$Register,
11850                         $cnt1$$Register, $cnt2$$Register,
11851                         icnt2, $result$$Register,
11852                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11853     }
11854   %}
11855   ins_pipe( pipe_slow );
11856 %}
11857 
11858 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11859                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11860 %{
11861   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11862   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11863   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11864 
11865   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11866   ins_encode %{
11867     __ string_indexof($str1$$Register, $str2$$Register,
11868                       $cnt1$$Register, $cnt2$$Register,
11869                       (-1), $result$$Register,
11870                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11871   %}
11872   ins_pipe( pipe_slow );
11873 %}
11874 
11875 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11876                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11877 %{
11878   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11879   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11880   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11881 
11882   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11883   ins_encode %{
11884     __ string_indexof($str1$$Register, $str2$$Register,
11885                       $cnt1$$Register, $cnt2$$Register,
11886                       (-1), $result$$Register,
11887                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11888   %}
11889   ins_pipe( pipe_slow );
11890 %}
11891 
11892 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11893                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11894 %{
11895   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11896   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11897   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11898 
11899   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11900   ins_encode %{
11901     __ string_indexof($str1$$Register, $str2$$Register,
11902                       $cnt1$$Register, $cnt2$$Register,
11903                       (-1), $result$$Register,
11904                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11905   %}
11906   ins_pipe( pipe_slow );
11907 %}
11908 
11909 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11910                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11911 %{
11912   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11913   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11914   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11915   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11916   ins_encode %{
11917     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11918                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11919   %}
11920   ins_pipe( pipe_slow );
11921 %}
11922 
11923 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11924                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11925 %{
11926   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11927   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11928   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11929   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11930   ins_encode %{
11931     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11932                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11933   %}
11934   ins_pipe( pipe_slow );
11935 %}
11936 
11937 // fast string equals
11938 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11939                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11940 %{
11941   predicate(!VM_Version::supports_avx512vlbw());
11942   match(Set result (StrEquals (Binary str1 str2) cnt));
11943   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11944 
11945   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11946   ins_encode %{
11947     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11948                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11949                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11950   %}
11951   ins_pipe( pipe_slow );
11952 %}
11953 
11954 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11955                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11956 %{
11957   predicate(VM_Version::supports_avx512vlbw());
11958   match(Set result (StrEquals (Binary str1 str2) cnt));
11959   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11960 
11961   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11962   ins_encode %{
11963     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11964                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11965                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11966   %}
11967   ins_pipe( pipe_slow );
11968 %}
11969 
11970 // fast array equals
11971 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11972                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11973 %{
11974   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11975   match(Set result (AryEq ary1 ary2));
11976   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11977 
11978   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11979   ins_encode %{
11980     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11981                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11982                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11983   %}
11984   ins_pipe( pipe_slow );
11985 %}
11986 
11987 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11988                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11989 %{
11990   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11991   match(Set result (AryEq ary1 ary2));
11992   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11993 
11994   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11995   ins_encode %{
11996     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11997                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11998                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11999   %}
12000   ins_pipe( pipe_slow );
12001 %}
12002 
12003 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12004                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12005 %{
12006   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12007   match(Set result (AryEq ary1 ary2));
12008   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12009 
12010   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12011   ins_encode %{
12012     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12013                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12014                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12015   %}
12016   ins_pipe( pipe_slow );
12017 %}
12018 
12019 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12020                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12021 %{
12022   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12023   match(Set result (AryEq ary1 ary2));
12024   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12025 
12026   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12027   ins_encode %{
12028     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12029                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12030                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12031   %}
12032   ins_pipe( pipe_slow );
12033 %}
12034 
12035 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12036                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12037 %{
12038   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12039   match(Set result (CountPositives ary1 len));
12040   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12041 
12042   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12043   ins_encode %{
12044     __ count_positives($ary1$$Register, $len$$Register,
12045                        $result$$Register, $tmp3$$Register,
12046                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12047   %}
12048   ins_pipe( pipe_slow );
12049 %}
12050 
12051 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12052                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12053 %{
12054   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12055   match(Set result (CountPositives ary1 len));
12056   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12057 
12058   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12059   ins_encode %{
12060     __ count_positives($ary1$$Register, $len$$Register,
12061                        $result$$Register, $tmp3$$Register,
12062                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12063   %}
12064   ins_pipe( pipe_slow );
12065 %}
12066 
12067 // fast char[] to byte[] compression
12068 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12069                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12070   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12071   match(Set result (StrCompressedCopy src (Binary dst len)));
12072   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12073          USE_KILL len, KILL tmp5, KILL cr);
12074 
12075   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12076   ins_encode %{
12077     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12078                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12079                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12080                            knoreg, knoreg);
12081   %}
12082   ins_pipe( pipe_slow );
12083 %}
12084 
12085 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12086                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12087   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12088   match(Set result (StrCompressedCopy src (Binary dst len)));
12089   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12090          USE_KILL len, KILL tmp5, KILL cr);
12091 
12092   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12093   ins_encode %{
12094     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12095                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12096                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12097                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12098   %}
12099   ins_pipe( pipe_slow );
12100 %}
12101 // fast byte[] to char[] inflation
12102 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12103                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12104   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12105   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12106   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12107 
12108   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12109   ins_encode %{
12110     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12111                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12112   %}
12113   ins_pipe( pipe_slow );
12114 %}
12115 
12116 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12117                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12118   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12119   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12120   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12121 
12122   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12123   ins_encode %{
12124     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12125                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12126   %}
12127   ins_pipe( pipe_slow );
12128 %}
12129 
12130 // encode char[] to byte[] in ISO_8859_1
12131 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12132                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12133                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12134   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12135   match(Set result (EncodeISOArray src (Binary dst len)));
12136   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12137 
12138   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12139   ins_encode %{
12140     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12141                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12142                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12143   %}
12144   ins_pipe( pipe_slow );
12145 %}
12146 
12147 // encode char[] to byte[] in ASCII
12148 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12149                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12150                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12151   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12152   match(Set result (EncodeISOArray src (Binary dst len)));
12153   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12154 
12155   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12156   ins_encode %{
12157     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12158                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12159                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12160   %}
12161   ins_pipe( pipe_slow );
12162 %}
12163 
12164 //----------Overflow Math Instructions-----------------------------------------
12165 
12166 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12167 %{
12168   match(Set cr (OverflowAddI op1 op2));
12169   effect(DEF cr, USE_KILL op1, USE op2);
12170 
12171   format %{ "addl    $op1, $op2\t# overflow check int" %}
12172 
12173   ins_encode %{
12174     __ addl($op1$$Register, $op2$$Register);
12175   %}
12176   ins_pipe(ialu_reg_reg);
12177 %}
12178 
12179 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12180 %{
12181   match(Set cr (OverflowAddI op1 op2));
12182   effect(DEF cr, USE_KILL op1, USE op2);
12183 
12184   format %{ "addl    $op1, $op2\t# overflow check int" %}
12185 
12186   ins_encode %{
12187     __ addl($op1$$Register, $op2$$constant);
12188   %}
12189   ins_pipe(ialu_reg_reg);
12190 %}
12191 
12192 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12193 %{
12194   match(Set cr (OverflowAddL op1 op2));
12195   effect(DEF cr, USE_KILL op1, USE op2);
12196 
12197   format %{ "addq    $op1, $op2\t# overflow check long" %}
12198   ins_encode %{
12199     __ addq($op1$$Register, $op2$$Register);
12200   %}
12201   ins_pipe(ialu_reg_reg);
12202 %}
12203 
12204 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12205 %{
12206   match(Set cr (OverflowAddL op1 op2));
12207   effect(DEF cr, USE_KILL op1, USE op2);
12208 
12209   format %{ "addq    $op1, $op2\t# overflow check long" %}
12210   ins_encode %{
12211     __ addq($op1$$Register, $op2$$constant);
12212   %}
12213   ins_pipe(ialu_reg_reg);
12214 %}
12215 
12216 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12217 %{
12218   match(Set cr (OverflowSubI op1 op2));
12219 
12220   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12221   ins_encode %{
12222     __ cmpl($op1$$Register, $op2$$Register);
12223   %}
12224   ins_pipe(ialu_reg_reg);
12225 %}
12226 
12227 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12228 %{
12229   match(Set cr (OverflowSubI op1 op2));
12230 
12231   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12232   ins_encode %{
12233     __ cmpl($op1$$Register, $op2$$constant);
12234   %}
12235   ins_pipe(ialu_reg_reg);
12236 %}
12237 
12238 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12239 %{
12240   match(Set cr (OverflowSubL op1 op2));
12241 
12242   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12243   ins_encode %{
12244     __ cmpq($op1$$Register, $op2$$Register);
12245   %}
12246   ins_pipe(ialu_reg_reg);
12247 %}
12248 
12249 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12250 %{
12251   match(Set cr (OverflowSubL op1 op2));
12252 
12253   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12254   ins_encode %{
12255     __ cmpq($op1$$Register, $op2$$constant);
12256   %}
12257   ins_pipe(ialu_reg_reg);
12258 %}
12259 
12260 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12261 %{
12262   match(Set cr (OverflowSubI zero op2));
12263   effect(DEF cr, USE_KILL op2);
12264 
12265   format %{ "negl    $op2\t# overflow check int" %}
12266   ins_encode %{
12267     __ negl($op2$$Register);
12268   %}
12269   ins_pipe(ialu_reg_reg);
12270 %}
12271 
12272 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12273 %{
12274   match(Set cr (OverflowSubL zero op2));
12275   effect(DEF cr, USE_KILL op2);
12276 
12277   format %{ "negq    $op2\t# overflow check long" %}
12278   ins_encode %{
12279     __ negq($op2$$Register);
12280   %}
12281   ins_pipe(ialu_reg_reg);
12282 %}
12283 
12284 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12285 %{
12286   match(Set cr (OverflowMulI op1 op2));
12287   effect(DEF cr, USE_KILL op1, USE op2);
12288 
12289   format %{ "imull    $op1, $op2\t# overflow check int" %}
12290   ins_encode %{
12291     __ imull($op1$$Register, $op2$$Register);
12292   %}
12293   ins_pipe(ialu_reg_reg_alu0);
12294 %}
12295 
12296 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12297 %{
12298   match(Set cr (OverflowMulI op1 op2));
12299   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12300 
12301   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12302   ins_encode %{
12303     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12304   %}
12305   ins_pipe(ialu_reg_reg_alu0);
12306 %}
12307 
12308 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12309 %{
12310   match(Set cr (OverflowMulL op1 op2));
12311   effect(DEF cr, USE_KILL op1, USE op2);
12312 
12313   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12314   ins_encode %{
12315     __ imulq($op1$$Register, $op2$$Register);
12316   %}
12317   ins_pipe(ialu_reg_reg_alu0);
12318 %}
12319 
12320 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12321 %{
12322   match(Set cr (OverflowMulL op1 op2));
12323   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12324 
12325   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12326   ins_encode %{
12327     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12328   %}
12329   ins_pipe(ialu_reg_reg_alu0);
12330 %}
12331 
12332 
12333 //----------Control Flow Instructions------------------------------------------
12334 // Signed compare Instructions
12335 
12336 // XXX more variants!!
12337 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12338 %{
12339   match(Set cr (CmpI op1 op2));
12340   effect(DEF cr, USE op1, USE op2);
12341 
12342   format %{ "cmpl    $op1, $op2" %}
12343   ins_encode %{
12344     __ cmpl($op1$$Register, $op2$$Register);
12345   %}
12346   ins_pipe(ialu_cr_reg_reg);
12347 %}
12348 
12349 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12350 %{
12351   match(Set cr (CmpI op1 op2));
12352 
12353   format %{ "cmpl    $op1, $op2" %}
12354   ins_encode %{
12355     __ cmpl($op1$$Register, $op2$$constant);
12356   %}
12357   ins_pipe(ialu_cr_reg_imm);
12358 %}
12359 
12360 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12361 %{
12362   match(Set cr (CmpI op1 (LoadI op2)));
12363 
12364   ins_cost(500); // XXX
12365   format %{ "cmpl    $op1, $op2" %}
12366   ins_encode %{
12367     __ cmpl($op1$$Register, $op2$$Address);
12368   %}
12369   ins_pipe(ialu_cr_reg_mem);
12370 %}
12371 
12372 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12373 %{
12374   match(Set cr (CmpI src zero));
12375 
12376   format %{ "testl   $src, $src" %}
12377   ins_encode %{
12378     __ testl($src$$Register, $src$$Register);
12379   %}
12380   ins_pipe(ialu_cr_reg_imm);
12381 %}
12382 
12383 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12384 %{
12385   match(Set cr (CmpI (AndI src con) zero));
12386 
12387   format %{ "testl   $src, $con" %}
12388   ins_encode %{
12389     __ testl($src$$Register, $con$$constant);
12390   %}
12391   ins_pipe(ialu_cr_reg_imm);
12392 %}
12393 
12394 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12395 %{
12396   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12397 
12398   format %{ "testl   $src, $mem" %}
12399   ins_encode %{
12400     __ testl($src$$Register, $mem$$Address);
12401   %}
12402   ins_pipe(ialu_cr_reg_mem);
12403 %}
12404 
12405 // Unsigned compare Instructions; really, same as signed except they
12406 // produce an rFlagsRegU instead of rFlagsReg.
12407 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12408 %{
12409   match(Set cr (CmpU op1 op2));
12410 
12411   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12412   ins_encode %{
12413     __ cmpl($op1$$Register, $op2$$Register);
12414   %}
12415   ins_pipe(ialu_cr_reg_reg);
12416 %}
12417 
12418 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12419 %{
12420   match(Set cr (CmpU op1 op2));
12421 
12422   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12423   ins_encode %{
12424     __ cmpl($op1$$Register, $op2$$constant);
12425   %}
12426   ins_pipe(ialu_cr_reg_imm);
12427 %}
12428 
12429 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12430 %{
12431   match(Set cr (CmpU op1 (LoadI op2)));
12432 
12433   ins_cost(500); // XXX
12434   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12435   ins_encode %{
12436     __ cmpl($op1$$Register, $op2$$Address);
12437   %}
12438   ins_pipe(ialu_cr_reg_mem);
12439 %}
12440 
12441 // // // Cisc-spilled version of cmpU_rReg
12442 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12443 // //%{
12444 // //  match(Set cr (CmpU (LoadI op1) op2));
12445 // //
12446 // //  format %{ "CMPu   $op1,$op2" %}
12447 // //  ins_cost(500);
12448 // //  opcode(0x39);  /* Opcode 39 /r */
12449 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12450 // //%}
12451 
12452 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12453 %{
12454   match(Set cr (CmpU src zero));
12455 
12456   format %{ "testl   $src, $src\t# unsigned" %}
12457   ins_encode %{
12458     __ testl($src$$Register, $src$$Register);
12459   %}
12460   ins_pipe(ialu_cr_reg_imm);
12461 %}
12462 
12463 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12464 %{
12465   match(Set cr (CmpP op1 op2));
12466 
12467   format %{ "cmpq    $op1, $op2\t# ptr" %}
12468   ins_encode %{
12469     __ cmpq($op1$$Register, $op2$$Register);
12470   %}
12471   ins_pipe(ialu_cr_reg_reg);
12472 %}
12473 
12474 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12475 %{
12476   match(Set cr (CmpP op1 (LoadP op2)));
12477   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12478 
12479   ins_cost(500); // XXX
12480   format %{ "cmpq    $op1, $op2\t# ptr" %}
12481   ins_encode %{
12482     __ cmpq($op1$$Register, $op2$$Address);
12483   %}
12484   ins_pipe(ialu_cr_reg_mem);
12485 %}
12486 
12487 // // // Cisc-spilled version of cmpP_rReg
12488 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12489 // //%{
12490 // //  match(Set cr (CmpP (LoadP op1) op2));
12491 // //
12492 // //  format %{ "CMPu   $op1,$op2" %}
12493 // //  ins_cost(500);
12494 // //  opcode(0x39);  /* Opcode 39 /r */
12495 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12496 // //%}
12497 
12498 // XXX this is generalized by compP_rReg_mem???
12499 // Compare raw pointer (used in out-of-heap check).
12500 // Only works because non-oop pointers must be raw pointers
12501 // and raw pointers have no anti-dependencies.
12502 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12503 %{
12504   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12505             n->in(2)->as_Load()->barrier_data() == 0);
12506   match(Set cr (CmpP op1 (LoadP op2)));
12507 
12508   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12509   ins_encode %{
12510     __ cmpq($op1$$Register, $op2$$Address);
12511   %}
12512   ins_pipe(ialu_cr_reg_mem);
12513 %}
12514 
12515 // This will generate a signed flags result. This should be OK since
12516 // any compare to a zero should be eq/neq.
12517 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12518 %{
12519   match(Set cr (CmpP src zero));
12520 
12521   format %{ "testq   $src, $src\t# ptr" %}
12522   ins_encode %{
12523     __ testq($src$$Register, $src$$Register);
12524   %}
12525   ins_pipe(ialu_cr_reg_imm);
12526 %}
12527 
12528 // This will generate a signed flags result. This should be OK since
12529 // any compare to a zero should be eq/neq.
12530 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12531 %{
12532   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12533             n->in(1)->as_Load()->barrier_data() == 0);
12534   match(Set cr (CmpP (LoadP op) zero));
12535 
12536   ins_cost(500); // XXX
12537   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12538   ins_encode %{
12539     __ testq($op$$Address, 0xFFFFFFFF);
12540   %}
12541   ins_pipe(ialu_cr_reg_imm);
12542 %}
12543 
12544 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12545 %{
12546   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12547             n->in(1)->as_Load()->barrier_data() == 0);
12548   match(Set cr (CmpP (LoadP mem) zero));
12549 
12550   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12551   ins_encode %{
12552     __ cmpq(r12, $mem$$Address);
12553   %}
12554   ins_pipe(ialu_cr_reg_mem);
12555 %}
12556 
12557 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12558 %{
12559   match(Set cr (CmpN op1 op2));
12560 
12561   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12562   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12563   ins_pipe(ialu_cr_reg_reg);
12564 %}
12565 
12566 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12567 %{
12568   match(Set cr (CmpN src (LoadN mem)));
12569 
12570   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12571   ins_encode %{
12572     __ cmpl($src$$Register, $mem$$Address);
12573   %}
12574   ins_pipe(ialu_cr_reg_mem);
12575 %}
12576 
12577 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12578   match(Set cr (CmpN op1 op2));
12579 
12580   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12581   ins_encode %{
12582     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12583   %}
12584   ins_pipe(ialu_cr_reg_imm);
12585 %}
12586 
12587 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12588 %{
12589   match(Set cr (CmpN src (LoadN mem)));
12590 
12591   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12592   ins_encode %{
12593     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12594   %}
12595   ins_pipe(ialu_cr_reg_mem);
12596 %}
12597 
12598 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12599   match(Set cr (CmpN op1 op2));
12600 
12601   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12602   ins_encode %{
12603     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12604   %}
12605   ins_pipe(ialu_cr_reg_imm);
12606 %}
12607 
12608 // Disabled because the compressed Klass* in header cannot be safely
12609 // accessed. TODO: Re-enable it as soon as synchronization does not
12610 // overload the upper header bits anymore.
12611 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12612 %{
12613   predicate(false);
12614   match(Set cr (CmpN src (LoadNKlass mem)));
12615 
12616   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12617   ins_encode %{
12618     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12619   %}
12620   ins_pipe(ialu_cr_reg_mem);
12621 %}
12622 
12623 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12624   match(Set cr (CmpN src zero));
12625 
12626   format %{ "testl   $src, $src\t# compressed ptr" %}
12627   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12628   ins_pipe(ialu_cr_reg_imm);
12629 %}
12630 
12631 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12632 %{
12633   predicate(CompressedOops::base() != NULL);
12634   match(Set cr (CmpN (LoadN mem) zero));
12635 
12636   ins_cost(500); // XXX
12637   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12638   ins_encode %{
12639     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12640   %}
12641   ins_pipe(ialu_cr_reg_mem);
12642 %}
12643 
12644 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12645 %{
12646   predicate(CompressedOops::base() == NULL);
12647   match(Set cr (CmpN (LoadN mem) zero));
12648 
12649   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12650   ins_encode %{
12651     __ cmpl(r12, $mem$$Address);
12652   %}
12653   ins_pipe(ialu_cr_reg_mem);
12654 %}
12655 
12656 // Yanked all unsigned pointer compare operations.
12657 // Pointer compares are done with CmpP which is already unsigned.
12658 
12659 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12660 %{
12661   match(Set cr (CmpL op1 op2));
12662 
12663   format %{ "cmpq    $op1, $op2" %}
12664   ins_encode %{
12665     __ cmpq($op1$$Register, $op2$$Register);
12666   %}
12667   ins_pipe(ialu_cr_reg_reg);
12668 %}
12669 
12670 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12671 %{
12672   match(Set cr (CmpL op1 op2));
12673 
12674   format %{ "cmpq    $op1, $op2" %}
12675   ins_encode %{
12676     __ cmpq($op1$$Register, $op2$$constant);
12677   %}
12678   ins_pipe(ialu_cr_reg_imm);
12679 %}
12680 
12681 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12682 %{
12683   match(Set cr (CmpL op1 (LoadL op2)));
12684 
12685   format %{ "cmpq    $op1, $op2" %}
12686   ins_encode %{
12687     __ cmpq($op1$$Register, $op2$$Address);
12688   %}
12689   ins_pipe(ialu_cr_reg_mem);
12690 %}
12691 
12692 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12693 %{
12694   match(Set cr (CmpL src zero));
12695 
12696   format %{ "testq   $src, $src" %}
12697   ins_encode %{
12698     __ testq($src$$Register, $src$$Register);
12699   %}
12700   ins_pipe(ialu_cr_reg_imm);
12701 %}
12702 
12703 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12704 %{
12705   match(Set cr (CmpL (AndL src con) zero));
12706 
12707   format %{ "testq   $src, $con\t# long" %}
12708   ins_encode %{
12709     __ testq($src$$Register, $con$$constant);
12710   %}
12711   ins_pipe(ialu_cr_reg_imm);
12712 %}
12713 
12714 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12715 %{
12716   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12717 
12718   format %{ "testq   $src, $mem" %}
12719   ins_encode %{
12720     __ testq($src$$Register, $mem$$Address);
12721   %}
12722   ins_pipe(ialu_cr_reg_mem);
12723 %}
12724 
12725 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12726 %{
12727   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12728 
12729   format %{ "testq   $src, $mem" %}
12730   ins_encode %{
12731     __ testq($src$$Register, $mem$$Address);
12732   %}
12733   ins_pipe(ialu_cr_reg_mem);
12734 %}
12735 
12736 // Manifest a CmpU result in an integer register.  Very painful.
12737 // This is the test to avoid.
12738 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
12739 %{
12740   match(Set dst (CmpU3 src1 src2));
12741   effect(KILL flags);
12742 
12743   ins_cost(275); // XXX
12744   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
12745             "movl    $dst, -1\n\t"
12746             "jb,u    done\n\t"
12747             "setne   $dst\n\t"
12748             "movzbl  $dst, $dst\n\t"
12749     "done:" %}
12750   ins_encode %{
12751     Label done;
12752     __ cmpl($src1$$Register, $src2$$Register);
12753     __ movl($dst$$Register, -1);
12754     __ jccb(Assembler::below, done);
12755     __ setne($dst$$Register);
12756     __ movzbl($dst$$Register, $dst$$Register);
12757     __ bind(done);
12758   %}
12759   ins_pipe(pipe_slow);
12760 %}
12761 
12762 // Manifest a CmpL result in an integer register.  Very painful.
12763 // This is the test to avoid.
12764 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12765 %{
12766   match(Set dst (CmpL3 src1 src2));
12767   effect(KILL flags);
12768 
12769   ins_cost(275); // XXX
12770   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12771             "movl    $dst, -1\n\t"
12772             "jl,s    done\n\t"
12773             "setne   $dst\n\t"
12774             "movzbl  $dst, $dst\n\t"
12775     "done:" %}
12776   ins_encode %{
12777     Label done;
12778     __ cmpq($src1$$Register, $src2$$Register);
12779     __ movl($dst$$Register, -1);
12780     __ jccb(Assembler::less, done);
12781     __ setne($dst$$Register);
12782     __ movzbl($dst$$Register, $dst$$Register);
12783     __ bind(done);
12784   %}
12785   ins_pipe(pipe_slow);
12786 %}
12787 
12788 // Manifest a CmpUL result in an integer register.  Very painful.
12789 // This is the test to avoid.
12790 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12791 %{
12792   match(Set dst (CmpUL3 src1 src2));
12793   effect(KILL flags);
12794 
12795   ins_cost(275); // XXX
12796   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12797             "movl    $dst, -1\n\t"
12798             "jb,u    done\n\t"
12799             "setne   $dst\n\t"
12800             "movzbl  $dst, $dst\n\t"
12801     "done:" %}
12802   ins_encode %{
12803     Label done;
12804     __ cmpq($src1$$Register, $src2$$Register);
12805     __ movl($dst$$Register, -1);
12806     __ jccb(Assembler::below, done);
12807     __ setne($dst$$Register);
12808     __ movzbl($dst$$Register, $dst$$Register);
12809     __ bind(done);
12810   %}
12811   ins_pipe(pipe_slow);
12812 %}
12813 
12814 // Unsigned long compare Instructions; really, same as signed long except they
12815 // produce an rFlagsRegU instead of rFlagsReg.
12816 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12817 %{
12818   match(Set cr (CmpUL op1 op2));
12819 
12820   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12821   ins_encode %{
12822     __ cmpq($op1$$Register, $op2$$Register);
12823   %}
12824   ins_pipe(ialu_cr_reg_reg);
12825 %}
12826 
12827 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12828 %{
12829   match(Set cr (CmpUL op1 op2));
12830 
12831   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12832   ins_encode %{
12833     __ cmpq($op1$$Register, $op2$$constant);
12834   %}
12835   ins_pipe(ialu_cr_reg_imm);
12836 %}
12837 
12838 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12839 %{
12840   match(Set cr (CmpUL op1 (LoadL op2)));
12841 
12842   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12843   ins_encode %{
12844     __ cmpq($op1$$Register, $op2$$Address);
12845   %}
12846   ins_pipe(ialu_cr_reg_mem);
12847 %}
12848 
12849 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12850 %{
12851   match(Set cr (CmpUL src zero));
12852 
12853   format %{ "testq   $src, $src\t# unsigned" %}
12854   ins_encode %{
12855     __ testq($src$$Register, $src$$Register);
12856   %}
12857   ins_pipe(ialu_cr_reg_imm);
12858 %}
12859 
12860 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12861 %{
12862   match(Set cr (CmpI (LoadB mem) imm));
12863 
12864   ins_cost(125);
12865   format %{ "cmpb    $mem, $imm" %}
12866   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12867   ins_pipe(ialu_cr_reg_mem);
12868 %}
12869 
12870 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12871 %{
12872   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12873 
12874   ins_cost(125);
12875   format %{ "testb   $mem, $imm\t# ubyte" %}
12876   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12877   ins_pipe(ialu_cr_reg_mem);
12878 %}
12879 
12880 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12881 %{
12882   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12883 
12884   ins_cost(125);
12885   format %{ "testb   $mem, $imm\t# byte" %}
12886   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12887   ins_pipe(ialu_cr_reg_mem);
12888 %}
12889 
12890 //----------Max and Min--------------------------------------------------------
12891 // Min Instructions
12892 
12893 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12894 %{
12895   effect(USE_DEF dst, USE src, USE cr);
12896 
12897   format %{ "cmovlgt $dst, $src\t# min" %}
12898   ins_encode %{
12899     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12900   %}
12901   ins_pipe(pipe_cmov_reg);
12902 %}
12903 
12904 
12905 instruct minI_rReg(rRegI dst, rRegI src)
12906 %{
12907   match(Set dst (MinI dst src));
12908 
12909   ins_cost(200);
12910   expand %{
12911     rFlagsReg cr;
12912     compI_rReg(cr, dst, src);
12913     cmovI_reg_g(dst, src, cr);
12914   %}
12915 %}
12916 
12917 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12918 %{
12919   effect(USE_DEF dst, USE src, USE cr);
12920 
12921   format %{ "cmovllt $dst, $src\t# max" %}
12922   ins_encode %{
12923     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12924   %}
12925   ins_pipe(pipe_cmov_reg);
12926 %}
12927 
12928 
12929 instruct maxI_rReg(rRegI dst, rRegI src)
12930 %{
12931   match(Set dst (MaxI dst src));
12932 
12933   ins_cost(200);
12934   expand %{
12935     rFlagsReg cr;
12936     compI_rReg(cr, dst, src);
12937     cmovI_reg_l(dst, src, cr);
12938   %}
12939 %}
12940 
12941 // ============================================================================
12942 // Branch Instructions
12943 
12944 // Jump Direct - Label defines a relative address from JMP+1
12945 instruct jmpDir(label labl)
12946 %{
12947   match(Goto);
12948   effect(USE labl);
12949 
12950   ins_cost(300);
12951   format %{ "jmp     $labl" %}
12952   size(5);
12953   ins_encode %{
12954     Label* L = $labl$$label;
12955     __ jmp(*L, false); // Always long jump
12956   %}
12957   ins_pipe(pipe_jmp);
12958 %}
12959 
12960 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12961 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12962 %{
12963   match(If cop cr);
12964   effect(USE labl);
12965 
12966   ins_cost(300);
12967   format %{ "j$cop     $labl" %}
12968   size(6);
12969   ins_encode %{
12970     Label* L = $labl$$label;
12971     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12972   %}
12973   ins_pipe(pipe_jcc);
12974 %}
12975 
12976 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12977 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12978 %{
12979   match(CountedLoopEnd cop cr);
12980   effect(USE labl);
12981 
12982   ins_cost(300);
12983   format %{ "j$cop     $labl\t# loop end" %}
12984   size(6);
12985   ins_encode %{
12986     Label* L = $labl$$label;
12987     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12988   %}
12989   ins_pipe(pipe_jcc);
12990 %}
12991 
12992 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12993 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12994   match(CountedLoopEnd cop cmp);
12995   effect(USE labl);
12996 
12997   ins_cost(300);
12998   format %{ "j$cop,u   $labl\t# loop end" %}
12999   size(6);
13000   ins_encode %{
13001     Label* L = $labl$$label;
13002     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13003   %}
13004   ins_pipe(pipe_jcc);
13005 %}
13006 
13007 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13008   match(CountedLoopEnd cop cmp);
13009   effect(USE labl);
13010 
13011   ins_cost(200);
13012   format %{ "j$cop,u   $labl\t# loop end" %}
13013   size(6);
13014   ins_encode %{
13015     Label* L = $labl$$label;
13016     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13017   %}
13018   ins_pipe(pipe_jcc);
13019 %}
13020 
13021 // Jump Direct Conditional - using unsigned comparison
13022 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13023   match(If cop cmp);
13024   effect(USE labl);
13025 
13026   ins_cost(300);
13027   format %{ "j$cop,u   $labl" %}
13028   size(6);
13029   ins_encode %{
13030     Label* L = $labl$$label;
13031     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13032   %}
13033   ins_pipe(pipe_jcc);
13034 %}
13035 
13036 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13037   match(If cop cmp);
13038   effect(USE labl);
13039 
13040   ins_cost(200);
13041   format %{ "j$cop,u   $labl" %}
13042   size(6);
13043   ins_encode %{
13044     Label* L = $labl$$label;
13045     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13046   %}
13047   ins_pipe(pipe_jcc);
13048 %}
13049 
13050 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13051   match(If cop cmp);
13052   effect(USE labl);
13053 
13054   ins_cost(200);
13055   format %{ $$template
13056     if ($cop$$cmpcode == Assembler::notEqual) {
13057       $$emit$$"jp,u    $labl\n\t"
13058       $$emit$$"j$cop,u   $labl"
13059     } else {
13060       $$emit$$"jp,u    done\n\t"
13061       $$emit$$"j$cop,u   $labl\n\t"
13062       $$emit$$"done:"
13063     }
13064   %}
13065   ins_encode %{
13066     Label* l = $labl$$label;
13067     if ($cop$$cmpcode == Assembler::notEqual) {
13068       __ jcc(Assembler::parity, *l, false);
13069       __ jcc(Assembler::notEqual, *l, false);
13070     } else if ($cop$$cmpcode == Assembler::equal) {
13071       Label done;
13072       __ jccb(Assembler::parity, done);
13073       __ jcc(Assembler::equal, *l, false);
13074       __ bind(done);
13075     } else {
13076        ShouldNotReachHere();
13077     }
13078   %}
13079   ins_pipe(pipe_jcc);
13080 %}
13081 
13082 // ============================================================================
13083 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13084 // superklass array for an instance of the superklass.  Set a hidden
13085 // internal cache on a hit (cache is checked with exposed code in
13086 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13087 // encoding ALSO sets flags.
13088 
13089 instruct partialSubtypeCheck(rdi_RegP result,
13090                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13091                              rFlagsReg cr)
13092 %{
13093   match(Set result (PartialSubtypeCheck sub super));
13094   effect(KILL rcx, KILL cr);
13095 
13096   ins_cost(1100);  // slightly larger than the next version
13097   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13098             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13099             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13100             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13101             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13102             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13103             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13104     "miss:\t" %}
13105 
13106   opcode(0x1); // Force a XOR of RDI
13107   ins_encode(enc_PartialSubtypeCheck());
13108   ins_pipe(pipe_slow);
13109 %}
13110 
13111 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13112                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13113                                      immP0 zero,
13114                                      rdi_RegP result)
13115 %{
13116   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13117   effect(KILL rcx, KILL result);
13118 
13119   ins_cost(1000);
13120   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13121             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13122             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13123             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13124             "jne,s   miss\t\t# Missed: flags nz\n\t"
13125             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13126     "miss:\t" %}
13127 
13128   opcode(0x0); // No need to XOR RDI
13129   ins_encode(enc_PartialSubtypeCheck());
13130   ins_pipe(pipe_slow);
13131 %}
13132 
13133 // ============================================================================
13134 // Branch Instructions -- short offset versions
13135 //
13136 // These instructions are used to replace jumps of a long offset (the default
13137 // match) with jumps of a shorter offset.  These instructions are all tagged
13138 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13139 // match rules in general matching.  Instead, the ADLC generates a conversion
13140 // method in the MachNode which can be used to do in-place replacement of the
13141 // long variant with the shorter variant.  The compiler will determine if a
13142 // branch can be taken by the is_short_branch_offset() predicate in the machine
13143 // specific code section of the file.
13144 
13145 // Jump Direct - Label defines a relative address from JMP+1
13146 instruct jmpDir_short(label labl) %{
13147   match(Goto);
13148   effect(USE labl);
13149 
13150   ins_cost(300);
13151   format %{ "jmp,s   $labl" %}
13152   size(2);
13153   ins_encode %{
13154     Label* L = $labl$$label;
13155     __ jmpb(*L);
13156   %}
13157   ins_pipe(pipe_jmp);
13158   ins_short_branch(1);
13159 %}
13160 
13161 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13162 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13163   match(If cop cr);
13164   effect(USE labl);
13165 
13166   ins_cost(300);
13167   format %{ "j$cop,s   $labl" %}
13168   size(2);
13169   ins_encode %{
13170     Label* L = $labl$$label;
13171     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13172   %}
13173   ins_pipe(pipe_jcc);
13174   ins_short_branch(1);
13175 %}
13176 
13177 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13178 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13179   match(CountedLoopEnd cop cr);
13180   effect(USE labl);
13181 
13182   ins_cost(300);
13183   format %{ "j$cop,s   $labl\t# loop end" %}
13184   size(2);
13185   ins_encode %{
13186     Label* L = $labl$$label;
13187     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13188   %}
13189   ins_pipe(pipe_jcc);
13190   ins_short_branch(1);
13191 %}
13192 
13193 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13194 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13195   match(CountedLoopEnd cop cmp);
13196   effect(USE labl);
13197 
13198   ins_cost(300);
13199   format %{ "j$cop,us  $labl\t# loop end" %}
13200   size(2);
13201   ins_encode %{
13202     Label* L = $labl$$label;
13203     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13204   %}
13205   ins_pipe(pipe_jcc);
13206   ins_short_branch(1);
13207 %}
13208 
13209 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13210   match(CountedLoopEnd cop cmp);
13211   effect(USE labl);
13212 
13213   ins_cost(300);
13214   format %{ "j$cop,us  $labl\t# loop end" %}
13215   size(2);
13216   ins_encode %{
13217     Label* L = $labl$$label;
13218     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13219   %}
13220   ins_pipe(pipe_jcc);
13221   ins_short_branch(1);
13222 %}
13223 
13224 // Jump Direct Conditional - using unsigned comparison
13225 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13226   match(If cop cmp);
13227   effect(USE labl);
13228 
13229   ins_cost(300);
13230   format %{ "j$cop,us  $labl" %}
13231   size(2);
13232   ins_encode %{
13233     Label* L = $labl$$label;
13234     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13235   %}
13236   ins_pipe(pipe_jcc);
13237   ins_short_branch(1);
13238 %}
13239 
13240 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13241   match(If cop cmp);
13242   effect(USE labl);
13243 
13244   ins_cost(300);
13245   format %{ "j$cop,us  $labl" %}
13246   size(2);
13247   ins_encode %{
13248     Label* L = $labl$$label;
13249     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13250   %}
13251   ins_pipe(pipe_jcc);
13252   ins_short_branch(1);
13253 %}
13254 
13255 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13256   match(If cop cmp);
13257   effect(USE labl);
13258 
13259   ins_cost(300);
13260   format %{ $$template
13261     if ($cop$$cmpcode == Assembler::notEqual) {
13262       $$emit$$"jp,u,s  $labl\n\t"
13263       $$emit$$"j$cop,u,s  $labl"
13264     } else {
13265       $$emit$$"jp,u,s  done\n\t"
13266       $$emit$$"j$cop,u,s  $labl\n\t"
13267       $$emit$$"done:"
13268     }
13269   %}
13270   size(4);
13271   ins_encode %{
13272     Label* l = $labl$$label;
13273     if ($cop$$cmpcode == Assembler::notEqual) {
13274       __ jccb(Assembler::parity, *l);
13275       __ jccb(Assembler::notEqual, *l);
13276     } else if ($cop$$cmpcode == Assembler::equal) {
13277       Label done;
13278       __ jccb(Assembler::parity, done);
13279       __ jccb(Assembler::equal, *l);
13280       __ bind(done);
13281     } else {
13282        ShouldNotReachHere();
13283     }
13284   %}
13285   ins_pipe(pipe_jcc);
13286   ins_short_branch(1);
13287 %}
13288 
13289 // ============================================================================
13290 // inlined locking and unlocking
13291 
13292 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13293   predicate(Compile::current()->use_rtm());
13294   match(Set cr (FastLock object));
13295   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, TEMP box);
13296   ins_cost(300);
13297   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13298   ins_encode %{
13299     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13300                  $scr$$Register, $cx1$$Register, $cx2$$Register, r15_thread,
13301                  _rtm_counters, _stack_rtm_counters,
13302                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13303                  true, ra_->C->profile_rtm());
13304   %}
13305   ins_pipe(pipe_slow);
13306 %}
13307 
13308 instruct cmpFastLock(rFlagsReg cr, rRegP object, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13309   predicate(!Compile::current()->use_rtm());
13310   match(Set cr (FastLock object));
13311   effect(TEMP tmp, TEMP scr, TEMP cx1);
13312   ins_cost(300);
13313   format %{ "fastlock $object\t! kills $tmp,$scr" %}
13314   ins_encode %{
13315     __ fast_lock($object$$Register, noreg, $tmp$$Register,
13316                  $scr$$Register, $cx1$$Register, noreg, r15_thread, NULL, NULL, NULL, false, false);
13317   %}
13318   ins_pipe(pipe_slow);
13319 %}
13320 
13321 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13322   match(Set cr (FastUnlock object));
13323   effect(TEMP tmp, TEMP box);
13324   ins_cost(300);
13325   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13326   ins_encode %{
13327     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13328   %}
13329   ins_pipe(pipe_slow);
13330 %}
13331 
13332 
13333 // ============================================================================
13334 // Safepoint Instructions
13335 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13336 %{
13337   match(SafePoint poll);
13338   effect(KILL cr, USE poll);
13339 
13340   format %{ "testl   rax, [$poll]\t"
13341             "# Safepoint: poll for GC" %}
13342   ins_cost(125);
13343   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13344   ins_encode %{
13345     __ relocate(relocInfo::poll_type);
13346     address pre_pc = __ pc();
13347     __ testl(rax, Address($poll$$Register, 0));
13348     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13349   %}
13350   ins_pipe(ialu_reg_mem);
13351 %}
13352 
13353 instruct mask_all_evexL(kReg dst, rRegL src) %{
13354   match(Set dst (MaskAll src));
13355   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13356   ins_encode %{
13357     int mask_len = Matcher::vector_length(this);
13358     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13359   %}
13360   ins_pipe( pipe_slow );
13361 %}
13362 
13363 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13364   predicate(Matcher::vector_length(n) > 32);
13365   match(Set dst (MaskAll src));
13366   effect(TEMP tmp);
13367   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13368   ins_encode %{
13369     int mask_len = Matcher::vector_length(this);
13370     __ movslq($tmp$$Register, $src$$Register);
13371     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13372   %}
13373   ins_pipe( pipe_slow );
13374 %}
13375 
13376 // ============================================================================
13377 // Procedure Call/Return Instructions
13378 // Call Java Static Instruction
13379 // Note: If this code changes, the corresponding ret_addr_offset() and
13380 //       compute_padding() functions will have to be adjusted.
13381 instruct CallStaticJavaDirect(method meth) %{
13382   match(CallStaticJava);
13383   effect(USE meth);
13384 
13385   ins_cost(300);
13386   format %{ "call,static " %}
13387   opcode(0xE8); /* E8 cd */
13388   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13389   ins_pipe(pipe_slow);
13390   ins_alignment(4);
13391 %}
13392 
13393 // Call Java Dynamic Instruction
13394 // Note: If this code changes, the corresponding ret_addr_offset() and
13395 //       compute_padding() functions will have to be adjusted.
13396 instruct CallDynamicJavaDirect(method meth)
13397 %{
13398   match(CallDynamicJava);
13399   effect(USE meth);
13400 
13401   ins_cost(300);
13402   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13403             "call,dynamic " %}
13404   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13405   ins_pipe(pipe_slow);
13406   ins_alignment(4);
13407 %}
13408 
13409 // Call Runtime Instruction
13410 instruct CallRuntimeDirect(method meth)
13411 %{
13412   match(CallRuntime);
13413   effect(USE meth);
13414 
13415   ins_cost(300);
13416   format %{ "call,runtime " %}
13417   ins_encode(clear_avx, Java_To_Runtime(meth));
13418   ins_pipe(pipe_slow);
13419 %}
13420 
13421 // Call runtime without safepoint
13422 instruct CallLeafDirect(method meth)
13423 %{
13424   match(CallLeaf);
13425   effect(USE meth);
13426 
13427   ins_cost(300);
13428   format %{ "call_leaf,runtime " %}
13429   ins_encode(clear_avx, Java_To_Runtime(meth));
13430   ins_pipe(pipe_slow);
13431 %}
13432 
13433 // Call runtime without safepoint and with vector arguments
13434 instruct CallLeafDirectVector(method meth)
13435 %{
13436   match(CallLeafVector);
13437   effect(USE meth);
13438 
13439   ins_cost(300);
13440   format %{ "call_leaf,vector " %}
13441   ins_encode(Java_To_Runtime(meth));
13442   ins_pipe(pipe_slow);
13443 %}
13444 
13445 // Call runtime without safepoint
13446 instruct CallLeafNoFPDirect(method meth)
13447 %{
13448   match(CallLeafNoFP);
13449   effect(USE meth);
13450 
13451   ins_cost(300);
13452   format %{ "call_leaf_nofp,runtime " %}
13453   ins_encode(clear_avx, Java_To_Runtime(meth));
13454   ins_pipe(pipe_slow);
13455 %}
13456 
13457 // Return Instruction
13458 // Remove the return address & jump to it.
13459 // Notice: We always emit a nop after a ret to make sure there is room
13460 // for safepoint patching
13461 instruct Ret()
13462 %{
13463   match(Return);
13464 
13465   format %{ "ret" %}
13466   ins_encode %{
13467     __ ret(0);
13468   %}
13469   ins_pipe(pipe_jmp);
13470 %}
13471 
13472 // Tail Call; Jump from runtime stub to Java code.
13473 // Also known as an 'interprocedural jump'.
13474 // Target of jump will eventually return to caller.
13475 // TailJump below removes the return address.
13476 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13477 %{
13478   match(TailCall jump_target method_ptr);
13479 
13480   ins_cost(300);
13481   format %{ "jmp     $jump_target\t# rbx holds method" %}
13482   ins_encode %{
13483     __ jmp($jump_target$$Register);
13484   %}
13485   ins_pipe(pipe_jmp);
13486 %}
13487 
13488 // Tail Jump; remove the return address; jump to target.
13489 // TailCall above leaves the return address around.
13490 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13491 %{
13492   match(TailJump jump_target ex_oop);
13493 
13494   ins_cost(300);
13495   format %{ "popq    rdx\t# pop return address\n\t"
13496             "jmp     $jump_target" %}
13497   ins_encode %{
13498     __ popq(as_Register(RDX_enc));
13499     __ jmp($jump_target$$Register);
13500   %}
13501   ins_pipe(pipe_jmp);
13502 %}
13503 
13504 // Create exception oop: created by stack-crawling runtime code.
13505 // Created exception is now available to this handler, and is setup
13506 // just prior to jumping to this handler.  No code emitted.
13507 instruct CreateException(rax_RegP ex_oop)
13508 %{
13509   match(Set ex_oop (CreateEx));
13510 
13511   size(0);
13512   // use the following format syntax
13513   format %{ "# exception oop is in rax; no code emitted" %}
13514   ins_encode();
13515   ins_pipe(empty);
13516 %}
13517 
13518 // Rethrow exception:
13519 // The exception oop will come in the first argument position.
13520 // Then JUMP (not call) to the rethrow stub code.
13521 instruct RethrowException()
13522 %{
13523   match(Rethrow);
13524 
13525   // use the following format syntax
13526   format %{ "jmp     rethrow_stub" %}
13527   ins_encode(enc_rethrow);
13528   ins_pipe(pipe_jmp);
13529 %}
13530 
13531 // ============================================================================
13532 // This name is KNOWN by the ADLC and cannot be changed.
13533 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13534 // for this guy.
13535 instruct tlsLoadP(r15_RegP dst) %{
13536   match(Set dst (ThreadLocal));
13537   effect(DEF dst);
13538 
13539   size(0);
13540   format %{ "# TLS is in R15" %}
13541   ins_encode( /*empty encoding*/ );
13542   ins_pipe(ialu_reg_reg);
13543 %}
13544 
13545 
13546 //----------PEEPHOLE RULES-----------------------------------------------------
13547 // These must follow all instruction definitions as they use the names
13548 // defined in the instructions definitions.
13549 //
13550 // peepmatch ( root_instr_name [preceding_instruction]* );
13551 //
13552 // peepconstraint %{
13553 // (instruction_number.operand_name relational_op instruction_number.operand_name
13554 //  [, ...] );
13555 // // instruction numbers are zero-based using left to right order in peepmatch
13556 //
13557 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13558 // // provide an instruction_number.operand_name for each operand that appears
13559 // // in the replacement instruction's match rule
13560 //
13561 // ---------VM FLAGS---------------------------------------------------------
13562 //
13563 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13564 //
13565 // Each peephole rule is given an identifying number starting with zero and
13566 // increasing by one in the order seen by the parser.  An individual peephole
13567 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13568 // on the command-line.
13569 //
13570 // ---------CURRENT LIMITATIONS----------------------------------------------
13571 //
13572 // Only match adjacent instructions in same basic block
13573 // Only equality constraints
13574 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13575 // Only one replacement instruction
13576 //
13577 // ---------EXAMPLE----------------------------------------------------------
13578 //
13579 // // pertinent parts of existing instructions in architecture description
13580 // instruct movI(rRegI dst, rRegI src)
13581 // %{
13582 //   match(Set dst (CopyI src));
13583 // %}
13584 //
13585 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13586 // %{
13587 //   match(Set dst (AddI dst src));
13588 //   effect(KILL cr);
13589 // %}
13590 //
13591 // // Change (inc mov) to lea
13592 // peephole %{
13593 //   // increment preceded by register-register move
13594 //   peepmatch ( incI_rReg movI );
13595 //   // require that the destination register of the increment
13596 //   // match the destination register of the move
13597 //   peepconstraint ( 0.dst == 1.dst );
13598 //   // construct a replacement instruction that sets
13599 //   // the destination to ( move's source register + one )
13600 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13601 // %}
13602 //
13603 
13604 // Implementation no longer uses movX instructions since
13605 // machine-independent system no longer uses CopyX nodes.
13606 //
13607 // peephole
13608 // %{
13609 //   peepmatch (incI_rReg movI);
13610 //   peepconstraint (0.dst == 1.dst);
13611 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13612 // %}
13613 
13614 // peephole
13615 // %{
13616 //   peepmatch (decI_rReg movI);
13617 //   peepconstraint (0.dst == 1.dst);
13618 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13619 // %}
13620 
13621 // peephole
13622 // %{
13623 //   peepmatch (addI_rReg_imm movI);
13624 //   peepconstraint (0.dst == 1.dst);
13625 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13626 // %}
13627 
13628 // peephole
13629 // %{
13630 //   peepmatch (incL_rReg movL);
13631 //   peepconstraint (0.dst == 1.dst);
13632 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13633 // %}
13634 
13635 // peephole
13636 // %{
13637 //   peepmatch (decL_rReg movL);
13638 //   peepconstraint (0.dst == 1.dst);
13639 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13640 // %}
13641 
13642 // peephole
13643 // %{
13644 //   peepmatch (addL_rReg_imm movL);
13645 //   peepconstraint (0.dst == 1.dst);
13646 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13647 // %}
13648 
13649 // peephole
13650 // %{
13651 //   peepmatch (addP_rReg_imm movP);
13652 //   peepconstraint (0.dst == 1.dst);
13653 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13654 // %}
13655 
13656 // // Change load of spilled value to only a spill
13657 // instruct storeI(memory mem, rRegI src)
13658 // %{
13659 //   match(Set mem (StoreI mem src));
13660 // %}
13661 //
13662 // instruct loadI(rRegI dst, memory mem)
13663 // %{
13664 //   match(Set dst (LoadI mem));
13665 // %}
13666 //
13667 
13668 peephole
13669 %{
13670   peepmatch (loadI storeI);
13671   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13672   peepreplace (storeI(1.mem 1.mem 1.src));
13673 %}
13674 
13675 peephole
13676 %{
13677   peepmatch (loadL storeL);
13678   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13679   peepreplace (storeL(1.mem 1.mem 1.src));
13680 %}
13681 
13682 //----------SMARTSPILL RULES---------------------------------------------------
13683 // These must follow all instruction definitions as they use the names
13684 // defined in the instructions definitions.