1 //
    2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // archtecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 
  132 // Floating Point Registers
  133 
  134 // Specify priority of register selection within phases of register
  135 // allocation.  Highest priority is first.  A useful heuristic is to
  136 // give registers a low priority when they are required by machine
  137 // instructions, like EAX and EDX on I486, and choose no-save registers
  138 // before save-on-call, & save-on-call before save-on-entry.  Registers
  139 // which participate in fixed calling sequences should come last.
  140 // Registers which are used as pairs must fall on an even boundary.
  141 
  142 alloc_class chunk0(R10,         R10_H,
  143                    R11,         R11_H,
  144                    R8,          R8_H,
  145                    R9,          R9_H,
  146                    R12,         R12_H,
  147                    RCX,         RCX_H,
  148                    RBX,         RBX_H,
  149                    RDI,         RDI_H,
  150                    RDX,         RDX_H,
  151                    RSI,         RSI_H,
  152                    RAX,         RAX_H,
  153                    RBP,         RBP_H,
  154                    R13,         R13_H,
  155                    R14,         R14_H,
  156                    R15,         R15_H,
  157                    RSP,         RSP_H);
  158 
  159 
  160 //----------Architecture Description Register Classes--------------------------
  161 // Several register classes are automatically defined based upon information in
  162 // this architecture description.
  163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  164 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  165 //
  166 
  167 // Empty register class.
  168 reg_class no_reg();
  169 
  170 // Class for all pointer/long registers
  171 reg_class all_reg(RAX, RAX_H,
  172                   RDX, RDX_H,
  173                   RBP, RBP_H,
  174                   RDI, RDI_H,
  175                   RSI, RSI_H,
  176                   RCX, RCX_H,
  177                   RBX, RBX_H,
  178                   RSP, RSP_H,
  179                   R8,  R8_H,
  180                   R9,  R9_H,
  181                   R10, R10_H,
  182                   R11, R11_H,
  183                   R12, R12_H,
  184                   R13, R13_H,
  185                   R14, R14_H,
  186                   R15, R15_H);
  187 
  188 // Class for all int registers
  189 reg_class all_int_reg(RAX
  190                       RDX,
  191                       RBP,
  192                       RDI,
  193                       RSI,
  194                       RCX,
  195                       RBX,
  196                       R8,
  197                       R9,
  198                       R10,
  199                       R11,
  200                       R12,
  201                       R13,
  202                       R14);
  203 
  204 // Class for all pointer registers
  205 reg_class any_reg %{
  206   return _ANY_REG_mask;
  207 %}
  208 
  209 // Class for all pointer registers (excluding RSP)
  210 reg_class ptr_reg %{
  211   return _PTR_REG_mask;
  212 %}
  213 
  214 // Class for all pointer registers (excluding RSP and RBP)
  215 reg_class ptr_reg_no_rbp %{
  216   return _PTR_REG_NO_RBP_mask;
  217 %}
  218 
  219 // Class for all pointer registers (excluding RAX and RSP)
  220 reg_class ptr_no_rax_reg %{
  221   return _PTR_NO_RAX_REG_mask;
  222 %}
  223 
  224 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  225 reg_class ptr_no_rax_rbx_reg %{
  226   return _PTR_NO_RAX_RBX_REG_mask;
  227 %}
  228 
  229 // Class for all long registers (excluding RSP)
  230 reg_class long_reg %{
  231   return _LONG_REG_mask;
  232 %}
  233 
  234 // Class for all long registers (excluding RAX, RDX and RSP)
  235 reg_class long_no_rax_rdx_reg %{
  236   return _LONG_NO_RAX_RDX_REG_mask;
  237 %}
  238 
  239 // Class for all long registers (excluding RCX and RSP)
  240 reg_class long_no_rcx_reg %{
  241   return _LONG_NO_RCX_REG_mask;
  242 %}
  243 
  244 // Class for all long registers (excluding RBP and R13)
  245 reg_class long_no_rbp_r13_reg %{
  246   return _LONG_NO_RBP_R13_REG_mask;
  247 %}
  248 
  249 // Class for all int registers (excluding RSP)
  250 reg_class int_reg %{
  251   return _INT_REG_mask;
  252 %}
  253 
  254 // Class for all int registers (excluding RAX, RDX, and RSP)
  255 reg_class int_no_rax_rdx_reg %{
  256   return _INT_NO_RAX_RDX_REG_mask;
  257 %}
  258 
  259 // Class for all int registers (excluding RCX and RSP)
  260 reg_class int_no_rcx_reg %{
  261   return _INT_NO_RCX_REG_mask;
  262 %}
  263 
  264 // Class for all int registers (excluding RBP and R13)
  265 reg_class int_no_rbp_r13_reg %{
  266   return _INT_NO_RBP_R13_REG_mask;
  267 %}
  268 
  269 // Singleton class for RAX pointer register
  270 reg_class ptr_rax_reg(RAX, RAX_H);
  271 
  272 // Singleton class for RBX pointer register
  273 reg_class ptr_rbx_reg(RBX, RBX_H);
  274 
  275 // Singleton class for RSI pointer register
  276 reg_class ptr_rsi_reg(RSI, RSI_H);
  277 
  278 // Singleton class for RBP pointer register
  279 reg_class ptr_rbp_reg(RBP, RBP_H);
  280 
  281 // Singleton class for RDI pointer register
  282 reg_class ptr_rdi_reg(RDI, RDI_H);
  283 
  284 // Singleton class for stack pointer
  285 reg_class ptr_rsp_reg(RSP, RSP_H);
  286 
  287 // Singleton class for TLS pointer
  288 reg_class ptr_r15_reg(R15, R15_H);
  289 
  290 // Singleton class for RAX long register
  291 reg_class long_rax_reg(RAX, RAX_H);
  292 
  293 // Singleton class for RCX long register
  294 reg_class long_rcx_reg(RCX, RCX_H);
  295 
  296 // Singleton class for RDX long register
  297 reg_class long_rdx_reg(RDX, RDX_H);
  298 
  299 // Singleton class for RAX int register
  300 reg_class int_rax_reg(RAX);
  301 
  302 // Singleton class for RBX int register
  303 reg_class int_rbx_reg(RBX);
  304 
  305 // Singleton class for RCX int register
  306 reg_class int_rcx_reg(RCX);
  307 
  308 // Singleton class for RCX int register
  309 reg_class int_rdx_reg(RDX);
  310 
  311 // Singleton class for RCX int register
  312 reg_class int_rdi_reg(RDI);
  313 
  314 // Singleton class for instruction pointer
  315 // reg_class ip_reg(RIP);
  316 
  317 %}
  318 
  319 //----------SOURCE BLOCK-------------------------------------------------------
  320 // This is a block of C++ code which provides values, functions, and
  321 // definitions necessary in the rest of the architecture description
  322 source_hpp %{
  323 
  324 extern RegMask _ANY_REG_mask;
  325 extern RegMask _PTR_REG_mask;
  326 extern RegMask _PTR_REG_NO_RBP_mask;
  327 extern RegMask _PTR_NO_RAX_REG_mask;
  328 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  329 extern RegMask _LONG_REG_mask;
  330 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  331 extern RegMask _LONG_NO_RCX_REG_mask;
  332 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  333 extern RegMask _INT_REG_mask;
  334 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  335 extern RegMask _INT_NO_RCX_REG_mask;
  336 extern RegMask _INT_NO_RBP_R13_REG_mask;
  337 extern RegMask _FLOAT_REG_mask;
  338 
  339 extern RegMask _STACK_OR_PTR_REG_mask;
  340 extern RegMask _STACK_OR_LONG_REG_mask;
  341 extern RegMask _STACK_OR_INT_REG_mask;
  342 
  343 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  344 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  345 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  346 
  347 %}
  348 
  349 source %{
  350 #define   RELOC_IMM64    Assembler::imm_operand
  351 #define   RELOC_DISP32   Assembler::disp32_operand
  352 
  353 #define __ _masm.
  354 
  355 RegMask _ANY_REG_mask;
  356 RegMask _PTR_REG_mask;
  357 RegMask _PTR_REG_NO_RBP_mask;
  358 RegMask _PTR_NO_RAX_REG_mask;
  359 RegMask _PTR_NO_RAX_RBX_REG_mask;
  360 RegMask _LONG_REG_mask;
  361 RegMask _LONG_NO_RAX_RDX_REG_mask;
  362 RegMask _LONG_NO_RCX_REG_mask;
  363 RegMask _LONG_NO_RBP_R13_REG_mask;
  364 RegMask _INT_REG_mask;
  365 RegMask _INT_NO_RAX_RDX_REG_mask;
  366 RegMask _INT_NO_RCX_REG_mask;
  367 RegMask _INT_NO_RBP_R13_REG_mask;
  368 RegMask _FLOAT_REG_mask;
  369 RegMask _STACK_OR_PTR_REG_mask;
  370 RegMask _STACK_OR_LONG_REG_mask;
  371 RegMask _STACK_OR_INT_REG_mask;
  372 
  373 static bool need_r12_heapbase() {
  374   return UseCompressedOops;
  375 }
  376 
  377 void reg_mask_init() {
  378   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  379   // We derive a number of subsets from it.
  380   _ANY_REG_mask = _ALL_REG_mask;
  381 
  382   if (PreserveFramePointer) {
  383     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  384     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  385   }
  386   if (need_r12_heapbase()) {
  387     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  388     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  389   }
  390 
  391   _PTR_REG_mask = _ANY_REG_mask;
  392   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  393   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  394   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  395   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  396 
  397   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  398   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  399 
  400   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  401   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  402   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  403 
  404   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  405   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  406   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  407 
  408   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  409   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  410   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  411 
  412   _LONG_REG_mask = _PTR_REG_mask;
  413   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  414   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  415 
  416   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  417   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  418   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  419   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  420   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  421 
  422   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  423   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  424   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  425 
  426   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  427   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  428   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  429   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  430   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  431 
  432   _INT_REG_mask = _ALL_INT_REG_mask;
  433   if (PreserveFramePointer) {
  434     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  435   }
  436   if (need_r12_heapbase()) {
  437     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  438   }
  439 
  440   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  441   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  442 
  443   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  444   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  445   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  446 
  447   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  448   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  449 
  450   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  451   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  452   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  453 
  454   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  455   // from the float_reg_legacy/float_reg_evex register class.
  456   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  457 
  458   if (Matcher::has_predicated_vectors()) {
  459     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
  460     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
  461     // curruption of value held in K1 register.
  462     if (PostLoopMultiversioning) {
  463       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()));
  464       const_cast<RegMask*>(&_VECTMASK_REG_mask)->Remove(OptoReg::as_OptoReg(k1->as_VMReg()->next()));
  465     }
  466   }
  467 }
  468 
  469 static bool generate_vzeroupper(Compile* C) {
  470   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  471 }
  472 
  473 static int clear_avx_size() {
  474   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  475 }
  476 
  477 // !!!!! Special hack to get all types of calls to specify the byte offset
  478 //       from the start of the call to the point where the return address
  479 //       will point.
  480 int MachCallStaticJavaNode::ret_addr_offset()
  481 {
  482   int offset = 5; // 5 bytes from start of call to where return address points
  483   offset += clear_avx_size();
  484   return offset;
  485 }
  486 
  487 int MachCallDynamicJavaNode::ret_addr_offset()
  488 {
  489   int offset = 15; // 15 bytes from start of call to where return address points
  490   offset += clear_avx_size();
  491   return offset;
  492 }
  493 
  494 int MachCallRuntimeNode::ret_addr_offset() {
  495   if (_entry_point == NULL) {
  496     // CallLeafNoFPInDirect
  497     return 3; // callq (register)
  498   }
  499   int offset = 13; // movq r10,#addr; callq (r10)
  500   if (this->ideal_Opcode() != Op_CallLeafVector) {
  501     offset += clear_avx_size();
  502   }
  503   return offset;
  504 }
  505 
  506 int MachCallNativeNode::ret_addr_offset() {
  507   int offset = 13; // movq r10,#addr; callq (r10)
  508   offset += clear_avx_size();
  509   return offset;
  510 }
  511 
  512 //
  513 // Compute padding required for nodes which need alignment
  514 //
  515 
  516 // The address of the call instruction needs to be 4-byte aligned to
  517 // ensure that it does not span a cache line so that it can be patched.
  518 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  519 {
  520   current_offset += clear_avx_size(); // skip vzeroupper
  521   current_offset += 1; // skip call opcode byte
  522   return align_up(current_offset, alignment_required()) - current_offset;
  523 }
  524 
  525 // The address of the call instruction needs to be 4-byte aligned to
  526 // ensure that it does not span a cache line so that it can be patched.
  527 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  528 {
  529   current_offset += clear_avx_size(); // skip vzeroupper
  530   current_offset += 11; // skip movq instruction + call opcode byte
  531   return align_up(current_offset, alignment_required()) - current_offset;
  532 }
  533 
  534 // EMIT_RM()
  535 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
  536   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
  537   cbuf.insts()->emit_int8(c);
  538 }
  539 
  540 // EMIT_CC()
  541 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
  542   unsigned char c = (unsigned char) (f1 | f2);
  543   cbuf.insts()->emit_int8(c);
  544 }
  545 
  546 // EMIT_OPCODE()
  547 void emit_opcode(CodeBuffer &cbuf, int code) {
  548   cbuf.insts()->emit_int8((unsigned char) code);
  549 }
  550 
  551 // EMIT_OPCODE() w/ relocation information
  552 void emit_opcode(CodeBuffer &cbuf,
  553                  int code, relocInfo::relocType reloc, int offset, int format)
  554 {
  555   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
  556   emit_opcode(cbuf, code);
  557 }
  558 
  559 // EMIT_D8()
  560 void emit_d8(CodeBuffer &cbuf, int d8) {
  561   cbuf.insts()->emit_int8((unsigned char) d8);
  562 }
  563 
  564 // EMIT_D16()
  565 void emit_d16(CodeBuffer &cbuf, int d16) {
  566   cbuf.insts()->emit_int16(d16);
  567 }
  568 
  569 // EMIT_D32()
  570 void emit_d32(CodeBuffer &cbuf, int d32) {
  571   cbuf.insts()->emit_int32(d32);
  572 }
  573 
  574 // EMIT_D64()
  575 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
  576   cbuf.insts()->emit_int64(d64);
  577 }
  578 
  579 // emit 32 bit value and construct relocation entry from relocInfo::relocType
  580 void emit_d32_reloc(CodeBuffer& cbuf,
  581                     int d32,
  582                     relocInfo::relocType reloc,
  583                     int format)
  584 {
  585   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
  586   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  587   cbuf.insts()->emit_int32(d32);
  588 }
  589 
  590 // emit 32 bit value and construct relocation entry from RelocationHolder
  591 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
  592 #ifdef ASSERT
  593   if (rspec.reloc()->type() == relocInfo::oop_type &&
  594       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
  595     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
  596     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
  597   }
  598 #endif
  599   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  600   cbuf.insts()->emit_int32(d32);
  601 }
  602 
  603 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
  604   address next_ip = cbuf.insts_end() + 4;
  605   emit_d32_reloc(cbuf, (int) (addr - next_ip),
  606                  external_word_Relocation::spec(addr),
  607                  RELOC_DISP32);
  608 }
  609 
  610 
  611 // emit 64 bit value and construct relocation entry from relocInfo::relocType
  612 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
  613   cbuf.relocate(cbuf.insts_mark(), reloc, format);
  614   cbuf.insts()->emit_int64(d64);
  615 }
  616 
  617 // emit 64 bit value and construct relocation entry from RelocationHolder
  618 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
  619 #ifdef ASSERT
  620   if (rspec.reloc()->type() == relocInfo::oop_type &&
  621       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
  622     assert(Universe::heap()->is_in((address)d64), "should be real oop");
  623     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
  624   }
  625 #endif
  626   cbuf.relocate(cbuf.insts_mark(), rspec, format);
  627   cbuf.insts()->emit_int64(d64);
  628 }
  629 
  630 // Access stack slot for load or store
  631 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
  632 {
  633   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
  634   if (-0x80 <= disp && disp < 0x80) {
  635     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
  636     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  637     emit_d8(cbuf, disp);     // Displacement  // R/M byte
  638   } else {
  639     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
  640     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
  641     emit_d32(cbuf, disp);     // Displacement // R/M byte
  642   }
  643 }
  644 
  645    // rRegI ereg, memory mem) %{    // emit_reg_mem
  646 void encode_RegMem(CodeBuffer &cbuf,
  647                    int reg,
  648                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
  649 {
  650   assert(disp_reloc == relocInfo::none, "cannot have disp");
  651   int regenc = reg & 7;
  652   int baseenc = base & 7;
  653   int indexenc = index & 7;
  654 
  655   // There is no index & no scale, use form without SIB byte
  656   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
  657     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  658     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  659       emit_rm(cbuf, 0x0, regenc, baseenc); // *
  660     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  661       // If 8-bit displacement, mode 0x1
  662       emit_rm(cbuf, 0x1, regenc, baseenc); // *
  663       emit_d8(cbuf, disp);
  664     } else {
  665       // If 32-bit displacement
  666       if (base == -1) { // Special flag for absolute address
  667         emit_rm(cbuf, 0x0, regenc, 0x5); // *
  668         if (disp_reloc != relocInfo::none) {
  669           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  670         } else {
  671           emit_d32(cbuf, disp);
  672         }
  673       } else {
  674         // Normal base + offset
  675         emit_rm(cbuf, 0x2, regenc, baseenc); // *
  676         if (disp_reloc != relocInfo::none) {
  677           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  678         } else {
  679           emit_d32(cbuf, disp);
  680         }
  681       }
  682     }
  683   } else {
  684     // Else, encode with the SIB byte
  685     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
  686     if (disp == 0 && base != RBP_enc && base != R13_enc) {
  687       // If no displacement
  688       emit_rm(cbuf, 0x0, regenc, 0x4); // *
  689       emit_rm(cbuf, scale, indexenc, baseenc);
  690     } else {
  691       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
  692         // If 8-bit displacement, mode 0x1
  693         emit_rm(cbuf, 0x1, regenc, 0x4); // *
  694         emit_rm(cbuf, scale, indexenc, baseenc);
  695         emit_d8(cbuf, disp);
  696       } else {
  697         // If 32-bit displacement
  698         if (base == 0x04 ) {
  699           emit_rm(cbuf, 0x2, regenc, 0x4);
  700           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
  701         } else {
  702           emit_rm(cbuf, 0x2, regenc, 0x4);
  703           emit_rm(cbuf, scale, indexenc, baseenc); // *
  704         }
  705         if (disp_reloc != relocInfo::none) {
  706           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
  707         } else {
  708           emit_d32(cbuf, disp);
  709         }
  710       }
  711     }
  712   }
  713 }
  714 
  715 // This could be in MacroAssembler but it's fairly C2 specific
  716 void emit_cmpfp_fixup(MacroAssembler& _masm) {
  717   Label exit;
  718   __ jccb(Assembler::noParity, exit);
  719   __ pushf();
  720   //
  721   // comiss/ucomiss instructions set ZF,PF,CF flags and
  722   // zero OF,AF,SF for NaN values.
  723   // Fixup flags by zeroing ZF,PF so that compare of NaN
  724   // values returns 'less than' result (CF is set).
  725   // Leave the rest of flags unchanged.
  726   //
  727   //    7 6 5 4 3 2 1 0
  728   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  729   //    0 0 1 0 1 0 1 1   (0x2B)
  730   //
  731   __ andq(Address(rsp, 0), 0xffffff2b);
  732   __ popf();
  733   __ bind(exit);
  734 }
  735 
  736 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
  737   Label done;
  738   __ movl(dst, -1);
  739   __ jcc(Assembler::parity, done);
  740   __ jcc(Assembler::below, done);
  741   __ setb(Assembler::notEqual, dst);
  742   __ movzbl(dst, dst);
  743   __ bind(done);
  744 }
  745 
  746 // Math.min()    # Math.max()
  747 // --------------------------
  748 // ucomis[s/d]   #
  749 // ja   -> b     # a
  750 // jp   -> NaN   # NaN
  751 // jb   -> a     # b
  752 // je            #
  753 // |-jz -> a | b # a & b
  754 // |    -> a     #
  755 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
  756                      XMMRegister a, XMMRegister b,
  757                      XMMRegister xmmt, Register rt,
  758                      bool min, bool single) {
  759 
  760   Label nan, zero, below, above, done;
  761 
  762   if (single)
  763     __ ucomiss(a, b);
  764   else
  765     __ ucomisd(a, b);
  766 
  767   if (dst->encoding() != (min ? b : a)->encoding())
  768     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  769   else
  770     __ jccb(Assembler::above, done);
  771 
  772   __ jccb(Assembler::parity, nan);  // PF=1
  773   __ jccb(Assembler::below, below); // CF=1
  774 
  775   // equal
  776   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  777   if (single) {
  778     __ ucomiss(a, xmmt);
  779     __ jccb(Assembler::equal, zero);
  780 
  781     __ movflt(dst, a);
  782     __ jmp(done);
  783   }
  784   else {
  785     __ ucomisd(a, xmmt);
  786     __ jccb(Assembler::equal, zero);
  787 
  788     __ movdbl(dst, a);
  789     __ jmp(done);
  790   }
  791 
  792   __ bind(zero);
  793   if (min)
  794     __ vpor(dst, a, b, Assembler::AVX_128bit);
  795   else
  796     __ vpand(dst, a, b, Assembler::AVX_128bit);
  797 
  798   __ jmp(done);
  799 
  800   __ bind(above);
  801   if (single)
  802     __ movflt(dst, min ? b : a);
  803   else
  804     __ movdbl(dst, min ? b : a);
  805 
  806   __ jmp(done);
  807 
  808   __ bind(nan);
  809   if (single) {
  810     __ movl(rt, 0x7fc00000); // Float.NaN
  811     __ movdl(dst, rt);
  812   }
  813   else {
  814     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  815     __ movdq(dst, rt);
  816   }
  817   __ jmp(done);
  818 
  819   __ bind(below);
  820   if (single)
  821     __ movflt(dst, min ? a : b);
  822   else
  823     __ movdbl(dst, min ? a : b);
  824 
  825   __ bind(done);
  826 }
  827 
  828 //=============================================================================
  829 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  830 
  831 int ConstantTable::calculate_table_base_offset() const {
  832   return 0;  // absolute addressing, no offset
  833 }
  834 
  835 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  836 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  837   ShouldNotReachHere();
  838 }
  839 
  840 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
  841   // Empty encoding
  842 }
  843 
  844 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  845   return 0;
  846 }
  847 
  848 #ifndef PRODUCT
  849 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  850   st->print("# MachConstantBaseNode (empty encoding)");
  851 }
  852 #endif
  853 
  854 
  855 //=============================================================================
  856 #ifndef PRODUCT
  857 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  858   Compile* C = ra_->C;
  859 
  860   int framesize = C->output()->frame_size_in_bytes();
  861   int bangsize = C->output()->bang_size_in_bytes();
  862   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  863   // Remove wordSize for return addr which is already pushed.
  864   framesize -= wordSize;
  865 
  866   if (C->output()->need_stack_bang(bangsize)) {
  867     framesize -= wordSize;
  868     st->print("# stack bang (%d bytes)", bangsize);
  869     st->print("\n\t");
  870     st->print("pushq   rbp\t# Save rbp");
  871     if (PreserveFramePointer) {
  872         st->print("\n\t");
  873         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  874     }
  875     if (framesize) {
  876       st->print("\n\t");
  877       st->print("subq    rsp, #%d\t# Create frame",framesize);
  878     }
  879   } else {
  880     st->print("subq    rsp, #%d\t# Create frame",framesize);
  881     st->print("\n\t");
  882     framesize -= wordSize;
  883     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  884     if (PreserveFramePointer) {
  885       st->print("\n\t");
  886       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  887       if (framesize > 0) {
  888         st->print("\n\t");
  889         st->print("addq    rbp, #%d", framesize);
  890       }
  891     }
  892   }
  893 
  894   if (VerifyStackAtCalls) {
  895     st->print("\n\t");
  896     framesize -= wordSize;
  897     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  898 #ifdef ASSERT
  899     st->print("\n\t");
  900     st->print("# stack alignment check");
  901 #endif
  902   }
  903   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
  904     st->print("\n\t");
  905     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
  906     st->print("\n\t");
  907     st->print("je      fast_entry\t");
  908     st->print("\n\t");
  909     st->print("call    #nmethod_entry_barrier_stub\t");
  910     st->print("\n\tfast_entry:");
  911   }
  912   st->cr();
  913 }
  914 #endif
  915 
  916 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
  917   Compile* C = ra_->C;
  918   MacroAssembler _masm(&cbuf);
  919 
  920   if (C->clinit_barrier_on_entry()) {
  921     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  922     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  923 
  924     Label L_skip_barrier;
  925     Register klass = rscratch1;
  926 
  927     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  928     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
  929 
  930     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  931 
  932     __ bind(L_skip_barrier);
  933   }
  934 
  935   __ verified_entry(C);
  936   __ bind(*_verified_entry);
  937 
  938   if (C->stub_function() == NULL) {
  939     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
  940     bs->nmethod_entry_barrier(&_masm);
  941   }
  942 
  943   C->output()->set_frame_complete(cbuf.insts_size());
  944 
  945   if (C->has_mach_constant_base_node()) {
  946     // NOTE: We set the table base offset here because users might be
  947     // emitted before MachConstantBaseNode.
  948     ConstantTable& constant_table = C->output()->constant_table();
  949     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  950   }
  951 }
  952 
  953 int MachPrologNode::reloc() const
  954 {
  955   return 0; // a large enough number
  956 }
  957 
  958 //=============================================================================
  959 #ifndef PRODUCT
  960 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  961 {
  962   Compile* C = ra_->C;
  963   if (generate_vzeroupper(C)) {
  964     st->print("vzeroupper");
  965     st->cr(); st->print("\t");
  966   }
  967 
  968   int framesize = C->output()->frame_size_in_bytes();
  969   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  970   // Remove word for return adr already pushed
  971   // and RBP
  972   framesize -= 2*wordSize;
  973 
  974   if (framesize) {
  975     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  976     st->print("\t");
  977   }
  978 
  979   st->print_cr("popq    rbp");
  980   if (do_polling() && C->is_method_compilation()) {
  981     st->print("\t");
  982     st->print_cr("cmpq     rsp, poll_offset[r15_thread] \n\t"
  983                  "ja       #safepoint_stub\t"
  984                  "# Safepoint: poll for GC");
  985   }
  986 }
  987 #endif
  988 
  989 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
  990 {
  991   Compile* C = ra_->C;
  992   MacroAssembler _masm(&cbuf);
  993 
  994   if (generate_vzeroupper(C)) {
  995     // Clear upper bits of YMM registers when current compiled code uses
  996     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  997     __ vzeroupper();
  998   }
  999 
 1000   // Subtract two words to account for return address and rbp
 1001   int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
 1002   __ remove_frame(initial_framesize, C->needs_stack_repair());
 1003 
 1004   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 1005     __ reserved_stack_check();
 1006   }
 1007 
 1008   if (do_polling() && C->is_method_compilation()) {
 1009     MacroAssembler _masm(&cbuf);
 1010     Label dummy_label;
 1011     Label* code_stub = &dummy_label;
 1012     if (!C->output()->in_scratch_emit_size()) {
 1013       code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
 1014     }
 1015     __ relocate(relocInfo::poll_return_type);
 1016     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
 1017   }
 1018 }
 1019 
 1020 int MachEpilogNode::reloc() const
 1021 {
 1022   return 2; // a large enough number
 1023 }
 1024 
 1025 const Pipeline* MachEpilogNode::pipeline() const
 1026 {
 1027   return MachNode::pipeline_class();
 1028 }
 1029 
 1030 //=============================================================================
 1031 
 1032 enum RC {
 1033   rc_bad,
 1034   rc_int,
 1035   rc_kreg,
 1036   rc_float,
 1037   rc_stack
 1038 };
 1039 
 1040 static enum RC rc_class(OptoReg::Name reg)
 1041 {
 1042   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 1043 
 1044   if (OptoReg::is_stack(reg)) return rc_stack;
 1045 
 1046   VMReg r = OptoReg::as_VMReg(reg);
 1047 
 1048   if (r->is_Register()) return rc_int;
 1049 
 1050   if (r->is_KRegister()) return rc_kreg;
 1051 
 1052   assert(r->is_XMMRegister(), "must be");
 1053   return rc_float;
 1054 }
 1055 
 1056 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1057 static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
 1058                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1059 
 1060 void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
 1061                      int stack_offset, int reg, uint ireg, outputStream* st);
 1062 
 1063 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 1064                                       int dst_offset, uint ireg, outputStream* st) {
 1065   if (cbuf) {
 1066     MacroAssembler _masm(cbuf);
 1067     switch (ireg) {
 1068     case Op_VecS:
 1069       __ movq(Address(rsp, -8), rax);
 1070       __ movl(rax, Address(rsp, src_offset));
 1071       __ movl(Address(rsp, dst_offset), rax);
 1072       __ movq(rax, Address(rsp, -8));
 1073       break;
 1074     case Op_VecD:
 1075       __ pushq(Address(rsp, src_offset));
 1076       __ popq (Address(rsp, dst_offset));
 1077       break;
 1078     case Op_VecX:
 1079       __ pushq(Address(rsp, src_offset));
 1080       __ popq (Address(rsp, dst_offset));
 1081       __ pushq(Address(rsp, src_offset+8));
 1082       __ popq (Address(rsp, dst_offset+8));
 1083       break;
 1084     case Op_VecY:
 1085       __ vmovdqu(Address(rsp, -32), xmm0);
 1086       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1087       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1088       __ vmovdqu(xmm0, Address(rsp, -32));
 1089       break;
 1090     case Op_VecZ:
 1091       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1092       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1093       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1094       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1095       break;
 1096     default:
 1097       ShouldNotReachHere();
 1098     }
 1099 #ifndef PRODUCT
 1100   } else {
 1101     switch (ireg) {
 1102     case Op_VecS:
 1103       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1104                 "movl    rax, [rsp + #%d]\n\t"
 1105                 "movl    [rsp + #%d], rax\n\t"
 1106                 "movq    rax, [rsp - #8]",
 1107                 src_offset, dst_offset);
 1108       break;
 1109     case Op_VecD:
 1110       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1111                 "popq    [rsp + #%d]",
 1112                 src_offset, dst_offset);
 1113       break;
 1114      case Op_VecX:
 1115       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1116                 "popq    [rsp + #%d]\n\t"
 1117                 "pushq   [rsp + #%d]\n\t"
 1118                 "popq    [rsp + #%d]",
 1119                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1120       break;
 1121     case Op_VecY:
 1122       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1123                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1124                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1125                 "vmovdqu xmm0, [rsp - #32]",
 1126                 src_offset, dst_offset);
 1127       break;
 1128     case Op_VecZ:
 1129       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1130                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1131                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1132                 "vmovdqu xmm0, [rsp - #64]",
 1133                 src_offset, dst_offset);
 1134       break;
 1135     default:
 1136       ShouldNotReachHere();
 1137     }
 1138 #endif
 1139   }
 1140 }
 1141 
 1142 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 1143                                        PhaseRegAlloc* ra_,
 1144                                        bool do_size,
 1145                                        outputStream* st) const {
 1146   assert(cbuf != NULL || st  != NULL, "sanity");
 1147   // Get registers to move
 1148   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1149   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1150   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1151   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1152 
 1153   enum RC src_second_rc = rc_class(src_second);
 1154   enum RC src_first_rc = rc_class(src_first);
 1155   enum RC dst_second_rc = rc_class(dst_second);
 1156   enum RC dst_first_rc = rc_class(dst_first);
 1157 
 1158   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1159          "must move at least 1 register" );
 1160 
 1161   if (src_first == dst_first && src_second == dst_second) {
 1162     // Self copy, no move
 1163     return 0;
 1164   }
 1165   if (bottom_type()->isa_vect() != NULL && bottom_type()->isa_vectmask() == NULL) {
 1166     uint ireg = ideal_reg();
 1167     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1168     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1169     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1170       // mem -> mem
 1171       int src_offset = ra_->reg2offset(src_first);
 1172       int dst_offset = ra_->reg2offset(dst_first);
 1173       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
 1174     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1175       vec_mov_helper(cbuf, src_first, dst_first, src_second, dst_second, ireg, st);
 1176     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1177       int stack_offset = ra_->reg2offset(dst_first);
 1178       vec_spill_helper(cbuf, false, stack_offset, src_first, ireg, st);
 1179     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1180       int stack_offset = ra_->reg2offset(src_first);
 1181       vec_spill_helper(cbuf, true,  stack_offset, dst_first, ireg, st);
 1182     } else {
 1183       ShouldNotReachHere();
 1184     }
 1185     return 0;
 1186   }
 1187   if (src_first_rc == rc_stack) {
 1188     // mem ->
 1189     if (dst_first_rc == rc_stack) {
 1190       // mem -> mem
 1191       assert(src_second != dst_first, "overlap");
 1192       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1193           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1194         // 64-bit
 1195         int src_offset = ra_->reg2offset(src_first);
 1196         int dst_offset = ra_->reg2offset(dst_first);
 1197         if (cbuf) {
 1198           MacroAssembler _masm(cbuf);
 1199           __ pushq(Address(rsp, src_offset));
 1200           __ popq (Address(rsp, dst_offset));
 1201 #ifndef PRODUCT
 1202         } else {
 1203           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1204                     "popq    [rsp + #%d]",
 1205                      src_offset, dst_offset);
 1206 #endif
 1207         }
 1208       } else {
 1209         // 32-bit
 1210         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1211         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1212         // No pushl/popl, so:
 1213         int src_offset = ra_->reg2offset(src_first);
 1214         int dst_offset = ra_->reg2offset(dst_first);
 1215         if (cbuf) {
 1216           MacroAssembler _masm(cbuf);
 1217           __ movq(Address(rsp, -8), rax);
 1218           __ movl(rax, Address(rsp, src_offset));
 1219           __ movl(Address(rsp, dst_offset), rax);
 1220           __ movq(rax, Address(rsp, -8));
 1221 #ifndef PRODUCT
 1222         } else {
 1223           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1224                     "movl    rax, [rsp + #%d]\n\t"
 1225                     "movl    [rsp + #%d], rax\n\t"
 1226                     "movq    rax, [rsp - #8]",
 1227                      src_offset, dst_offset);
 1228 #endif
 1229         }
 1230       }
 1231       return 0;
 1232     } else if (dst_first_rc == rc_int) {
 1233       // mem -> gpr
 1234       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1235           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1236         // 64-bit
 1237         int offset = ra_->reg2offset(src_first);
 1238         if (cbuf) {
 1239           MacroAssembler _masm(cbuf);
 1240           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1241 #ifndef PRODUCT
 1242         } else {
 1243           st->print("movq    %s, [rsp + #%d]\t# spill",
 1244                      Matcher::regName[dst_first],
 1245                      offset);
 1246 #endif
 1247         }
 1248       } else {
 1249         // 32-bit
 1250         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1251         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1252         int offset = ra_->reg2offset(src_first);
 1253         if (cbuf) {
 1254           MacroAssembler _masm(cbuf);
 1255           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1256 #ifndef PRODUCT
 1257         } else {
 1258           st->print("movl    %s, [rsp + #%d]\t# spill",
 1259                      Matcher::regName[dst_first],
 1260                      offset);
 1261 #endif
 1262         }
 1263       }
 1264       return 0;
 1265     } else if (dst_first_rc == rc_float) {
 1266       // mem-> xmm
 1267       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1268           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1269         // 64-bit
 1270         int offset = ra_->reg2offset(src_first);
 1271         if (cbuf) {
 1272           MacroAssembler _masm(cbuf);
 1273           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1274 #ifndef PRODUCT
 1275         } else {
 1276           st->print("%s  %s, [rsp + #%d]\t# spill",
 1277                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1278                      Matcher::regName[dst_first],
 1279                      offset);
 1280 #endif
 1281         }
 1282       } else {
 1283         // 32-bit
 1284         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1285         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1286         int offset = ra_->reg2offset(src_first);
 1287         if (cbuf) {
 1288           MacroAssembler _masm(cbuf);
 1289           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1290 #ifndef PRODUCT
 1291         } else {
 1292           st->print("movss   %s, [rsp + #%d]\t# spill",
 1293                      Matcher::regName[dst_first],
 1294                      offset);
 1295 #endif
 1296         }
 1297       }
 1298       return 0;
 1299     } else if (dst_first_rc == rc_kreg) {
 1300       // mem -> kreg
 1301       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1302           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1303         // 64-bit
 1304         int offset = ra_->reg2offset(src_first);
 1305         if (cbuf) {
 1306           MacroAssembler _masm(cbuf);
 1307           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1308 #ifndef PRODUCT
 1309         } else {
 1310           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1311                      Matcher::regName[dst_first],
 1312                      offset);
 1313 #endif
 1314         }
 1315       }
 1316       return 0;
 1317     }
 1318   } else if (src_first_rc == rc_int) {
 1319     // gpr ->
 1320     if (dst_first_rc == rc_stack) {
 1321       // gpr -> mem
 1322       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1323           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1324         // 64-bit
 1325         int offset = ra_->reg2offset(dst_first);
 1326         if (cbuf) {
 1327           MacroAssembler _masm(cbuf);
 1328           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1329 #ifndef PRODUCT
 1330         } else {
 1331           st->print("movq    [rsp + #%d], %s\t# spill",
 1332                      offset,
 1333                      Matcher::regName[src_first]);
 1334 #endif
 1335         }
 1336       } else {
 1337         // 32-bit
 1338         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1339         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1340         int offset = ra_->reg2offset(dst_first);
 1341         if (cbuf) {
 1342           MacroAssembler _masm(cbuf);
 1343           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1344 #ifndef PRODUCT
 1345         } else {
 1346           st->print("movl    [rsp + #%d], %s\t# spill",
 1347                      offset,
 1348                      Matcher::regName[src_first]);
 1349 #endif
 1350         }
 1351       }
 1352       return 0;
 1353     } else if (dst_first_rc == rc_int) {
 1354       // gpr -> gpr
 1355       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1356           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1357         // 64-bit
 1358         if (cbuf) {
 1359           MacroAssembler _masm(cbuf);
 1360           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1361                   as_Register(Matcher::_regEncode[src_first]));
 1362 #ifndef PRODUCT
 1363         } else {
 1364           st->print("movq    %s, %s\t# spill",
 1365                      Matcher::regName[dst_first],
 1366                      Matcher::regName[src_first]);
 1367 #endif
 1368         }
 1369         return 0;
 1370       } else {
 1371         // 32-bit
 1372         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1373         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1374         if (cbuf) {
 1375           MacroAssembler _masm(cbuf);
 1376           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1377                   as_Register(Matcher::_regEncode[src_first]));
 1378 #ifndef PRODUCT
 1379         } else {
 1380           st->print("movl    %s, %s\t# spill",
 1381                      Matcher::regName[dst_first],
 1382                      Matcher::regName[src_first]);
 1383 #endif
 1384         }
 1385         return 0;
 1386       }
 1387     } else if (dst_first_rc == rc_float) {
 1388       // gpr -> xmm
 1389       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1390           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1391         // 64-bit
 1392         if (cbuf) {
 1393           MacroAssembler _masm(cbuf);
 1394           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1395 #ifndef PRODUCT
 1396         } else {
 1397           st->print("movdq   %s, %s\t# spill",
 1398                      Matcher::regName[dst_first],
 1399                      Matcher::regName[src_first]);
 1400 #endif
 1401         }
 1402       } else {
 1403         // 32-bit
 1404         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1405         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1406         if (cbuf) {
 1407           MacroAssembler _masm(cbuf);
 1408           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdl   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       }
 1417       return 0;
 1418     } else if (dst_first_rc == rc_kreg) {
 1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1421         // 64-bit
 1422         if (cbuf) {
 1423           MacroAssembler _masm(cbuf);
 1424           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1425   #ifndef PRODUCT
 1426         } else {
 1427            st->print("kmovq   %s, %s\t# spill",
 1428                        Matcher::regName[dst_first],
 1429                        Matcher::regName[src_first]);
 1430   #endif
 1431         }
 1432       }
 1433       Unimplemented();
 1434       return 0;
 1435     }
 1436   } else if (src_first_rc == rc_float) {
 1437     // xmm ->
 1438     if (dst_first_rc == rc_stack) {
 1439       // xmm -> mem
 1440       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1441           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1442         // 64-bit
 1443         int offset = ra_->reg2offset(dst_first);
 1444         if (cbuf) {
 1445           MacroAssembler _masm(cbuf);
 1446           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1447 #ifndef PRODUCT
 1448         } else {
 1449           st->print("movsd   [rsp + #%d], %s\t# spill",
 1450                      offset,
 1451                      Matcher::regName[src_first]);
 1452 #endif
 1453         }
 1454       } else {
 1455         // 32-bit
 1456         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1457         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1458         int offset = ra_->reg2offset(dst_first);
 1459         if (cbuf) {
 1460           MacroAssembler _masm(cbuf);
 1461           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1462 #ifndef PRODUCT
 1463         } else {
 1464           st->print("movss   [rsp + #%d], %s\t# spill",
 1465                      offset,
 1466                      Matcher::regName[src_first]);
 1467 #endif
 1468         }
 1469       }
 1470       return 0;
 1471     } else if (dst_first_rc == rc_int) {
 1472       // xmm -> gpr
 1473       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1474           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1475         // 64-bit
 1476         if (cbuf) {
 1477           MacroAssembler _masm(cbuf);
 1478           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1479 #ifndef PRODUCT
 1480         } else {
 1481           st->print("movdq   %s, %s\t# spill",
 1482                      Matcher::regName[dst_first],
 1483                      Matcher::regName[src_first]);
 1484 #endif
 1485         }
 1486       } else {
 1487         // 32-bit
 1488         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1489         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1490         if (cbuf) {
 1491           MacroAssembler _masm(cbuf);
 1492           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1493 #ifndef PRODUCT
 1494         } else {
 1495           st->print("movdl   %s, %s\t# spill",
 1496                      Matcher::regName[dst_first],
 1497                      Matcher::regName[src_first]);
 1498 #endif
 1499         }
 1500       }
 1501       return 0;
 1502     } else if (dst_first_rc == rc_float) {
 1503       // xmm -> xmm
 1504       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1505           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1506         // 64-bit
 1507         if (cbuf) {
 1508           MacroAssembler _masm(cbuf);
 1509           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1510 #ifndef PRODUCT
 1511         } else {
 1512           st->print("%s  %s, %s\t# spill",
 1513                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1514                      Matcher::regName[dst_first],
 1515                      Matcher::regName[src_first]);
 1516 #endif
 1517         }
 1518       } else {
 1519         // 32-bit
 1520         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1521         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1522         if (cbuf) {
 1523           MacroAssembler _masm(cbuf);
 1524           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1525 #ifndef PRODUCT
 1526         } else {
 1527           st->print("%s  %s, %s\t# spill",
 1528                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1529                      Matcher::regName[dst_first],
 1530                      Matcher::regName[src_first]);
 1531 #endif
 1532         }
 1533       }
 1534       return 0;
 1535     } else if (dst_first_rc == rc_kreg) {
 1536       assert(false, "Illegal spilling");
 1537       return 0;
 1538     }
 1539   } else if (src_first_rc == rc_kreg) {
 1540     if (dst_first_rc == rc_stack) {
 1541       // mem -> kreg
 1542       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1543           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1544         // 64-bit
 1545         int offset = ra_->reg2offset(dst_first);
 1546         if (cbuf) {
 1547           MacroAssembler _masm(cbuf);
 1548           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1549 #ifndef PRODUCT
 1550         } else {
 1551           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1552                      offset,
 1553                      Matcher::regName[src_first]);
 1554 #endif
 1555         }
 1556       }
 1557       return 0;
 1558     } else if (dst_first_rc == rc_int) {
 1559       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1560           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1561         // 64-bit
 1562         if (cbuf) {
 1563           MacroAssembler _masm(cbuf);
 1564           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1565 #ifndef PRODUCT
 1566         } else {
 1567          st->print("kmovq   %s, %s\t# spill",
 1568                      Matcher::regName[dst_first],
 1569                      Matcher::regName[src_first]);
 1570 #endif
 1571         }
 1572       }
 1573       Unimplemented();
 1574       return 0;
 1575     } else if (dst_first_rc == rc_kreg) {
 1576       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1577           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1578         // 64-bit
 1579         if (cbuf) {
 1580           MacroAssembler _masm(cbuf);
 1581           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1582 #ifndef PRODUCT
 1583         } else {
 1584          st->print("kmovq   %s, %s\t# spill",
 1585                      Matcher::regName[dst_first],
 1586                      Matcher::regName[src_first]);
 1587 #endif
 1588         }
 1589       }
 1590       return 0;
 1591     } else if (dst_first_rc == rc_float) {
 1592       assert(false, "Illegal spill");
 1593       return 0;
 1594     }
 1595   }
 1596 
 1597   assert(0," foo ");
 1598   Unimplemented();
 1599   return 0;
 1600 }
 1601 
 1602 #ifndef PRODUCT
 1603 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1604   implementation(NULL, ra_, false, st);
 1605 }
 1606 #endif
 1607 
 1608 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 1609   implementation(&cbuf, ra_, false, NULL);
 1610 }
 1611 
 1612 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1613   return MachNode::size(ra_);
 1614 }
 1615 
 1616 //=============================================================================
 1617 #ifndef PRODUCT
 1618 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1619 {
 1620   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1621   int reg = ra_->get_reg_first(this);
 1622   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1623             Matcher::regName[reg], offset);
 1624 }
 1625 #endif
 1626 
 1627 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1628 {
 1629   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1630   int reg = ra_->get_encode(this);
 1631   if (offset >= 0x80) {
 1632     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1633     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1634     emit_rm(cbuf, 0x2, reg & 7, 0x04);
 1635     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1636     emit_d32(cbuf, offset);
 1637   } else {
 1638     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
 1639     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
 1640     emit_rm(cbuf, 0x1, reg & 7, 0x04);
 1641     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
 1642     emit_d8(cbuf, offset);
 1643   }
 1644 }
 1645 
 1646 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1647 {
 1648   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1649   return (offset < 0x80) ? 5 : 8; // REX
 1650 }
 1651 
 1652 //=============================================================================
 1653 #ifndef PRODUCT
 1654 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1655 {
 1656   st->print_cr("MachVEPNode");
 1657 }
 1658 #endif
 1659 
 1660 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1661 {
 1662   MacroAssembler _masm(&cbuf);
 1663   if (!_verified) {
 1664     uint insts_size = cbuf.insts_size();
 1665     if (UseCompressedClassPointers) {
 1666       __ load_klass(rscratch1, j_rarg0, rscratch2);
 1667       __ cmpptr(rax, rscratch1);
 1668     } else {
 1669       __ cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1670     }
 1671     __ jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1672   } else {
 1673     // Unpack inline type args passed as oop and then jump to
 1674     // the verified entry point (skipping the unverified entry).
 1675     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 1676     // Emit code for verified entry and save increment for stack repair on return
 1677     __ verified_entry(ra_->C, sp_inc);
 1678     __ jmp(*_verified_entry);
 1679   }
 1680 }
 1681 
 1682 //=============================================================================
 1683 #ifndef PRODUCT
 1684 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1685 {
 1686   if (UseCompressedClassPointers) {
 1687     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1688     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
 1689     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
 1690   } else {
 1691     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
 1692                  "# Inline cache check");
 1693   }
 1694   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1695   st->print_cr("\tnop\t# nops to align entry point");
 1696 }
 1697 #endif
 1698 
 1699 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 1700 {
 1701   MacroAssembler masm(&cbuf);
 1702   uint insts_size = cbuf.insts_size();
 1703   if (UseCompressedClassPointers) {
 1704     masm.load_klass(rscratch1, j_rarg0, rscratch2);
 1705     masm.cmpptr(rax, rscratch1);
 1706   } else {
 1707     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
 1708   }
 1709 
 1710   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1711 
 1712   /* WARNING these NOPs are critical so that verified entry point is properly
 1713      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1714   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
 1715   if (OptoBreakpoint) {
 1716     // Leave space for int3
 1717     nops_cnt -= 1;
 1718   }
 1719   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1720   if (nops_cnt > 0)
 1721     masm.nop(nops_cnt);
 1722 }
 1723 
 1724 //=============================================================================
 1725 
 1726 const bool Matcher::supports_vector_calling_convention(void) {
 1727   if (EnableVectorSupport && UseVectorStubs) {
 1728     return true;
 1729   }
 1730   return false;
 1731 }
 1732 
 1733 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1734   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1735   int lo = XMM0_num;
 1736   int hi = XMM0b_num;
 1737   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1738   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1739   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1740   return OptoRegPair(hi, lo);
 1741 }
 1742 
 1743 // Is this branch offset short enough that a short branch can be used?
 1744 //
 1745 // NOTE: If the platform does not provide any short branch variants, then
 1746 //       this method should return false for offset 0.
 1747 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1748   // The passed offset is relative to address of the branch.
 1749   // On 86 a branch displacement is calculated relative to address
 1750   // of a next instruction.
 1751   offset -= br_size;
 1752 
 1753   // the short version of jmpConUCF2 contains multiple branches,
 1754   // making the reach slightly less
 1755   if (rule == jmpConUCF2_rule)
 1756     return (-126 <= offset && offset <= 125);
 1757   return (-128 <= offset && offset <= 127);
 1758 }
 1759 
 1760 // Return whether or not this register is ever used as an argument.
 1761 // This function is used on startup to build the trampoline stubs in
 1762 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1763 // call in the trampoline, and arguments in those registers not be
 1764 // available to the callee.
 1765 bool Matcher::can_be_java_arg(int reg)
 1766 {
 1767   return
 1768     reg ==  RDI_num || reg == RDI_H_num ||
 1769     reg ==  RSI_num || reg == RSI_H_num ||
 1770     reg ==  RDX_num || reg == RDX_H_num ||
 1771     reg ==  RCX_num || reg == RCX_H_num ||
 1772     reg ==   R8_num || reg ==  R8_H_num ||
 1773     reg ==   R9_num || reg ==  R9_H_num ||
 1774     reg ==  R12_num || reg == R12_H_num ||
 1775     reg == XMM0_num || reg == XMM0b_num ||
 1776     reg == XMM1_num || reg == XMM1b_num ||
 1777     reg == XMM2_num || reg == XMM2b_num ||
 1778     reg == XMM3_num || reg == XMM3b_num ||
 1779     reg == XMM4_num || reg == XMM4b_num ||
 1780     reg == XMM5_num || reg == XMM5b_num ||
 1781     reg == XMM6_num || reg == XMM6b_num ||
 1782     reg == XMM7_num || reg == XMM7b_num;
 1783 }
 1784 
 1785 bool Matcher::is_spillable_arg(int reg)
 1786 {
 1787   return can_be_java_arg(reg);
 1788 }
 1789 
 1790 uint Matcher::int_pressure_limit()
 1791 {
 1792   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1793 }
 1794 
 1795 uint Matcher::float_pressure_limit()
 1796 {
 1797   // After experiment around with different values, the following default threshold
 1798   // works best for LCM's register pressure scheduling on x64.
 1799   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1800   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1801   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1802 }
 1803 
 1804 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1805   // In 64 bit mode a code which use multiply when
 1806   // devisor is constant is faster than hardware
 1807   // DIV instruction (it uses MulHiL).
 1808   return false;
 1809 }
 1810 
 1811 // Register for DIVI projection of divmodI
 1812 RegMask Matcher::divI_proj_mask() {
 1813   return INT_RAX_REG_mask();
 1814 }
 1815 
 1816 // Register for MODI projection of divmodI
 1817 RegMask Matcher::modI_proj_mask() {
 1818   return INT_RDX_REG_mask();
 1819 }
 1820 
 1821 // Register for DIVL projection of divmodL
 1822 RegMask Matcher::divL_proj_mask() {
 1823   return LONG_RAX_REG_mask();
 1824 }
 1825 
 1826 // Register for MODL projection of divmodL
 1827 RegMask Matcher::modL_proj_mask() {
 1828   return LONG_RDX_REG_mask();
 1829 }
 1830 
 1831 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1832 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1833     return NO_REG_mask();
 1834 }
 1835 
 1836 %}
 1837 
 1838 //----------ENCODING BLOCK-----------------------------------------------------
 1839 // This block specifies the encoding classes used by the compiler to
 1840 // output byte streams.  Encoding classes are parameterized macros
 1841 // used by Machine Instruction Nodes in order to generate the bit
 1842 // encoding of the instruction.  Operands specify their base encoding
 1843 // interface with the interface keyword.  There are currently
 1844 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1845 // COND_INTER.  REG_INTER causes an operand to generate a function
 1846 // which returns its register number when queried.  CONST_INTER causes
 1847 // an operand to generate a function which returns the value of the
 1848 // constant when queried.  MEMORY_INTER causes an operand to generate
 1849 // four functions which return the Base Register, the Index Register,
 1850 // the Scale Value, and the Offset Value of the operand when queried.
 1851 // COND_INTER causes an operand to generate six functions which return
 1852 // the encoding code (ie - encoding bits for the instruction)
 1853 // associated with each basic boolean condition for a conditional
 1854 // instruction.
 1855 //
 1856 // Instructions specify two basic values for encoding.  Again, a
 1857 // function is available to check if the constant displacement is an
 1858 // oop. They use the ins_encode keyword to specify their encoding
 1859 // classes (which must be a sequence of enc_class names, and their
 1860 // parameters, specified in the encoding block), and they use the
 1861 // opcode keyword to specify, in order, their primary, secondary, and
 1862 // tertiary opcode.  Only the opcode sections which a particular
 1863 // instruction needs for encoding need to be specified.
 1864 encode %{
 1865   // Build emit functions for each basic byte or larger field in the
 1866   // intel encoding scheme (opcode, rm, sib, immediate), and call them
 1867   // from C++ code in the enc_class source block.  Emit functions will
 1868   // live in the main source block for now.  In future, we can
 1869   // generalize this by adding a syntax that specifies the sizes of
 1870   // fields in an order, so that the adlc can build the emit functions
 1871   // automagically
 1872 
 1873   // Emit primary opcode
 1874   enc_class OpcP
 1875   %{
 1876     emit_opcode(cbuf, $primary);
 1877   %}
 1878 
 1879   // Emit secondary opcode
 1880   enc_class OpcS
 1881   %{
 1882     emit_opcode(cbuf, $secondary);
 1883   %}
 1884 
 1885   // Emit tertiary opcode
 1886   enc_class OpcT
 1887   %{
 1888     emit_opcode(cbuf, $tertiary);
 1889   %}
 1890 
 1891   // Emit opcode directly
 1892   enc_class Opcode(immI d8)
 1893   %{
 1894     emit_opcode(cbuf, $d8$$constant);
 1895   %}
 1896 
 1897   // Emit size prefix
 1898   enc_class SizePrefix
 1899   %{
 1900     emit_opcode(cbuf, 0x66);
 1901   %}
 1902 
 1903   enc_class reg(rRegI reg)
 1904   %{
 1905     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
 1906   %}
 1907 
 1908   enc_class reg_reg(rRegI dst, rRegI src)
 1909   %{
 1910     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1911   %}
 1912 
 1913   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
 1914   %{
 1915     emit_opcode(cbuf, $opcode$$constant);
 1916     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 1917   %}
 1918 
 1919   enc_class cdql_enc(no_rax_rdx_RegI div)
 1920   %{
 1921     // Full implementation of Java idiv and irem; checks for
 1922     // special case as described in JVM spec., p.243 & p.271.
 1923     //
 1924     //         normal case                           special case
 1925     //
 1926     // input : rax: dividend                         min_int
 1927     //         reg: divisor                          -1
 1928     //
 1929     // output: rax: quotient  (= rax idiv reg)       min_int
 1930     //         rdx: remainder (= rax irem reg)       0
 1931     //
 1932     //  Code sequnce:
 1933     //
 1934     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1935     //    5:   75 07/08                jne    e <normal>
 1936     //    7:   33 d2                   xor    %edx,%edx
 1937     //  [div >= 8 -> offset + 1]
 1938     //  [REX_B]
 1939     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1940     //    c:   74 03/04                je     11 <done>
 1941     // 000000000000000e <normal>:
 1942     //    e:   99                      cltd
 1943     //  [div >= 8 -> offset + 1]
 1944     //  [REX_B]
 1945     //    f:   f7 f9                   idiv   $div
 1946     // 0000000000000011 <done>:
 1947     MacroAssembler _masm(&cbuf);
 1948     Label normal;
 1949     Label done;
 1950 
 1951     // cmp    $0x80000000,%eax
 1952     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1953 
 1954     // jne    e <normal>
 1955     __ jccb(Assembler::notEqual, normal);
 1956 
 1957     // xor    %edx,%edx
 1958     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1959 
 1960     // cmp    $0xffffffffffffffff,%ecx
 1961     __ cmpl($div$$Register, -1);
 1962 
 1963     // je     11 <done>
 1964     __ jccb(Assembler::equal, done);
 1965 
 1966     // <normal>
 1967     // cltd
 1968     __ bind(normal);
 1969     __ cdql();
 1970 
 1971     // idivl
 1972     // <done>
 1973     __ idivl($div$$Register);
 1974     __ bind(done);
 1975   %}
 1976 
 1977   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1978   %{
 1979     // Full implementation of Java ldiv and lrem; checks for
 1980     // special case as described in JVM spec., p.243 & p.271.
 1981     //
 1982     //         normal case                           special case
 1983     //
 1984     // input : rax: dividend                         min_long
 1985     //         reg: divisor                          -1
 1986     //
 1987     // output: rax: quotient  (= rax idiv reg)       min_long
 1988     //         rdx: remainder (= rax irem reg)       0
 1989     //
 1990     //  Code sequnce:
 1991     //
 1992     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1993     //    7:   00 00 80
 1994     //    a:   48 39 d0                cmp    %rdx,%rax
 1995     //    d:   75 08                   jne    17 <normal>
 1996     //    f:   33 d2                   xor    %edx,%edx
 1997     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1998     //   15:   74 05                   je     1c <done>
 1999     // 0000000000000017 <normal>:
 2000     //   17:   48 99                   cqto
 2001     //   19:   48 f7 f9                idiv   $div
 2002     // 000000000000001c <done>:
 2003     MacroAssembler _masm(&cbuf);
 2004     Label normal;
 2005     Label done;
 2006 
 2007     // mov    $0x8000000000000000,%rdx
 2008     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 2009 
 2010     // cmp    %rdx,%rax
 2011     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 2012 
 2013     // jne    17 <normal>
 2014     __ jccb(Assembler::notEqual, normal);
 2015 
 2016     // xor    %edx,%edx
 2017     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 2018 
 2019     // cmp    $0xffffffffffffffff,$div
 2020     __ cmpq($div$$Register, -1);
 2021 
 2022     // je     1e <done>
 2023     __ jccb(Assembler::equal, done);
 2024 
 2025     // <normal>
 2026     // cqto
 2027     __ bind(normal);
 2028     __ cdqq();
 2029 
 2030     // idivq (note: must be emitted by the user of this rule)
 2031     // <done>
 2032     __ idivq($div$$Register);
 2033     __ bind(done);
 2034   %}
 2035 
 2036   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
 2037   enc_class OpcSE(immI imm)
 2038   %{
 2039     // Emit primary opcode and set sign-extend bit
 2040     // Check for 8-bit immediate, and set sign extend bit in opcode
 2041     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2042       emit_opcode(cbuf, $primary | 0x02);
 2043     } else {
 2044       // 32-bit immediate
 2045       emit_opcode(cbuf, $primary);
 2046     }
 2047   %}
 2048 
 2049   enc_class OpcSErm(rRegI dst, immI imm)
 2050   %{
 2051     // OpcSEr/m
 2052     int dstenc = $dst$$reg;
 2053     if (dstenc >= 8) {
 2054       emit_opcode(cbuf, Assembler::REX_B);
 2055       dstenc -= 8;
 2056     }
 2057     // Emit primary opcode and set sign-extend bit
 2058     // Check for 8-bit immediate, and set sign extend bit in opcode
 2059     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2060       emit_opcode(cbuf, $primary | 0x02);
 2061     } else {
 2062       // 32-bit immediate
 2063       emit_opcode(cbuf, $primary);
 2064     }
 2065     // Emit r/m byte with secondary opcode, after primary opcode.
 2066     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2067   %}
 2068 
 2069   enc_class OpcSErm_wide(rRegL dst, immI imm)
 2070   %{
 2071     // OpcSEr/m
 2072     int dstenc = $dst$$reg;
 2073     if (dstenc < 8) {
 2074       emit_opcode(cbuf, Assembler::REX_W);
 2075     } else {
 2076       emit_opcode(cbuf, Assembler::REX_WB);
 2077       dstenc -= 8;
 2078     }
 2079     // Emit primary opcode and set sign-extend bit
 2080     // Check for 8-bit immediate, and set sign extend bit in opcode
 2081     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2082       emit_opcode(cbuf, $primary | 0x02);
 2083     } else {
 2084       // 32-bit immediate
 2085       emit_opcode(cbuf, $primary);
 2086     }
 2087     // Emit r/m byte with secondary opcode, after primary opcode.
 2088     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2089   %}
 2090 
 2091   enc_class Con8or32(immI imm)
 2092   %{
 2093     // Check for 8-bit immediate, and set sign extend bit in opcode
 2094     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
 2095       $$$emit8$imm$$constant;
 2096     } else {
 2097       // 32-bit immediate
 2098       $$$emit32$imm$$constant;
 2099     }
 2100   %}
 2101 
 2102   enc_class opc2_reg(rRegI dst)
 2103   %{
 2104     // BSWAP
 2105     emit_cc(cbuf, $secondary, $dst$$reg);
 2106   %}
 2107 
 2108   enc_class opc3_reg(rRegI dst)
 2109   %{
 2110     // BSWAP
 2111     emit_cc(cbuf, $tertiary, $dst$$reg);
 2112   %}
 2113 
 2114   enc_class reg_opc(rRegI div)
 2115   %{
 2116     // INC, DEC, IDIV, IMOD, JMP indirect, ...
 2117     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
 2118   %}
 2119 
 2120   enc_class enc_cmov(cmpOp cop)
 2121   %{
 2122     // CMOV
 2123     $$$emit8$primary;
 2124     emit_cc(cbuf, $secondary, $cop$$cmpcode);
 2125   %}
 2126 
 2127   enc_class enc_PartialSubtypeCheck()
 2128   %{
 2129     Register Rrdi = as_Register(RDI_enc); // result register
 2130     Register Rrax = as_Register(RAX_enc); // super class
 2131     Register Rrcx = as_Register(RCX_enc); // killed
 2132     Register Rrsi = as_Register(RSI_enc); // sub class
 2133     Label miss;
 2134     const bool set_cond_codes = true;
 2135 
 2136     MacroAssembler _masm(&cbuf);
 2137     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
 2138                                      NULL, &miss,
 2139                                      /*set_cond_codes:*/ true);
 2140     if ($primary) {
 2141       __ xorptr(Rrdi, Rrdi);
 2142     }
 2143     __ bind(miss);
 2144   %}
 2145 
 2146   enc_class clear_avx %{
 2147     debug_only(int off0 = cbuf.insts_size());
 2148     if (generate_vzeroupper(Compile::current())) {
 2149       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 2150       // Clear upper bits of YMM registers when current compiled code uses
 2151       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 2152       MacroAssembler _masm(&cbuf);
 2153       __ vzeroupper();
 2154     }
 2155     debug_only(int off1 = cbuf.insts_size());
 2156     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 2157   %}
 2158 
 2159   enc_class Java_To_Runtime(method meth) %{
 2160     // No relocation needed
 2161     MacroAssembler _masm(&cbuf);
 2162     __ mov64(r10, (int64_t) $meth$$method);
 2163     __ call(r10);
 2164   %}
 2165 
 2166   enc_class Java_To_Interpreter(method meth)
 2167   %{
 2168     // CALL Java_To_Interpreter
 2169     // This is the instruction starting address for relocation info.
 2170     cbuf.set_insts_mark();
 2171     $$$emit8$primary;
 2172     // CALL directly to the runtime
 2173     emit_d32_reloc(cbuf,
 2174                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2175                    runtime_call_Relocation::spec(),
 2176                    RELOC_DISP32);
 2177   %}
 2178 
 2179   enc_class Java_Static_Call(method meth)
 2180   %{
 2181     // JAVA STATIC CALL
 2182     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 2183     // determine who we intended to call.
 2184     cbuf.set_insts_mark();
 2185     $$$emit8$primary;
 2186 
 2187     if (!_method) {
 2188       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2189                      runtime_call_Relocation::spec(),
 2190                      RELOC_DISP32);
 2191     } else {
 2192       int method_index = resolved_method_index(cbuf);
 2193       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 2194                                                   : static_call_Relocation::spec(method_index);
 2195       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 2196                      rspec, RELOC_DISP32);
 2197       // Emit stubs for static call.
 2198       address mark = cbuf.insts_mark();
 2199       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
 2200       if (stub == NULL) {
 2201         ciEnv::current()->record_failure("CodeCache is full");
 2202         return;
 2203       }
 2204     }
 2205   %}
 2206 
 2207   enc_class Java_Dynamic_Call(method meth) %{
 2208     MacroAssembler _masm(&cbuf);
 2209     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
 2210   %}
 2211 
 2212   enc_class Java_Compiled_Call(method meth)
 2213   %{
 2214     // JAVA COMPILED CALL
 2215     int disp = in_bytes(Method:: from_compiled_offset());
 2216 
 2217     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
 2218     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
 2219 
 2220     // callq *disp(%rax)
 2221     cbuf.set_insts_mark();
 2222     $$$emit8$primary;
 2223     if (disp < 0x80) {
 2224       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
 2225       emit_d8(cbuf, disp); // Displacement
 2226     } else {
 2227       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
 2228       emit_d32(cbuf, disp); // Displacement
 2229     }
 2230   %}
 2231 
 2232   enc_class reg_opc_imm(rRegI dst, immI8 shift)
 2233   %{
 2234     // SAL, SAR, SHR
 2235     int dstenc = $dst$$reg;
 2236     if (dstenc >= 8) {
 2237       emit_opcode(cbuf, Assembler::REX_B);
 2238       dstenc -= 8;
 2239     }
 2240     $$$emit8$primary;
 2241     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2242     $$$emit8$shift$$constant;
 2243   %}
 2244 
 2245   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
 2246   %{
 2247     // SAL, SAR, SHR
 2248     int dstenc = $dst$$reg;
 2249     if (dstenc < 8) {
 2250       emit_opcode(cbuf, Assembler::REX_W);
 2251     } else {
 2252       emit_opcode(cbuf, Assembler::REX_WB);
 2253       dstenc -= 8;
 2254     }
 2255     $$$emit8$primary;
 2256     emit_rm(cbuf, 0x3, $secondary, dstenc);
 2257     $$$emit8$shift$$constant;
 2258   %}
 2259 
 2260   enc_class load_immI(rRegI dst, immI src)
 2261   %{
 2262     int dstenc = $dst$$reg;
 2263     if (dstenc >= 8) {
 2264       emit_opcode(cbuf, Assembler::REX_B);
 2265       dstenc -= 8;
 2266     }
 2267     emit_opcode(cbuf, 0xB8 | dstenc);
 2268     $$$emit32$src$$constant;
 2269   %}
 2270 
 2271   enc_class load_immL(rRegL dst, immL src)
 2272   %{
 2273     int dstenc = $dst$$reg;
 2274     if (dstenc < 8) {
 2275       emit_opcode(cbuf, Assembler::REX_W);
 2276     } else {
 2277       emit_opcode(cbuf, Assembler::REX_WB);
 2278       dstenc -= 8;
 2279     }
 2280     emit_opcode(cbuf, 0xB8 | dstenc);
 2281     emit_d64(cbuf, $src$$constant);
 2282   %}
 2283 
 2284   enc_class load_immUL32(rRegL dst, immUL32 src)
 2285   %{
 2286     // same as load_immI, but this time we care about zeroes in the high word
 2287     int dstenc = $dst$$reg;
 2288     if (dstenc >= 8) {
 2289       emit_opcode(cbuf, Assembler::REX_B);
 2290       dstenc -= 8;
 2291     }
 2292     emit_opcode(cbuf, 0xB8 | dstenc);
 2293     $$$emit32$src$$constant;
 2294   %}
 2295 
 2296   enc_class load_immL32(rRegL dst, immL32 src)
 2297   %{
 2298     int dstenc = $dst$$reg;
 2299     if (dstenc < 8) {
 2300       emit_opcode(cbuf, Assembler::REX_W);
 2301     } else {
 2302       emit_opcode(cbuf, Assembler::REX_WB);
 2303       dstenc -= 8;
 2304     }
 2305     emit_opcode(cbuf, 0xC7);
 2306     emit_rm(cbuf, 0x03, 0x00, dstenc);
 2307     $$$emit32$src$$constant;
 2308   %}
 2309 
 2310   enc_class load_immP31(rRegP dst, immP32 src)
 2311   %{
 2312     // same as load_immI, but this time we care about zeroes in the high word
 2313     int dstenc = $dst$$reg;
 2314     if (dstenc >= 8) {
 2315       emit_opcode(cbuf, Assembler::REX_B);
 2316       dstenc -= 8;
 2317     }
 2318     emit_opcode(cbuf, 0xB8 | dstenc);
 2319     $$$emit32$src$$constant;
 2320   %}
 2321 
 2322   enc_class load_immP(rRegP dst, immP src)
 2323   %{
 2324     int dstenc = $dst$$reg;
 2325     if (dstenc < 8) {
 2326       emit_opcode(cbuf, Assembler::REX_W);
 2327     } else {
 2328       emit_opcode(cbuf, Assembler::REX_WB);
 2329       dstenc -= 8;
 2330     }
 2331     emit_opcode(cbuf, 0xB8 | dstenc);
 2332     // This next line should be generated from ADLC
 2333     if ($src->constant_reloc() != relocInfo::none) {
 2334       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
 2335     } else {
 2336       emit_d64(cbuf, $src$$constant);
 2337     }
 2338   %}
 2339 
 2340   enc_class Con32(immI src)
 2341   %{
 2342     // Output immediate
 2343     $$$emit32$src$$constant;
 2344   %}
 2345 
 2346   enc_class Con32F_as_bits(immF src)
 2347   %{
 2348     // Output Float immediate bits
 2349     jfloat jf = $src$$constant;
 2350     jint jf_as_bits = jint_cast(jf);
 2351     emit_d32(cbuf, jf_as_bits);
 2352   %}
 2353 
 2354   enc_class Con16(immI src)
 2355   %{
 2356     // Output immediate
 2357     $$$emit16$src$$constant;
 2358   %}
 2359 
 2360   // How is this different from Con32??? XXX
 2361   enc_class Con_d32(immI src)
 2362   %{
 2363     emit_d32(cbuf,$src$$constant);
 2364   %}
 2365 
 2366   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
 2367     // Output immediate memory reference
 2368     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
 2369     emit_d32(cbuf, 0x00);
 2370   %}
 2371 
 2372   enc_class lock_prefix()
 2373   %{
 2374     emit_opcode(cbuf, 0xF0); // lock
 2375   %}
 2376 
 2377   enc_class REX_mem(memory mem)
 2378   %{
 2379     if ($mem$$base >= 8) {
 2380       if ($mem$$index < 8) {
 2381         emit_opcode(cbuf, Assembler::REX_B);
 2382       } else {
 2383         emit_opcode(cbuf, Assembler::REX_XB);
 2384       }
 2385     } else {
 2386       if ($mem$$index >= 8) {
 2387         emit_opcode(cbuf, Assembler::REX_X);
 2388       }
 2389     }
 2390   %}
 2391 
 2392   enc_class REX_mem_wide(memory mem)
 2393   %{
 2394     if ($mem$$base >= 8) {
 2395       if ($mem$$index < 8) {
 2396         emit_opcode(cbuf, Assembler::REX_WB);
 2397       } else {
 2398         emit_opcode(cbuf, Assembler::REX_WXB);
 2399       }
 2400     } else {
 2401       if ($mem$$index < 8) {
 2402         emit_opcode(cbuf, Assembler::REX_W);
 2403       } else {
 2404         emit_opcode(cbuf, Assembler::REX_WX);
 2405       }
 2406     }
 2407   %}
 2408 
 2409   // for byte regs
 2410   enc_class REX_breg(rRegI reg)
 2411   %{
 2412     if ($reg$$reg >= 4) {
 2413       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2414     }
 2415   %}
 2416 
 2417   // for byte regs
 2418   enc_class REX_reg_breg(rRegI dst, rRegI src)
 2419   %{
 2420     if ($dst$$reg < 8) {
 2421       if ($src$$reg >= 4) {
 2422         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
 2423       }
 2424     } else {
 2425       if ($src$$reg < 8) {
 2426         emit_opcode(cbuf, Assembler::REX_R);
 2427       } else {
 2428         emit_opcode(cbuf, Assembler::REX_RB);
 2429       }
 2430     }
 2431   %}
 2432 
 2433   // for byte regs
 2434   enc_class REX_breg_mem(rRegI reg, memory mem)
 2435   %{
 2436     if ($reg$$reg < 8) {
 2437       if ($mem$$base < 8) {
 2438         if ($mem$$index >= 8) {
 2439           emit_opcode(cbuf, Assembler::REX_X);
 2440         } else if ($reg$$reg >= 4) {
 2441           emit_opcode(cbuf, Assembler::REX);
 2442         }
 2443       } else {
 2444         if ($mem$$index < 8) {
 2445           emit_opcode(cbuf, Assembler::REX_B);
 2446         } else {
 2447           emit_opcode(cbuf, Assembler::REX_XB);
 2448         }
 2449       }
 2450     } else {
 2451       if ($mem$$base < 8) {
 2452         if ($mem$$index < 8) {
 2453           emit_opcode(cbuf, Assembler::REX_R);
 2454         } else {
 2455           emit_opcode(cbuf, Assembler::REX_RX);
 2456         }
 2457       } else {
 2458         if ($mem$$index < 8) {
 2459           emit_opcode(cbuf, Assembler::REX_RB);
 2460         } else {
 2461           emit_opcode(cbuf, Assembler::REX_RXB);
 2462         }
 2463       }
 2464     }
 2465   %}
 2466 
 2467   enc_class REX_reg(rRegI reg)
 2468   %{
 2469     if ($reg$$reg >= 8) {
 2470       emit_opcode(cbuf, Assembler::REX_B);
 2471     }
 2472   %}
 2473 
 2474   enc_class REX_reg_wide(rRegI reg)
 2475   %{
 2476     if ($reg$$reg < 8) {
 2477       emit_opcode(cbuf, Assembler::REX_W);
 2478     } else {
 2479       emit_opcode(cbuf, Assembler::REX_WB);
 2480     }
 2481   %}
 2482 
 2483   enc_class REX_reg_reg(rRegI dst, rRegI src)
 2484   %{
 2485     if ($dst$$reg < 8) {
 2486       if ($src$$reg >= 8) {
 2487         emit_opcode(cbuf, Assembler::REX_B);
 2488       }
 2489     } else {
 2490       if ($src$$reg < 8) {
 2491         emit_opcode(cbuf, Assembler::REX_R);
 2492       } else {
 2493         emit_opcode(cbuf, Assembler::REX_RB);
 2494       }
 2495     }
 2496   %}
 2497 
 2498   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
 2499   %{
 2500     if ($dst$$reg < 8) {
 2501       if ($src$$reg < 8) {
 2502         emit_opcode(cbuf, Assembler::REX_W);
 2503       } else {
 2504         emit_opcode(cbuf, Assembler::REX_WB);
 2505       }
 2506     } else {
 2507       if ($src$$reg < 8) {
 2508         emit_opcode(cbuf, Assembler::REX_WR);
 2509       } else {
 2510         emit_opcode(cbuf, Assembler::REX_WRB);
 2511       }
 2512     }
 2513   %}
 2514 
 2515   enc_class REX_reg_mem(rRegI reg, memory mem)
 2516   %{
 2517     if ($reg$$reg < 8) {
 2518       if ($mem$$base < 8) {
 2519         if ($mem$$index >= 8) {
 2520           emit_opcode(cbuf, Assembler::REX_X);
 2521         }
 2522       } else {
 2523         if ($mem$$index < 8) {
 2524           emit_opcode(cbuf, Assembler::REX_B);
 2525         } else {
 2526           emit_opcode(cbuf, Assembler::REX_XB);
 2527         }
 2528       }
 2529     } else {
 2530       if ($mem$$base < 8) {
 2531         if ($mem$$index < 8) {
 2532           emit_opcode(cbuf, Assembler::REX_R);
 2533         } else {
 2534           emit_opcode(cbuf, Assembler::REX_RX);
 2535         }
 2536       } else {
 2537         if ($mem$$index < 8) {
 2538           emit_opcode(cbuf, Assembler::REX_RB);
 2539         } else {
 2540           emit_opcode(cbuf, Assembler::REX_RXB);
 2541         }
 2542       }
 2543     }
 2544   %}
 2545 
 2546   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
 2547   %{
 2548     if ($reg$$reg < 8) {
 2549       if ($mem$$base < 8) {
 2550         if ($mem$$index < 8) {
 2551           emit_opcode(cbuf, Assembler::REX_W);
 2552         } else {
 2553           emit_opcode(cbuf, Assembler::REX_WX);
 2554         }
 2555       } else {
 2556         if ($mem$$index < 8) {
 2557           emit_opcode(cbuf, Assembler::REX_WB);
 2558         } else {
 2559           emit_opcode(cbuf, Assembler::REX_WXB);
 2560         }
 2561       }
 2562     } else {
 2563       if ($mem$$base < 8) {
 2564         if ($mem$$index < 8) {
 2565           emit_opcode(cbuf, Assembler::REX_WR);
 2566         } else {
 2567           emit_opcode(cbuf, Assembler::REX_WRX);
 2568         }
 2569       } else {
 2570         if ($mem$$index < 8) {
 2571           emit_opcode(cbuf, Assembler::REX_WRB);
 2572         } else {
 2573           emit_opcode(cbuf, Assembler::REX_WRXB);
 2574         }
 2575       }
 2576     }
 2577   %}
 2578 
 2579   enc_class reg_mem(rRegI ereg, memory mem)
 2580   %{
 2581     // High registers handle in encode_RegMem
 2582     int reg = $ereg$$reg;
 2583     int base = $mem$$base;
 2584     int index = $mem$$index;
 2585     int scale = $mem$$scale;
 2586     int disp = $mem$$disp;
 2587     relocInfo::relocType disp_reloc = $mem->disp_reloc();
 2588 
 2589     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
 2590   %}
 2591 
 2592   enc_class RM_opc_mem(immI rm_opcode, memory mem)
 2593   %{
 2594     int rm_byte_opcode = $rm_opcode$$constant;
 2595 
 2596     // High registers handle in encode_RegMem
 2597     int base = $mem$$base;
 2598     int index = $mem$$index;
 2599     int scale = $mem$$scale;
 2600     int displace = $mem$$disp;
 2601 
 2602     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
 2603                                             // working with static
 2604                                             // globals
 2605     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
 2606                   disp_reloc);
 2607   %}
 2608 
 2609   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
 2610   %{
 2611     int reg_encoding = $dst$$reg;
 2612     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
 2613     int index        = 0x04;            // 0x04 indicates no index
 2614     int scale        = 0x00;            // 0x00 indicates no scale
 2615     int displace     = $src1$$constant; // 0x00 indicates no displacement
 2616     relocInfo::relocType disp_reloc = relocInfo::none;
 2617     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
 2618                   disp_reloc);
 2619   %}
 2620 
 2621   enc_class neg_reg(rRegI dst)
 2622   %{
 2623     int dstenc = $dst$$reg;
 2624     if (dstenc >= 8) {
 2625       emit_opcode(cbuf, Assembler::REX_B);
 2626       dstenc -= 8;
 2627     }
 2628     // NEG $dst
 2629     emit_opcode(cbuf, 0xF7);
 2630     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2631   %}
 2632 
 2633   enc_class neg_reg_wide(rRegI dst)
 2634   %{
 2635     int dstenc = $dst$$reg;
 2636     if (dstenc < 8) {
 2637       emit_opcode(cbuf, Assembler::REX_W);
 2638     } else {
 2639       emit_opcode(cbuf, Assembler::REX_WB);
 2640       dstenc -= 8;
 2641     }
 2642     // NEG $dst
 2643     emit_opcode(cbuf, 0xF7);
 2644     emit_rm(cbuf, 0x3, 0x03, dstenc);
 2645   %}
 2646 
 2647   enc_class setLT_reg(rRegI dst)
 2648   %{
 2649     int dstenc = $dst$$reg;
 2650     if (dstenc >= 8) {
 2651       emit_opcode(cbuf, Assembler::REX_B);
 2652       dstenc -= 8;
 2653     } else if (dstenc >= 4) {
 2654       emit_opcode(cbuf, Assembler::REX);
 2655     }
 2656     // SETLT $dst
 2657     emit_opcode(cbuf, 0x0F);
 2658     emit_opcode(cbuf, 0x9C);
 2659     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2660   %}
 2661 
 2662   enc_class setNZ_reg(rRegI dst)
 2663   %{
 2664     int dstenc = $dst$$reg;
 2665     if (dstenc >= 8) {
 2666       emit_opcode(cbuf, Assembler::REX_B);
 2667       dstenc -= 8;
 2668     } else if (dstenc >= 4) {
 2669       emit_opcode(cbuf, Assembler::REX);
 2670     }
 2671     // SETNZ $dst
 2672     emit_opcode(cbuf, 0x0F);
 2673     emit_opcode(cbuf, 0x95);
 2674     emit_rm(cbuf, 0x3, 0x0, dstenc);
 2675   %}
 2676 
 2677 
 2678   // Compare the lonogs and set -1, 0, or 1 into dst
 2679   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
 2680   %{
 2681     int src1enc = $src1$$reg;
 2682     int src2enc = $src2$$reg;
 2683     int dstenc = $dst$$reg;
 2684 
 2685     // cmpq $src1, $src2
 2686     if (src1enc < 8) {
 2687       if (src2enc < 8) {
 2688         emit_opcode(cbuf, Assembler::REX_W);
 2689       } else {
 2690         emit_opcode(cbuf, Assembler::REX_WB);
 2691       }
 2692     } else {
 2693       if (src2enc < 8) {
 2694         emit_opcode(cbuf, Assembler::REX_WR);
 2695       } else {
 2696         emit_opcode(cbuf, Assembler::REX_WRB);
 2697       }
 2698     }
 2699     emit_opcode(cbuf, 0x3B);
 2700     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
 2701 
 2702     // movl $dst, -1
 2703     if (dstenc >= 8) {
 2704       emit_opcode(cbuf, Assembler::REX_B);
 2705     }
 2706     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
 2707     emit_d32(cbuf, -1);
 2708 
 2709     // jl,s done
 2710     emit_opcode(cbuf, 0x7C);
 2711     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
 2712 
 2713     // setne $dst
 2714     if (dstenc >= 4) {
 2715       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
 2716     }
 2717     emit_opcode(cbuf, 0x0F);
 2718     emit_opcode(cbuf, 0x95);
 2719     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
 2720 
 2721     // movzbl $dst, $dst
 2722     if (dstenc >= 4) {
 2723       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
 2724     }
 2725     emit_opcode(cbuf, 0x0F);
 2726     emit_opcode(cbuf, 0xB6);
 2727     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 2728   %}
 2729 
 2730   enc_class Push_ResultXD(regD dst) %{
 2731     MacroAssembler _masm(&cbuf);
 2732     __ fstp_d(Address(rsp, 0));
 2733     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
 2734     __ addptr(rsp, 8);
 2735   %}
 2736 
 2737   enc_class Push_SrcXD(regD src) %{
 2738     MacroAssembler _masm(&cbuf);
 2739     __ subptr(rsp, 8);
 2740     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
 2741     __ fld_d(Address(rsp, 0));
 2742   %}
 2743 
 2744 
 2745   enc_class enc_rethrow()
 2746   %{
 2747     cbuf.set_insts_mark();
 2748     emit_opcode(cbuf, 0xE9); // jmp entry
 2749     emit_d32_reloc(cbuf,
 2750                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 2751                    runtime_call_Relocation::spec(),
 2752                    RELOC_DISP32);
 2753   %}
 2754 
 2755 %}
 2756 
 2757 
 2758 
 2759 //----------FRAME--------------------------------------------------------------
 2760 // Definition of frame structure and management information.
 2761 //
 2762 //  S T A C K   L A Y O U T    Allocators stack-slot number
 2763 //                             |   (to get allocators register number
 2764 //  G  Owned by    |        |  v    add OptoReg::stack0())
 2765 //  r   CALLER     |        |
 2766 //  o     |        +--------+      pad to even-align allocators stack-slot
 2767 //  w     V        |  pad0  |        numbers; owned by CALLER
 2768 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 2769 //  h     ^        |   in   |  5
 2770 //        |        |  args  |  4   Holes in incoming args owned by SELF
 2771 //  |     |        |        |  3
 2772 //  |     |        +--------+
 2773 //  V     |        | old out|      Empty on Intel, window on Sparc
 2774 //        |    old |preserve|      Must be even aligned.
 2775 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 2776 //        |        |   in   |  3   area for Intel ret address
 2777 //     Owned by    |preserve|      Empty on Sparc.
 2778 //       SELF      +--------+
 2779 //        |        |  pad2  |  2   pad to align old SP
 2780 //        |        +--------+  1
 2781 //        |        | locks  |  0
 2782 //        |        +--------+----> OptoReg::stack0(), even aligned
 2783 //        |        |  pad1  | 11   pad to align new SP
 2784 //        |        +--------+
 2785 //        |        |        | 10
 2786 //        |        | spills |  9   spills
 2787 //        V        |        |  8   (pad0 slot for callee)
 2788 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 2789 //        ^        |  out   |  7
 2790 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 2791 //     Owned by    +--------+
 2792 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 2793 //        |    new |preserve|      Must be even-aligned.
 2794 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 2795 //        |        |        |
 2796 //
 2797 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 2798 //         known from SELF's arguments and the Java calling convention.
 2799 //         Region 6-7 is determined per call site.
 2800 // Note 2: If the calling convention leaves holes in the incoming argument
 2801 //         area, those holes are owned by SELF.  Holes in the outgoing area
 2802 //         are owned by the CALLEE.  Holes should not be nessecary in the
 2803 //         incoming area, as the Java calling convention is completely under
 2804 //         the control of the AD file.  Doubles can be sorted and packed to
 2805 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
 2806 //         varargs C calling conventions.
 2807 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 2808 //         even aligned with pad0 as needed.
 2809 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 2810 //         region 6-11 is even aligned; it may be padded out more so that
 2811 //         the region from SP to FP meets the minimum stack alignment.
 2812 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 2813 //         alignment.  Region 11, pad1, may be dynamically extended so that
 2814 //         SP meets the minimum alignment.
 2815 
 2816 frame
 2817 %{
 2818   // These three registers define part of the calling convention
 2819   // between compiled code and the interpreter.
 2820   inline_cache_reg(RAX);                // Inline Cache Register
 2821 
 2822   // Optional: name the operand used by cisc-spilling to access
 2823   // [stack_pointer + offset]
 2824   cisc_spilling_operand_name(indOffset32);
 2825 
 2826   // Number of stack slots consumed by locking an object
 2827   sync_stack_slots(2);
 2828 
 2829   // Compiled code's Frame Pointer
 2830   frame_pointer(RSP);
 2831 
 2832   // Interpreter stores its frame pointer in a register which is
 2833   // stored to the stack by I2CAdaptors.
 2834   // I2CAdaptors convert from interpreted java to compiled java.
 2835   interpreter_frame_pointer(RBP);
 2836 
 2837   // Stack alignment requirement
 2838   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 2839 
 2840   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 2841   // for calls to C.  Supports the var-args backing area for register parms.
 2842   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 2843 
 2844   // The after-PROLOG location of the return address.  Location of
 2845   // return address specifies a type (REG or STACK) and a number
 2846   // representing the register number (i.e. - use a register name) or
 2847   // stack slot.
 2848   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2849   // Otherwise, it is above the locks and verification slot and alignment word
 2850   return_addr(STACK - 2 +
 2851               align_up((Compile::current()->in_preserve_stack_slots() +
 2852                         Compile::current()->fixed_slots()),
 2853                        stack_alignment_in_slots()));
 2854 
 2855   // Location of compiled Java return values.  Same as C for now.
 2856   return_value
 2857   %{
 2858     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2859            "only return normal values");
 2860 
 2861     static const int lo[Op_RegL + 1] = {
 2862       0,
 2863       0,
 2864       RAX_num,  // Op_RegN
 2865       RAX_num,  // Op_RegI
 2866       RAX_num,  // Op_RegP
 2867       XMM0_num, // Op_RegF
 2868       XMM0_num, // Op_RegD
 2869       RAX_num   // Op_RegL
 2870     };
 2871     static const int hi[Op_RegL + 1] = {
 2872       0,
 2873       0,
 2874       OptoReg::Bad, // Op_RegN
 2875       OptoReg::Bad, // Op_RegI
 2876       RAX_H_num,    // Op_RegP
 2877       OptoReg::Bad, // Op_RegF
 2878       XMM0b_num,    // Op_RegD
 2879       RAX_H_num     // Op_RegL
 2880     };
 2881     // Excluded flags and vector registers.
 2882     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2883     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2884   %}
 2885 %}
 2886 
 2887 //----------ATTRIBUTES---------------------------------------------------------
 2888 //----------Operand Attributes-------------------------------------------------
 2889 op_attrib op_cost(0);        // Required cost attribute
 2890 
 2891 //----------Instruction Attributes---------------------------------------------
 2892 ins_attrib ins_cost(100);       // Required cost attribute
 2893 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2894 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2895                                 // a non-matching short branch variant
 2896                                 // of some long branch?
 2897 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2898                                 // be a power of 2) specifies the
 2899                                 // alignment that some part of the
 2900                                 // instruction (not necessarily the
 2901                                 // start) requires.  If > 1, a
 2902                                 // compute_padding() function must be
 2903                                 // provided for the instruction
 2904 
 2905 //----------OPERANDS-----------------------------------------------------------
 2906 // Operand definitions must precede instruction definitions for correct parsing
 2907 // in the ADLC because operands constitute user defined types which are used in
 2908 // instruction definitions.
 2909 
 2910 //----------Simple Operands----------------------------------------------------
 2911 // Immediate Operands
 2912 // Integer Immediate
 2913 operand immI()
 2914 %{
 2915   match(ConI);
 2916 
 2917   op_cost(10);
 2918   format %{ %}
 2919   interface(CONST_INTER);
 2920 %}
 2921 
 2922 // Constant for test vs zero
 2923 operand immI_0()
 2924 %{
 2925   predicate(n->get_int() == 0);
 2926   match(ConI);
 2927 
 2928   op_cost(0);
 2929   format %{ %}
 2930   interface(CONST_INTER);
 2931 %}
 2932 
 2933 // Constant for increment
 2934 operand immI_1()
 2935 %{
 2936   predicate(n->get_int() == 1);
 2937   match(ConI);
 2938 
 2939   op_cost(0);
 2940   format %{ %}
 2941   interface(CONST_INTER);
 2942 %}
 2943 
 2944 // Constant for decrement
 2945 operand immI_M1()
 2946 %{
 2947   predicate(n->get_int() == -1);
 2948   match(ConI);
 2949 
 2950   op_cost(0);
 2951   format %{ %}
 2952   interface(CONST_INTER);
 2953 %}
 2954 
 2955 operand immI_2()
 2956 %{
 2957   predicate(n->get_int() == 2);
 2958   match(ConI);
 2959 
 2960   op_cost(0);
 2961   format %{ %}
 2962   interface(CONST_INTER);
 2963 %}
 2964 
 2965 operand immI_4()
 2966 %{
 2967   predicate(n->get_int() == 4);
 2968   match(ConI);
 2969 
 2970   op_cost(0);
 2971   format %{ %}
 2972   interface(CONST_INTER);
 2973 %}
 2974 
 2975 operand immI_8()
 2976 %{
 2977   predicate(n->get_int() == 8);
 2978   match(ConI);
 2979 
 2980   op_cost(0);
 2981   format %{ %}
 2982   interface(CONST_INTER);
 2983 %}
 2984 
 2985 // Valid scale values for addressing modes
 2986 operand immI2()
 2987 %{
 2988   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2989   match(ConI);
 2990 
 2991   format %{ %}
 2992   interface(CONST_INTER);
 2993 %}
 2994 
 2995 operand immU7()
 2996 %{
 2997   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2998   match(ConI);
 2999 
 3000   op_cost(5);
 3001   format %{ %}
 3002   interface(CONST_INTER);
 3003 %}
 3004 
 3005 operand immI8()
 3006 %{
 3007   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 3008   match(ConI);
 3009 
 3010   op_cost(5);
 3011   format %{ %}
 3012   interface(CONST_INTER);
 3013 %}
 3014 
 3015 operand immU8()
 3016 %{
 3017   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 3018   match(ConI);
 3019 
 3020   op_cost(5);
 3021   format %{ %}
 3022   interface(CONST_INTER);
 3023 %}
 3024 
 3025 operand immI16()
 3026 %{
 3027   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 3028   match(ConI);
 3029 
 3030   op_cost(10);
 3031   format %{ %}
 3032   interface(CONST_INTER);
 3033 %}
 3034 
 3035 // Int Immediate non-negative
 3036 operand immU31()
 3037 %{
 3038   predicate(n->get_int() >= 0);
 3039   match(ConI);
 3040 
 3041   op_cost(0);
 3042   format %{ %}
 3043   interface(CONST_INTER);
 3044 %}
 3045 
 3046 // Constant for long shifts
 3047 operand immI_32()
 3048 %{
 3049   predicate( n->get_int() == 32 );
 3050   match(ConI);
 3051 
 3052   op_cost(0);
 3053   format %{ %}
 3054   interface(CONST_INTER);
 3055 %}
 3056 
 3057 // Constant for long shifts
 3058 operand immI_64()
 3059 %{
 3060   predicate( n->get_int() == 64 );
 3061   match(ConI);
 3062 
 3063   op_cost(0);
 3064   format %{ %}
 3065   interface(CONST_INTER);
 3066 %}
 3067 
 3068 // Pointer Immediate
 3069 operand immP()
 3070 %{
 3071   match(ConP);
 3072 
 3073   op_cost(10);
 3074   format %{ %}
 3075   interface(CONST_INTER);
 3076 %}
 3077 
 3078 // NULL Pointer Immediate
 3079 operand immP0()
 3080 %{
 3081   predicate(n->get_ptr() == 0);
 3082   match(ConP);
 3083 
 3084   op_cost(5);
 3085   format %{ %}
 3086   interface(CONST_INTER);
 3087 %}
 3088 
 3089 // Pointer Immediate
 3090 operand immN() %{
 3091   match(ConN);
 3092 
 3093   op_cost(10);
 3094   format %{ %}
 3095   interface(CONST_INTER);
 3096 %}
 3097 
 3098 operand immNKlass() %{
 3099   match(ConNKlass);
 3100 
 3101   op_cost(10);
 3102   format %{ %}
 3103   interface(CONST_INTER);
 3104 %}
 3105 
 3106 // NULL Pointer Immediate
 3107 operand immN0() %{
 3108   predicate(n->get_narrowcon() == 0);
 3109   match(ConN);
 3110 
 3111   op_cost(5);
 3112   format %{ %}
 3113   interface(CONST_INTER);
 3114 %}
 3115 
 3116 operand immP31()
 3117 %{
 3118   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 3119             && (n->get_ptr() >> 31) == 0);
 3120   match(ConP);
 3121 
 3122   op_cost(5);
 3123   format %{ %}
 3124   interface(CONST_INTER);
 3125 %}
 3126 
 3127 
 3128 // Long Immediate
 3129 operand immL()
 3130 %{
 3131   match(ConL);
 3132 
 3133   op_cost(20);
 3134   format %{ %}
 3135   interface(CONST_INTER);
 3136 %}
 3137 
 3138 // Long Immediate 8-bit
 3139 operand immL8()
 3140 %{
 3141   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 3142   match(ConL);
 3143 
 3144   op_cost(5);
 3145   format %{ %}
 3146   interface(CONST_INTER);
 3147 %}
 3148 
 3149 // Long Immediate 32-bit unsigned
 3150 operand immUL32()
 3151 %{
 3152   predicate(n->get_long() == (unsigned int) (n->get_long()));
 3153   match(ConL);
 3154 
 3155   op_cost(10);
 3156   format %{ %}
 3157   interface(CONST_INTER);
 3158 %}
 3159 
 3160 // Long Immediate 32-bit signed
 3161 operand immL32()
 3162 %{
 3163   predicate(n->get_long() == (int) (n->get_long()));
 3164   match(ConL);
 3165 
 3166   op_cost(15);
 3167   format %{ %}
 3168   interface(CONST_INTER);
 3169 %}
 3170 
 3171 operand immL_Pow2()
 3172 %{
 3173   predicate(is_power_of_2((julong)n->get_long()));
 3174   match(ConL);
 3175 
 3176   op_cost(15);
 3177   format %{ %}
 3178   interface(CONST_INTER);
 3179 %}
 3180 
 3181 operand immL_NotPow2()
 3182 %{
 3183   predicate(is_power_of_2((julong)~n->get_long()));
 3184   match(ConL);
 3185 
 3186   op_cost(15);
 3187   format %{ %}
 3188   interface(CONST_INTER);
 3189 %}
 3190 
 3191 // Long Immediate zero
 3192 operand immL0()
 3193 %{
 3194   predicate(n->get_long() == 0L);
 3195   match(ConL);
 3196 
 3197   op_cost(10);
 3198   format %{ %}
 3199   interface(CONST_INTER);
 3200 %}
 3201 
 3202 // Constant for increment
 3203 operand immL1()
 3204 %{
 3205   predicate(n->get_long() == 1);
 3206   match(ConL);
 3207 
 3208   format %{ %}
 3209   interface(CONST_INTER);
 3210 %}
 3211 
 3212 // Constant for decrement
 3213 operand immL_M1()
 3214 %{
 3215   predicate(n->get_long() == -1);
 3216   match(ConL);
 3217 
 3218   format %{ %}
 3219   interface(CONST_INTER);
 3220 %}
 3221 
 3222 // Long Immediate: the value 10
 3223 operand immL10()
 3224 %{
 3225   predicate(n->get_long() == 10);
 3226   match(ConL);
 3227 
 3228   format %{ %}
 3229   interface(CONST_INTER);
 3230 %}
 3231 
 3232 // Long immediate from 0 to 127.
 3233 // Used for a shorter form of long mul by 10.
 3234 operand immL_127()
 3235 %{
 3236   predicate(0 <= n->get_long() && n->get_long() < 0x80);
 3237   match(ConL);
 3238 
 3239   op_cost(10);
 3240   format %{ %}
 3241   interface(CONST_INTER);
 3242 %}
 3243 
 3244 // Long Immediate: low 32-bit mask
 3245 operand immL_32bits()
 3246 %{
 3247   predicate(n->get_long() == 0xFFFFFFFFL);
 3248   match(ConL);
 3249   op_cost(20);
 3250 
 3251   format %{ %}
 3252   interface(CONST_INTER);
 3253 %}
 3254 
 3255 // Int Immediate: 2^n-1, postive
 3256 operand immI_Pow2M1()
 3257 %{
 3258   predicate((n->get_int() > 0)
 3259             && is_power_of_2(n->get_int() + 1));
 3260   match(ConI);
 3261 
 3262   op_cost(20);
 3263   format %{ %}
 3264   interface(CONST_INTER);
 3265 %}
 3266 
 3267 // Float Immediate zero
 3268 operand immF0()
 3269 %{
 3270   predicate(jint_cast(n->getf()) == 0);
 3271   match(ConF);
 3272 
 3273   op_cost(5);
 3274   format %{ %}
 3275   interface(CONST_INTER);
 3276 %}
 3277 
 3278 // Float Immediate
 3279 operand immF()
 3280 %{
 3281   match(ConF);
 3282 
 3283   op_cost(15);
 3284   format %{ %}
 3285   interface(CONST_INTER);
 3286 %}
 3287 
 3288 // Double Immediate zero
 3289 operand immD0()
 3290 %{
 3291   predicate(jlong_cast(n->getd()) == 0);
 3292   match(ConD);
 3293 
 3294   op_cost(5);
 3295   format %{ %}
 3296   interface(CONST_INTER);
 3297 %}
 3298 
 3299 // Double Immediate
 3300 operand immD()
 3301 %{
 3302   match(ConD);
 3303 
 3304   op_cost(15);
 3305   format %{ %}
 3306   interface(CONST_INTER);
 3307 %}
 3308 
 3309 // Immediates for special shifts (sign extend)
 3310 
 3311 // Constants for increment
 3312 operand immI_16()
 3313 %{
 3314   predicate(n->get_int() == 16);
 3315   match(ConI);
 3316 
 3317   format %{ %}
 3318   interface(CONST_INTER);
 3319 %}
 3320 
 3321 operand immI_24()
 3322 %{
 3323   predicate(n->get_int() == 24);
 3324   match(ConI);
 3325 
 3326   format %{ %}
 3327   interface(CONST_INTER);
 3328 %}
 3329 
 3330 // Constant for byte-wide masking
 3331 operand immI_255()
 3332 %{
 3333   predicate(n->get_int() == 255);
 3334   match(ConI);
 3335 
 3336   format %{ %}
 3337   interface(CONST_INTER);
 3338 %}
 3339 
 3340 // Constant for short-wide masking
 3341 operand immI_65535()
 3342 %{
 3343   predicate(n->get_int() == 65535);
 3344   match(ConI);
 3345 
 3346   format %{ %}
 3347   interface(CONST_INTER);
 3348 %}
 3349 
 3350 // Constant for byte-wide masking
 3351 operand immL_255()
 3352 %{
 3353   predicate(n->get_long() == 255);
 3354   match(ConL);
 3355 
 3356   format %{ %}
 3357   interface(CONST_INTER);
 3358 %}
 3359 
 3360 // Constant for short-wide masking
 3361 operand immL_65535()
 3362 %{
 3363   predicate(n->get_long() == 65535);
 3364   match(ConL);
 3365 
 3366   format %{ %}
 3367   interface(CONST_INTER);
 3368 %}
 3369 
 3370 operand kReg()
 3371 %{
 3372   constraint(ALLOC_IN_RC(vectmask_reg));
 3373   match(RegVectMask);
 3374   format %{%}
 3375   interface(REG_INTER);
 3376 %}
 3377 
 3378 operand kReg_K1()
 3379 %{
 3380   constraint(ALLOC_IN_RC(vectmask_reg_K1));
 3381   match(RegVectMask);
 3382   format %{%}
 3383   interface(REG_INTER);
 3384 %}
 3385 
 3386 operand kReg_K2()
 3387 %{
 3388   constraint(ALLOC_IN_RC(vectmask_reg_K2));
 3389   match(RegVectMask);
 3390   format %{%}
 3391   interface(REG_INTER);
 3392 %}
 3393 
 3394 // Special Registers
 3395 operand kReg_K3()
 3396 %{
 3397   constraint(ALLOC_IN_RC(vectmask_reg_K3));
 3398   match(RegVectMask);
 3399   format %{%}
 3400   interface(REG_INTER);
 3401 %}
 3402 
 3403 operand kReg_K4()
 3404 %{
 3405   constraint(ALLOC_IN_RC(vectmask_reg_K4));
 3406   match(RegVectMask);
 3407   format %{%}
 3408   interface(REG_INTER);
 3409 %}
 3410 
 3411 operand kReg_K5()
 3412 %{
 3413   constraint(ALLOC_IN_RC(vectmask_reg_K5));
 3414   match(RegVectMask);
 3415   format %{%}
 3416   interface(REG_INTER);
 3417 %}
 3418 
 3419 operand kReg_K6()
 3420 %{
 3421   constraint(ALLOC_IN_RC(vectmask_reg_K6));
 3422   match(RegVectMask);
 3423   format %{%}
 3424   interface(REG_INTER);
 3425 %}
 3426 
 3427 // Special Registers
 3428 operand kReg_K7()
 3429 %{
 3430   constraint(ALLOC_IN_RC(vectmask_reg_K7));
 3431   match(RegVectMask);
 3432   format %{%}
 3433   interface(REG_INTER);
 3434 %}
 3435 
 3436 // Register Operands
 3437 // Integer Register
 3438 operand rRegI()
 3439 %{
 3440   constraint(ALLOC_IN_RC(int_reg));
 3441   match(RegI);
 3442 
 3443   match(rax_RegI);
 3444   match(rbx_RegI);
 3445   match(rcx_RegI);
 3446   match(rdx_RegI);
 3447   match(rdi_RegI);
 3448 
 3449   format %{ %}
 3450   interface(REG_INTER);
 3451 %}
 3452 
 3453 // Special Registers
 3454 operand rax_RegI()
 3455 %{
 3456   constraint(ALLOC_IN_RC(int_rax_reg));
 3457   match(RegI);
 3458   match(rRegI);
 3459 
 3460   format %{ "RAX" %}
 3461   interface(REG_INTER);
 3462 %}
 3463 
 3464 // Special Registers
 3465 operand rbx_RegI()
 3466 %{
 3467   constraint(ALLOC_IN_RC(int_rbx_reg));
 3468   match(RegI);
 3469   match(rRegI);
 3470 
 3471   format %{ "RBX" %}
 3472   interface(REG_INTER);
 3473 %}
 3474 
 3475 operand rcx_RegI()
 3476 %{
 3477   constraint(ALLOC_IN_RC(int_rcx_reg));
 3478   match(RegI);
 3479   match(rRegI);
 3480 
 3481   format %{ "RCX" %}
 3482   interface(REG_INTER);
 3483 %}
 3484 
 3485 operand rdx_RegI()
 3486 %{
 3487   constraint(ALLOC_IN_RC(int_rdx_reg));
 3488   match(RegI);
 3489   match(rRegI);
 3490 
 3491   format %{ "RDX" %}
 3492   interface(REG_INTER);
 3493 %}
 3494 
 3495 operand rdi_RegI()
 3496 %{
 3497   constraint(ALLOC_IN_RC(int_rdi_reg));
 3498   match(RegI);
 3499   match(rRegI);
 3500 
 3501   format %{ "RDI" %}
 3502   interface(REG_INTER);
 3503 %}
 3504 
 3505 operand no_rax_rdx_RegI()
 3506 %{
 3507   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 3508   match(RegI);
 3509   match(rbx_RegI);
 3510   match(rcx_RegI);
 3511   match(rdi_RegI);
 3512 
 3513   format %{ %}
 3514   interface(REG_INTER);
 3515 %}
 3516 
 3517 operand no_rbp_r13_RegI()
 3518 %{
 3519   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 3520   match(RegI);
 3521   match(rRegI);
 3522   match(rax_RegI);
 3523   match(rbx_RegI);
 3524   match(rcx_RegI);
 3525   match(rdx_RegI);
 3526   match(rdi_RegI);
 3527 
 3528   format %{ %}
 3529   interface(REG_INTER);
 3530 %}
 3531 
 3532 // Pointer Register
 3533 operand any_RegP()
 3534 %{
 3535   constraint(ALLOC_IN_RC(any_reg));
 3536   match(RegP);
 3537   match(rax_RegP);
 3538   match(rbx_RegP);
 3539   match(rdi_RegP);
 3540   match(rsi_RegP);
 3541   match(rbp_RegP);
 3542   match(r15_RegP);
 3543   match(rRegP);
 3544 
 3545   format %{ %}
 3546   interface(REG_INTER);
 3547 %}
 3548 
 3549 operand rRegP()
 3550 %{
 3551   constraint(ALLOC_IN_RC(ptr_reg));
 3552   match(RegP);
 3553   match(rax_RegP);
 3554   match(rbx_RegP);
 3555   match(rdi_RegP);
 3556   match(rsi_RegP);
 3557   match(rbp_RegP);  // See Q&A below about
 3558   match(r15_RegP);  // r15_RegP and rbp_RegP.
 3559 
 3560   format %{ %}
 3561   interface(REG_INTER);
 3562 %}
 3563 
 3564 operand rRegN() %{
 3565   constraint(ALLOC_IN_RC(int_reg));
 3566   match(RegN);
 3567 
 3568   format %{ %}
 3569   interface(REG_INTER);
 3570 %}
 3571 
 3572 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 3573 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 3574 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 3575 // The output of an instruction is controlled by the allocator, which respects
 3576 // register class masks, not match rules.  Unless an instruction mentions
 3577 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 3578 // by the allocator as an input.
 3579 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 3580 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 3581 // result, RBP is not included in the output of the instruction either.
 3582 
 3583 operand no_rax_RegP()
 3584 %{
 3585   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
 3586   match(RegP);
 3587   match(rbx_RegP);
 3588   match(rsi_RegP);
 3589   match(rdi_RegP);
 3590 
 3591   format %{ %}
 3592   interface(REG_INTER);
 3593 %}
 3594 
 3595 // This operand is not allowed to use RBP even if
 3596 // RBP is not used to hold the frame pointer.
 3597 operand no_rbp_RegP()
 3598 %{
 3599   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 3600   match(RegP);
 3601   match(rbx_RegP);
 3602   match(rsi_RegP);
 3603   match(rdi_RegP);
 3604 
 3605   format %{ %}
 3606   interface(REG_INTER);
 3607 %}
 3608 
 3609 operand no_rax_rbx_RegP()
 3610 %{
 3611   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
 3612   match(RegP);
 3613   match(rsi_RegP);
 3614   match(rdi_RegP);
 3615 
 3616   format %{ %}
 3617   interface(REG_INTER);
 3618 %}
 3619 
 3620 // Special Registers
 3621 // Return a pointer value
 3622 operand rax_RegP()
 3623 %{
 3624   constraint(ALLOC_IN_RC(ptr_rax_reg));
 3625   match(RegP);
 3626   match(rRegP);
 3627 
 3628   format %{ %}
 3629   interface(REG_INTER);
 3630 %}
 3631 
 3632 // Special Registers
 3633 // Return a compressed pointer value
 3634 operand rax_RegN()
 3635 %{
 3636   constraint(ALLOC_IN_RC(int_rax_reg));
 3637   match(RegN);
 3638   match(rRegN);
 3639 
 3640   format %{ %}
 3641   interface(REG_INTER);
 3642 %}
 3643 
 3644 // Used in AtomicAdd
 3645 operand rbx_RegP()
 3646 %{
 3647   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 3648   match(RegP);
 3649   match(rRegP);
 3650 
 3651   format %{ %}
 3652   interface(REG_INTER);
 3653 %}
 3654 
 3655 operand rsi_RegP()
 3656 %{
 3657   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 3658   match(RegP);
 3659   match(rRegP);
 3660 
 3661   format %{ %}
 3662   interface(REG_INTER);
 3663 %}
 3664 
 3665 operand rbp_RegP()
 3666 %{
 3667   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 3668   match(RegP);
 3669   match(rRegP);
 3670 
 3671   format %{ %}
 3672   interface(REG_INTER);
 3673 %}
 3674 
 3675 // Used in rep stosq
 3676 operand rdi_RegP()
 3677 %{
 3678   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 3679   match(RegP);
 3680   match(rRegP);
 3681 
 3682   format %{ %}
 3683   interface(REG_INTER);
 3684 %}
 3685 
 3686 operand r15_RegP()
 3687 %{
 3688   constraint(ALLOC_IN_RC(ptr_r15_reg));
 3689   match(RegP);
 3690   match(rRegP);
 3691 
 3692   format %{ %}
 3693   interface(REG_INTER);
 3694 %}
 3695 
 3696 operand rRegL()
 3697 %{
 3698   constraint(ALLOC_IN_RC(long_reg));
 3699   match(RegL);
 3700   match(rax_RegL);
 3701   match(rdx_RegL);
 3702 
 3703   format %{ %}
 3704   interface(REG_INTER);
 3705 %}
 3706 
 3707 // Special Registers
 3708 operand no_rax_rdx_RegL()
 3709 %{
 3710   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3711   match(RegL);
 3712   match(rRegL);
 3713 
 3714   format %{ %}
 3715   interface(REG_INTER);
 3716 %}
 3717 
 3718 operand no_rax_RegL()
 3719 %{
 3720   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 3721   match(RegL);
 3722   match(rRegL);
 3723   match(rdx_RegL);
 3724 
 3725   format %{ %}
 3726   interface(REG_INTER);
 3727 %}
 3728 
 3729 operand rax_RegL()
 3730 %{
 3731   constraint(ALLOC_IN_RC(long_rax_reg));
 3732   match(RegL);
 3733   match(rRegL);
 3734 
 3735   format %{ "RAX" %}
 3736   interface(REG_INTER);
 3737 %}
 3738 
 3739 operand rcx_RegL()
 3740 %{
 3741   constraint(ALLOC_IN_RC(long_rcx_reg));
 3742   match(RegL);
 3743   match(rRegL);
 3744 
 3745   format %{ %}
 3746   interface(REG_INTER);
 3747 %}
 3748 
 3749 operand rdx_RegL()
 3750 %{
 3751   constraint(ALLOC_IN_RC(long_rdx_reg));
 3752   match(RegL);
 3753   match(rRegL);
 3754 
 3755   format %{ %}
 3756   interface(REG_INTER);
 3757 %}
 3758 
 3759 operand no_rbp_r13_RegL()
 3760 %{
 3761   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 3762   match(RegL);
 3763   match(rRegL);
 3764   match(rax_RegL);
 3765   match(rcx_RegL);
 3766   match(rdx_RegL);
 3767 
 3768   format %{ %}
 3769   interface(REG_INTER);
 3770 %}
 3771 
 3772 // Flags register, used as output of compare instructions
 3773 operand rFlagsReg()
 3774 %{
 3775   constraint(ALLOC_IN_RC(int_flags));
 3776   match(RegFlags);
 3777 
 3778   format %{ "RFLAGS" %}
 3779   interface(REG_INTER);
 3780 %}
 3781 
 3782 // Flags register, used as output of FLOATING POINT compare instructions
 3783 operand rFlagsRegU()
 3784 %{
 3785   constraint(ALLOC_IN_RC(int_flags));
 3786   match(RegFlags);
 3787 
 3788   format %{ "RFLAGS_U" %}
 3789   interface(REG_INTER);
 3790 %}
 3791 
 3792 operand rFlagsRegUCF() %{
 3793   constraint(ALLOC_IN_RC(int_flags));
 3794   match(RegFlags);
 3795   predicate(false);
 3796 
 3797   format %{ "RFLAGS_U_CF" %}
 3798   interface(REG_INTER);
 3799 %}
 3800 
 3801 // Float register operands
 3802 operand regF() %{
 3803    constraint(ALLOC_IN_RC(float_reg));
 3804    match(RegF);
 3805 
 3806    format %{ %}
 3807    interface(REG_INTER);
 3808 %}
 3809 
 3810 // Float register operands
 3811 operand legRegF() %{
 3812    constraint(ALLOC_IN_RC(float_reg_legacy));
 3813    match(RegF);
 3814 
 3815    format %{ %}
 3816    interface(REG_INTER);
 3817 %}
 3818 
 3819 // Float register operands
 3820 operand vlRegF() %{
 3821    constraint(ALLOC_IN_RC(float_reg_vl));
 3822    match(RegF);
 3823 
 3824    format %{ %}
 3825    interface(REG_INTER);
 3826 %}
 3827 
 3828 // Double register operands
 3829 operand regD() %{
 3830    constraint(ALLOC_IN_RC(double_reg));
 3831    match(RegD);
 3832 
 3833    format %{ %}
 3834    interface(REG_INTER);
 3835 %}
 3836 
 3837 // Double register operands
 3838 operand legRegD() %{
 3839    constraint(ALLOC_IN_RC(double_reg_legacy));
 3840    match(RegD);
 3841 
 3842    format %{ %}
 3843    interface(REG_INTER);
 3844 %}
 3845 
 3846 // Double register operands
 3847 operand vlRegD() %{
 3848    constraint(ALLOC_IN_RC(double_reg_vl));
 3849    match(RegD);
 3850 
 3851    format %{ %}
 3852    interface(REG_INTER);
 3853 %}
 3854 
 3855 //----------Memory Operands----------------------------------------------------
 3856 // Direct Memory Operand
 3857 // operand direct(immP addr)
 3858 // %{
 3859 //   match(addr);
 3860 
 3861 //   format %{ "[$addr]" %}
 3862 //   interface(MEMORY_INTER) %{
 3863 //     base(0xFFFFFFFF);
 3864 //     index(0x4);
 3865 //     scale(0x0);
 3866 //     disp($addr);
 3867 //   %}
 3868 // %}
 3869 
 3870 // Indirect Memory Operand
 3871 operand indirect(any_RegP reg)
 3872 %{
 3873   constraint(ALLOC_IN_RC(ptr_reg));
 3874   match(reg);
 3875 
 3876   format %{ "[$reg]" %}
 3877   interface(MEMORY_INTER) %{
 3878     base($reg);
 3879     index(0x4);
 3880     scale(0x0);
 3881     disp(0x0);
 3882   %}
 3883 %}
 3884 
 3885 // Indirect Memory Plus Short Offset Operand
 3886 operand indOffset8(any_RegP reg, immL8 off)
 3887 %{
 3888   constraint(ALLOC_IN_RC(ptr_reg));
 3889   match(AddP reg off);
 3890 
 3891   format %{ "[$reg + $off (8-bit)]" %}
 3892   interface(MEMORY_INTER) %{
 3893     base($reg);
 3894     index(0x4);
 3895     scale(0x0);
 3896     disp($off);
 3897   %}
 3898 %}
 3899 
 3900 // Indirect Memory Plus Long Offset Operand
 3901 operand indOffset32(any_RegP reg, immL32 off)
 3902 %{
 3903   constraint(ALLOC_IN_RC(ptr_reg));
 3904   match(AddP reg off);
 3905 
 3906   format %{ "[$reg + $off (32-bit)]" %}
 3907   interface(MEMORY_INTER) %{
 3908     base($reg);
 3909     index(0x4);
 3910     scale(0x0);
 3911     disp($off);
 3912   %}
 3913 %}
 3914 
 3915 // Indirect Memory Plus Index Register Plus Offset Operand
 3916 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 3917 %{
 3918   constraint(ALLOC_IN_RC(ptr_reg));
 3919   match(AddP (AddP reg lreg) off);
 3920 
 3921   op_cost(10);
 3922   format %{"[$reg + $off + $lreg]" %}
 3923   interface(MEMORY_INTER) %{
 3924     base($reg);
 3925     index($lreg);
 3926     scale(0x0);
 3927     disp($off);
 3928   %}
 3929 %}
 3930 
 3931 // Indirect Memory Plus Index Register Plus Offset Operand
 3932 operand indIndex(any_RegP reg, rRegL lreg)
 3933 %{
 3934   constraint(ALLOC_IN_RC(ptr_reg));
 3935   match(AddP reg lreg);
 3936 
 3937   op_cost(10);
 3938   format %{"[$reg + $lreg]" %}
 3939   interface(MEMORY_INTER) %{
 3940     base($reg);
 3941     index($lreg);
 3942     scale(0x0);
 3943     disp(0x0);
 3944   %}
 3945 %}
 3946 
 3947 // Indirect Memory Times Scale Plus Index Register
 3948 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 3949 %{
 3950   constraint(ALLOC_IN_RC(ptr_reg));
 3951   match(AddP reg (LShiftL lreg scale));
 3952 
 3953   op_cost(10);
 3954   format %{"[$reg + $lreg << $scale]" %}
 3955   interface(MEMORY_INTER) %{
 3956     base($reg);
 3957     index($lreg);
 3958     scale($scale);
 3959     disp(0x0);
 3960   %}
 3961 %}
 3962 
 3963 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3964 %{
 3965   constraint(ALLOC_IN_RC(ptr_reg));
 3966   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3967   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3968 
 3969   op_cost(10);
 3970   format %{"[$reg + pos $idx << $scale]" %}
 3971   interface(MEMORY_INTER) %{
 3972     base($reg);
 3973     index($idx);
 3974     scale($scale);
 3975     disp(0x0);
 3976   %}
 3977 %}
 3978 
 3979 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3980 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3981 %{
 3982   constraint(ALLOC_IN_RC(ptr_reg));
 3983   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3984 
 3985   op_cost(10);
 3986   format %{"[$reg + $off + $lreg << $scale]" %}
 3987   interface(MEMORY_INTER) %{
 3988     base($reg);
 3989     index($lreg);
 3990     scale($scale);
 3991     disp($off);
 3992   %}
 3993 %}
 3994 
 3995 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3996 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3997 %{
 3998   constraint(ALLOC_IN_RC(ptr_reg));
 3999   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4000   match(AddP (AddP reg (ConvI2L idx)) off);
 4001 
 4002   op_cost(10);
 4003   format %{"[$reg + $off + $idx]" %}
 4004   interface(MEMORY_INTER) %{
 4005     base($reg);
 4006     index($idx);
 4007     scale(0x0);
 4008     disp($off);
 4009   %}
 4010 %}
 4011 
 4012 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4013 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 4014 %{
 4015   constraint(ALLOC_IN_RC(ptr_reg));
 4016   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4017   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 4018 
 4019   op_cost(10);
 4020   format %{"[$reg + $off + $idx << $scale]" %}
 4021   interface(MEMORY_INTER) %{
 4022     base($reg);
 4023     index($idx);
 4024     scale($scale);
 4025     disp($off);
 4026   %}
 4027 %}
 4028 
 4029 // Indirect Narrow Oop Operand
 4030 operand indCompressedOop(rRegN reg) %{
 4031   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 4032   constraint(ALLOC_IN_RC(ptr_reg));
 4033   match(DecodeN reg);
 4034 
 4035   op_cost(10);
 4036   format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
 4037   interface(MEMORY_INTER) %{
 4038     base(0xc); // R12
 4039     index($reg);
 4040     scale(0x3);
 4041     disp(0x0);
 4042   %}
 4043 %}
 4044 
 4045 // Indirect Narrow Oop Plus Offset Operand
 4046 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 4047 // we can't free r12 even with CompressedOops::base() == NULL.
 4048 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 4049   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 4050   constraint(ALLOC_IN_RC(ptr_reg));
 4051   match(AddP (DecodeN reg) off);
 4052 
 4053   op_cost(10);
 4054   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 4055   interface(MEMORY_INTER) %{
 4056     base(0xc); // R12
 4057     index($reg);
 4058     scale(0x3);
 4059     disp($off);
 4060   %}
 4061 %}
 4062 
 4063 // Indirect Memory Operand
 4064 operand indirectNarrow(rRegN reg)
 4065 %{
 4066   predicate(CompressedOops::shift() == 0);
 4067   constraint(ALLOC_IN_RC(ptr_reg));
 4068   match(DecodeN reg);
 4069 
 4070   format %{ "[$reg]" %}
 4071   interface(MEMORY_INTER) %{
 4072     base($reg);
 4073     index(0x4);
 4074     scale(0x0);
 4075     disp(0x0);
 4076   %}
 4077 %}
 4078 
 4079 // Indirect Memory Plus Short Offset Operand
 4080 operand indOffset8Narrow(rRegN reg, immL8 off)
 4081 %{
 4082   predicate(CompressedOops::shift() == 0);
 4083   constraint(ALLOC_IN_RC(ptr_reg));
 4084   match(AddP (DecodeN reg) off);
 4085 
 4086   format %{ "[$reg + $off (8-bit)]" %}
 4087   interface(MEMORY_INTER) %{
 4088     base($reg);
 4089     index(0x4);
 4090     scale(0x0);
 4091     disp($off);
 4092   %}
 4093 %}
 4094 
 4095 // Indirect Memory Plus Long Offset Operand
 4096 operand indOffset32Narrow(rRegN reg, immL32 off)
 4097 %{
 4098   predicate(CompressedOops::shift() == 0);
 4099   constraint(ALLOC_IN_RC(ptr_reg));
 4100   match(AddP (DecodeN reg) off);
 4101 
 4102   format %{ "[$reg + $off (32-bit)]" %}
 4103   interface(MEMORY_INTER) %{
 4104     base($reg);
 4105     index(0x4);
 4106     scale(0x0);
 4107     disp($off);
 4108   %}
 4109 %}
 4110 
 4111 // Indirect Memory Plus Index Register Plus Offset Operand
 4112 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 4113 %{
 4114   predicate(CompressedOops::shift() == 0);
 4115   constraint(ALLOC_IN_RC(ptr_reg));
 4116   match(AddP (AddP (DecodeN reg) lreg) off);
 4117 
 4118   op_cost(10);
 4119   format %{"[$reg + $off + $lreg]" %}
 4120   interface(MEMORY_INTER) %{
 4121     base($reg);
 4122     index($lreg);
 4123     scale(0x0);
 4124     disp($off);
 4125   %}
 4126 %}
 4127 
 4128 // Indirect Memory Plus Index Register Plus Offset Operand
 4129 operand indIndexNarrow(rRegN reg, rRegL lreg)
 4130 %{
 4131   predicate(CompressedOops::shift() == 0);
 4132   constraint(ALLOC_IN_RC(ptr_reg));
 4133   match(AddP (DecodeN reg) lreg);
 4134 
 4135   op_cost(10);
 4136   format %{"[$reg + $lreg]" %}
 4137   interface(MEMORY_INTER) %{
 4138     base($reg);
 4139     index($lreg);
 4140     scale(0x0);
 4141     disp(0x0);
 4142   %}
 4143 %}
 4144 
 4145 // Indirect Memory Times Scale Plus Index Register
 4146 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 4147 %{
 4148   predicate(CompressedOops::shift() == 0);
 4149   constraint(ALLOC_IN_RC(ptr_reg));
 4150   match(AddP (DecodeN reg) (LShiftL lreg scale));
 4151 
 4152   op_cost(10);
 4153   format %{"[$reg + $lreg << $scale]" %}
 4154   interface(MEMORY_INTER) %{
 4155     base($reg);
 4156     index($lreg);
 4157     scale($scale);
 4158     disp(0x0);
 4159   %}
 4160 %}
 4161 
 4162 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 4163 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 4164 %{
 4165   predicate(CompressedOops::shift() == 0);
 4166   constraint(ALLOC_IN_RC(ptr_reg));
 4167   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 4168 
 4169   op_cost(10);
 4170   format %{"[$reg + $off + $lreg << $scale]" %}
 4171   interface(MEMORY_INTER) %{
 4172     base($reg);
 4173     index($lreg);
 4174     scale($scale);
 4175     disp($off);
 4176   %}
 4177 %}
 4178 
 4179 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 4180 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 4181 %{
 4182   constraint(ALLOC_IN_RC(ptr_reg));
 4183   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 4184   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 4185 
 4186   op_cost(10);
 4187   format %{"[$reg + $off + $idx]" %}
 4188   interface(MEMORY_INTER) %{
 4189     base($reg);
 4190     index($idx);
 4191     scale(0x0);
 4192     disp($off);
 4193   %}
 4194 %}
 4195 
 4196 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 4197 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 4198 %{
 4199   constraint(ALLOC_IN_RC(ptr_reg));
 4200   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 4201   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 4202 
 4203   op_cost(10);
 4204   format %{"[$reg + $off + $idx << $scale]" %}
 4205   interface(MEMORY_INTER) %{
 4206     base($reg);
 4207     index($idx);
 4208     scale($scale);
 4209     disp($off);
 4210   %}
 4211 %}
 4212 
 4213 //----------Special Memory Operands--------------------------------------------
 4214 // Stack Slot Operand - This operand is used for loading and storing temporary
 4215 //                      values on the stack where a match requires a value to
 4216 //                      flow through memory.
 4217 operand stackSlotP(sRegP reg)
 4218 %{
 4219   constraint(ALLOC_IN_RC(stack_slots));
 4220   // No match rule because this operand is only generated in matching
 4221 
 4222   format %{ "[$reg]" %}
 4223   interface(MEMORY_INTER) %{
 4224     base(0x4);   // RSP
 4225     index(0x4);  // No Index
 4226     scale(0x0);  // No Scale
 4227     disp($reg);  // Stack Offset
 4228   %}
 4229 %}
 4230 
 4231 operand stackSlotI(sRegI reg)
 4232 %{
 4233   constraint(ALLOC_IN_RC(stack_slots));
 4234   // No match rule because this operand is only generated in matching
 4235 
 4236   format %{ "[$reg]" %}
 4237   interface(MEMORY_INTER) %{
 4238     base(0x4);   // RSP
 4239     index(0x4);  // No Index
 4240     scale(0x0);  // No Scale
 4241     disp($reg);  // Stack Offset
 4242   %}
 4243 %}
 4244 
 4245 operand stackSlotF(sRegF reg)
 4246 %{
 4247   constraint(ALLOC_IN_RC(stack_slots));
 4248   // No match rule because this operand is only generated in matching
 4249 
 4250   format %{ "[$reg]" %}
 4251   interface(MEMORY_INTER) %{
 4252     base(0x4);   // RSP
 4253     index(0x4);  // No Index
 4254     scale(0x0);  // No Scale
 4255     disp($reg);  // Stack Offset
 4256   %}
 4257 %}
 4258 
 4259 operand stackSlotD(sRegD reg)
 4260 %{
 4261   constraint(ALLOC_IN_RC(stack_slots));
 4262   // No match rule because this operand is only generated in matching
 4263 
 4264   format %{ "[$reg]" %}
 4265   interface(MEMORY_INTER) %{
 4266     base(0x4);   // RSP
 4267     index(0x4);  // No Index
 4268     scale(0x0);  // No Scale
 4269     disp($reg);  // Stack Offset
 4270   %}
 4271 %}
 4272 operand stackSlotL(sRegL reg)
 4273 %{
 4274   constraint(ALLOC_IN_RC(stack_slots));
 4275   // No match rule because this operand is only generated in matching
 4276 
 4277   format %{ "[$reg]" %}
 4278   interface(MEMORY_INTER) %{
 4279     base(0x4);   // RSP
 4280     index(0x4);  // No Index
 4281     scale(0x0);  // No Scale
 4282     disp($reg);  // Stack Offset
 4283   %}
 4284 %}
 4285 
 4286 //----------Conditional Branch Operands----------------------------------------
 4287 // Comparison Op  - This is the operation of the comparison, and is limited to
 4288 //                  the following set of codes:
 4289 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 4290 //
 4291 // Other attributes of the comparison, such as unsignedness, are specified
 4292 // by the comparison instruction that sets a condition code flags register.
 4293 // That result is represented by a flags operand whose subtype is appropriate
 4294 // to the unsignedness (etc.) of the comparison.
 4295 //
 4296 // Later, the instruction which matches both the Comparison Op (a Bool) and
 4297 // the flags (produced by the Cmp) specifies the coding of the comparison op
 4298 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 4299 
 4300 // Comparision Code
 4301 operand cmpOp()
 4302 %{
 4303   match(Bool);
 4304 
 4305   format %{ "" %}
 4306   interface(COND_INTER) %{
 4307     equal(0x4, "e");
 4308     not_equal(0x5, "ne");
 4309     less(0xC, "l");
 4310     greater_equal(0xD, "ge");
 4311     less_equal(0xE, "le");
 4312     greater(0xF, "g");
 4313     overflow(0x0, "o");
 4314     no_overflow(0x1, "no");
 4315   %}
 4316 %}
 4317 
 4318 // Comparison Code, unsigned compare.  Used by FP also, with
 4319 // C2 (unordered) turned into GT or LT already.  The other bits
 4320 // C0 and C3 are turned into Carry & Zero flags.
 4321 operand cmpOpU()
 4322 %{
 4323   match(Bool);
 4324 
 4325   format %{ "" %}
 4326   interface(COND_INTER) %{
 4327     equal(0x4, "e");
 4328     not_equal(0x5, "ne");
 4329     less(0x2, "b");
 4330     greater_equal(0x3, "nb");
 4331     less_equal(0x6, "be");
 4332     greater(0x7, "nbe");
 4333     overflow(0x0, "o");
 4334     no_overflow(0x1, "no");
 4335   %}
 4336 %}
 4337 
 4338 
 4339 // Floating comparisons that don't require any fixup for the unordered case
 4340 operand cmpOpUCF() %{
 4341   match(Bool);
 4342   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 4343             n->as_Bool()->_test._test == BoolTest::ge ||
 4344             n->as_Bool()->_test._test == BoolTest::le ||
 4345             n->as_Bool()->_test._test == BoolTest::gt);
 4346   format %{ "" %}
 4347   interface(COND_INTER) %{
 4348     equal(0x4, "e");
 4349     not_equal(0x5, "ne");
 4350     less(0x2, "b");
 4351     greater_equal(0x3, "nb");
 4352     less_equal(0x6, "be");
 4353     greater(0x7, "nbe");
 4354     overflow(0x0, "o");
 4355     no_overflow(0x1, "no");
 4356   %}
 4357 %}
 4358 
 4359 
 4360 // Floating comparisons that can be fixed up with extra conditional jumps
 4361 operand cmpOpUCF2() %{
 4362   match(Bool);
 4363   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
 4364             n->as_Bool()->_test._test == BoolTest::eq);
 4365   format %{ "" %}
 4366   interface(COND_INTER) %{
 4367     equal(0x4, "e");
 4368     not_equal(0x5, "ne");
 4369     less(0x2, "b");
 4370     greater_equal(0x3, "nb");
 4371     less_equal(0x6, "be");
 4372     greater(0x7, "nbe");
 4373     overflow(0x0, "o");
 4374     no_overflow(0x1, "no");
 4375   %}
 4376 %}
 4377 
 4378 //----------OPERAND CLASSES----------------------------------------------------
 4379 // Operand Classes are groups of operands that are used as to simplify
 4380 // instruction definitions by not requiring the AD writer to specify separate
 4381 // instructions for every form of operand when the instruction accepts
 4382 // multiple operand types with the same basic encoding and format.  The classic
 4383 // case of this is memory operands.
 4384 
 4385 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 4386                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 4387                indCompressedOop, indCompressedOopOffset,
 4388                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 4389                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 4390                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 4391 
 4392 //----------PIPELINE-----------------------------------------------------------
 4393 // Rules which define the behavior of the target architectures pipeline.
 4394 pipeline %{
 4395 
 4396 //----------ATTRIBUTES---------------------------------------------------------
 4397 attributes %{
 4398   variable_size_instructions;        // Fixed size instructions
 4399   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 4400   instruction_unit_size = 1;         // An instruction is 1 bytes long
 4401   instruction_fetch_unit_size = 16;  // The processor fetches one line
 4402   instruction_fetch_units = 1;       // of 16 bytes
 4403 
 4404   // List of nop instructions
 4405   nops( MachNop );
 4406 %}
 4407 
 4408 //----------RESOURCES----------------------------------------------------------
 4409 // Resources are the functional units available to the machine
 4410 
 4411 // Generic P2/P3 pipeline
 4412 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 4413 // 3 instructions decoded per cycle.
 4414 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 4415 // 3 ALU op, only ALU0 handles mul instructions.
 4416 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 4417            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 4418            BR, FPU,
 4419            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 4420 
 4421 //----------PIPELINE DESCRIPTION-----------------------------------------------
 4422 // Pipeline Description specifies the stages in the machine's pipeline
 4423 
 4424 // Generic P2/P3 pipeline
 4425 pipe_desc(S0, S1, S2, S3, S4, S5);
 4426 
 4427 //----------PIPELINE CLASSES---------------------------------------------------
 4428 // Pipeline Classes describe the stages in which input and output are
 4429 // referenced by the hardware pipeline.
 4430 
 4431 // Naming convention: ialu or fpu
 4432 // Then: _reg
 4433 // Then: _reg if there is a 2nd register
 4434 // Then: _long if it's a pair of instructions implementing a long
 4435 // Then: _fat if it requires the big decoder
 4436 //   Or: _mem if it requires the big decoder and a memory unit.
 4437 
 4438 // Integer ALU reg operation
 4439 pipe_class ialu_reg(rRegI dst)
 4440 %{
 4441     single_instruction;
 4442     dst    : S4(write);
 4443     dst    : S3(read);
 4444     DECODE : S0;        // any decoder
 4445     ALU    : S3;        // any alu
 4446 %}
 4447 
 4448 // Long ALU reg operation
 4449 pipe_class ialu_reg_long(rRegL dst)
 4450 %{
 4451     instruction_count(2);
 4452     dst    : S4(write);
 4453     dst    : S3(read);
 4454     DECODE : S0(2);     // any 2 decoders
 4455     ALU    : S3(2);     // both alus
 4456 %}
 4457 
 4458 // Integer ALU reg operation using big decoder
 4459 pipe_class ialu_reg_fat(rRegI dst)
 4460 %{
 4461     single_instruction;
 4462     dst    : S4(write);
 4463     dst    : S3(read);
 4464     D0     : S0;        // big decoder only
 4465     ALU    : S3;        // any alu
 4466 %}
 4467 
 4468 // Integer ALU reg-reg operation
 4469 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 4470 %{
 4471     single_instruction;
 4472     dst    : S4(write);
 4473     src    : S3(read);
 4474     DECODE : S0;        // any decoder
 4475     ALU    : S3;        // any alu
 4476 %}
 4477 
 4478 // Integer ALU reg-reg operation
 4479 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 4480 %{
 4481     single_instruction;
 4482     dst    : S4(write);
 4483     src    : S3(read);
 4484     D0     : S0;        // big decoder only
 4485     ALU    : S3;        // any alu
 4486 %}
 4487 
 4488 // Integer ALU reg-mem operation
 4489 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 4490 %{
 4491     single_instruction;
 4492     dst    : S5(write);
 4493     mem    : S3(read);
 4494     D0     : S0;        // big decoder only
 4495     ALU    : S4;        // any alu
 4496     MEM    : S3;        // any mem
 4497 %}
 4498 
 4499 // Integer mem operation (prefetch)
 4500 pipe_class ialu_mem(memory mem)
 4501 %{
 4502     single_instruction;
 4503     mem    : S3(read);
 4504     D0     : S0;        // big decoder only
 4505     MEM    : S3;        // any mem
 4506 %}
 4507 
 4508 // Integer Store to Memory
 4509 pipe_class ialu_mem_reg(memory mem, rRegI src)
 4510 %{
 4511     single_instruction;
 4512     mem    : S3(read);
 4513     src    : S5(read);
 4514     D0     : S0;        // big decoder only
 4515     ALU    : S4;        // any alu
 4516     MEM    : S3;
 4517 %}
 4518 
 4519 // // Long Store to Memory
 4520 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 4521 // %{
 4522 //     instruction_count(2);
 4523 //     mem    : S3(read);
 4524 //     src    : S5(read);
 4525 //     D0     : S0(2);          // big decoder only; twice
 4526 //     ALU    : S4(2);     // any 2 alus
 4527 //     MEM    : S3(2);  // Both mems
 4528 // %}
 4529 
 4530 // Integer Store to Memory
 4531 pipe_class ialu_mem_imm(memory mem)
 4532 %{
 4533     single_instruction;
 4534     mem    : S3(read);
 4535     D0     : S0;        // big decoder only
 4536     ALU    : S4;        // any alu
 4537     MEM    : S3;
 4538 %}
 4539 
 4540 // Integer ALU0 reg-reg operation
 4541 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 4542 %{
 4543     single_instruction;
 4544     dst    : S4(write);
 4545     src    : S3(read);
 4546     D0     : S0;        // Big decoder only
 4547     ALU0   : S3;        // only alu0
 4548 %}
 4549 
 4550 // Integer ALU0 reg-mem operation
 4551 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 4552 %{
 4553     single_instruction;
 4554     dst    : S5(write);
 4555     mem    : S3(read);
 4556     D0     : S0;        // big decoder only
 4557     ALU0   : S4;        // ALU0 only
 4558     MEM    : S3;        // any mem
 4559 %}
 4560 
 4561 // Integer ALU reg-reg operation
 4562 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 4563 %{
 4564     single_instruction;
 4565     cr     : S4(write);
 4566     src1   : S3(read);
 4567     src2   : S3(read);
 4568     DECODE : S0;        // any decoder
 4569     ALU    : S3;        // any alu
 4570 %}
 4571 
 4572 // Integer ALU reg-imm operation
 4573 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 4574 %{
 4575     single_instruction;
 4576     cr     : S4(write);
 4577     src1   : S3(read);
 4578     DECODE : S0;        // any decoder
 4579     ALU    : S3;        // any alu
 4580 %}
 4581 
 4582 // Integer ALU reg-mem operation
 4583 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 4584 %{
 4585     single_instruction;
 4586     cr     : S4(write);
 4587     src1   : S3(read);
 4588     src2   : S3(read);
 4589     D0     : S0;        // big decoder only
 4590     ALU    : S4;        // any alu
 4591     MEM    : S3;
 4592 %}
 4593 
 4594 // Conditional move reg-reg
 4595 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 4596 %{
 4597     instruction_count(4);
 4598     y      : S4(read);
 4599     q      : S3(read);
 4600     p      : S3(read);
 4601     DECODE : S0(4);     // any decoder
 4602 %}
 4603 
 4604 // Conditional move reg-reg
 4605 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 4606 %{
 4607     single_instruction;
 4608     dst    : S4(write);
 4609     src    : S3(read);
 4610     cr     : S3(read);
 4611     DECODE : S0;        // any decoder
 4612 %}
 4613 
 4614 // Conditional move reg-mem
 4615 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 4616 %{
 4617     single_instruction;
 4618     dst    : S4(write);
 4619     src    : S3(read);
 4620     cr     : S3(read);
 4621     DECODE : S0;        // any decoder
 4622     MEM    : S3;
 4623 %}
 4624 
 4625 // Conditional move reg-reg long
 4626 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 4627 %{
 4628     single_instruction;
 4629     dst    : S4(write);
 4630     src    : S3(read);
 4631     cr     : S3(read);
 4632     DECODE : S0(2);     // any 2 decoders
 4633 %}
 4634 
 4635 // XXX
 4636 // // Conditional move double reg-reg
 4637 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
 4638 // %{
 4639 //     single_instruction;
 4640 //     dst    : S4(write);
 4641 //     src    : S3(read);
 4642 //     cr     : S3(read);
 4643 //     DECODE : S0;     // any decoder
 4644 // %}
 4645 
 4646 // Float reg-reg operation
 4647 pipe_class fpu_reg(regD dst)
 4648 %{
 4649     instruction_count(2);
 4650     dst    : S3(read);
 4651     DECODE : S0(2);     // any 2 decoders
 4652     FPU    : S3;
 4653 %}
 4654 
 4655 // Float reg-reg operation
 4656 pipe_class fpu_reg_reg(regD dst, regD src)
 4657 %{
 4658     instruction_count(2);
 4659     dst    : S4(write);
 4660     src    : S3(read);
 4661     DECODE : S0(2);     // any 2 decoders
 4662     FPU    : S3;
 4663 %}
 4664 
 4665 // Float reg-reg operation
 4666 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 4667 %{
 4668     instruction_count(3);
 4669     dst    : S4(write);
 4670     src1   : S3(read);
 4671     src2   : S3(read);
 4672     DECODE : S0(3);     // any 3 decoders
 4673     FPU    : S3(2);
 4674 %}
 4675 
 4676 // Float reg-reg operation
 4677 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 4678 %{
 4679     instruction_count(4);
 4680     dst    : S4(write);
 4681     src1   : S3(read);
 4682     src2   : S3(read);
 4683     src3   : S3(read);
 4684     DECODE : S0(4);     // any 3 decoders
 4685     FPU    : S3(2);
 4686 %}
 4687 
 4688 // Float reg-reg operation
 4689 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 4690 %{
 4691     instruction_count(4);
 4692     dst    : S4(write);
 4693     src1   : S3(read);
 4694     src2   : S3(read);
 4695     src3   : S3(read);
 4696     DECODE : S1(3);     // any 3 decoders
 4697     D0     : S0;        // Big decoder only
 4698     FPU    : S3(2);
 4699     MEM    : S3;
 4700 %}
 4701 
 4702 // Float reg-mem operation
 4703 pipe_class fpu_reg_mem(regD dst, memory mem)
 4704 %{
 4705     instruction_count(2);
 4706     dst    : S5(write);
 4707     mem    : S3(read);
 4708     D0     : S0;        // big decoder only
 4709     DECODE : S1;        // any decoder for FPU POP
 4710     FPU    : S4;
 4711     MEM    : S3;        // any mem
 4712 %}
 4713 
 4714 // Float reg-mem operation
 4715 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 4716 %{
 4717     instruction_count(3);
 4718     dst    : S5(write);
 4719     src1   : S3(read);
 4720     mem    : S3(read);
 4721     D0     : S0;        // big decoder only
 4722     DECODE : S1(2);     // any decoder for FPU POP
 4723     FPU    : S4;
 4724     MEM    : S3;        // any mem
 4725 %}
 4726 
 4727 // Float mem-reg operation
 4728 pipe_class fpu_mem_reg(memory mem, regD src)
 4729 %{
 4730     instruction_count(2);
 4731     src    : S5(read);
 4732     mem    : S3(read);
 4733     DECODE : S0;        // any decoder for FPU PUSH
 4734     D0     : S1;        // big decoder only
 4735     FPU    : S4;
 4736     MEM    : S3;        // any mem
 4737 %}
 4738 
 4739 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 4740 %{
 4741     instruction_count(3);
 4742     src1   : S3(read);
 4743     src2   : S3(read);
 4744     mem    : S3(read);
 4745     DECODE : S0(2);     // any decoder for FPU PUSH
 4746     D0     : S1;        // big decoder only
 4747     FPU    : S4;
 4748     MEM    : S3;        // any mem
 4749 %}
 4750 
 4751 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 4752 %{
 4753     instruction_count(3);
 4754     src1   : S3(read);
 4755     src2   : S3(read);
 4756     mem    : S4(read);
 4757     DECODE : S0;        // any decoder for FPU PUSH
 4758     D0     : S0(2);     // big decoder only
 4759     FPU    : S4;
 4760     MEM    : S3(2);     // any mem
 4761 %}
 4762 
 4763 pipe_class fpu_mem_mem(memory dst, memory src1)
 4764 %{
 4765     instruction_count(2);
 4766     src1   : S3(read);
 4767     dst    : S4(read);
 4768     D0     : S0(2);     // big decoder only
 4769     MEM    : S3(2);     // any mem
 4770 %}
 4771 
 4772 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 4773 %{
 4774     instruction_count(3);
 4775     src1   : S3(read);
 4776     src2   : S3(read);
 4777     dst    : S4(read);
 4778     D0     : S0(3);     // big decoder only
 4779     FPU    : S4;
 4780     MEM    : S3(3);     // any mem
 4781 %}
 4782 
 4783 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 4784 %{
 4785     instruction_count(3);
 4786     src1   : S4(read);
 4787     mem    : S4(read);
 4788     DECODE : S0;        // any decoder for FPU PUSH
 4789     D0     : S0(2);     // big decoder only
 4790     FPU    : S4;
 4791     MEM    : S3(2);     // any mem
 4792 %}
 4793 
 4794 // Float load constant
 4795 pipe_class fpu_reg_con(regD dst)
 4796 %{
 4797     instruction_count(2);
 4798     dst    : S5(write);
 4799     D0     : S0;        // big decoder only for the load
 4800     DECODE : S1;        // any decoder for FPU POP
 4801     FPU    : S4;
 4802     MEM    : S3;        // any mem
 4803 %}
 4804 
 4805 // Float load constant
 4806 pipe_class fpu_reg_reg_con(regD dst, regD src)
 4807 %{
 4808     instruction_count(3);
 4809     dst    : S5(write);
 4810     src    : S3(read);
 4811     D0     : S0;        // big decoder only for the load
 4812     DECODE : S1(2);     // any decoder for FPU POP
 4813     FPU    : S4;
 4814     MEM    : S3;        // any mem
 4815 %}
 4816 
 4817 // UnConditional branch
 4818 pipe_class pipe_jmp(label labl)
 4819 %{
 4820     single_instruction;
 4821     BR   : S3;
 4822 %}
 4823 
 4824 // Conditional branch
 4825 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 4826 %{
 4827     single_instruction;
 4828     cr    : S1(read);
 4829     BR    : S3;
 4830 %}
 4831 
 4832 // Allocation idiom
 4833 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 4834 %{
 4835     instruction_count(1); force_serialization;
 4836     fixed_latency(6);
 4837     heap_ptr : S3(read);
 4838     DECODE   : S0(3);
 4839     D0       : S2;
 4840     MEM      : S3;
 4841     ALU      : S3(2);
 4842     dst      : S5(write);
 4843     BR       : S5;
 4844 %}
 4845 
 4846 // Generic big/slow expanded idiom
 4847 pipe_class pipe_slow()
 4848 %{
 4849     instruction_count(10); multiple_bundles; force_serialization;
 4850     fixed_latency(100);
 4851     D0  : S0(2);
 4852     MEM : S3(2);
 4853 %}
 4854 
 4855 // The real do-nothing guy
 4856 pipe_class empty()
 4857 %{
 4858     instruction_count(0);
 4859 %}
 4860 
 4861 // Define the class for the Nop node
 4862 define
 4863 %{
 4864    MachNop = empty;
 4865 %}
 4866 
 4867 %}
 4868 
 4869 //----------INSTRUCTIONS-------------------------------------------------------
 4870 //
 4871 // match      -- States which machine-independent subtree may be replaced
 4872 //               by this instruction.
 4873 // ins_cost   -- The estimated cost of this instruction is used by instruction
 4874 //               selection to identify a minimum cost tree of machine
 4875 //               instructions that matches a tree of machine-independent
 4876 //               instructions.
 4877 // format     -- A string providing the disassembly for this instruction.
 4878 //               The value of an instruction's operand may be inserted
 4879 //               by referring to it with a '$' prefix.
 4880 // opcode     -- Three instruction opcodes may be provided.  These are referred
 4881 //               to within an encode class as $primary, $secondary, and $tertiary
 4882 //               rrspectively.  The primary opcode is commonly used to
 4883 //               indicate the type of machine instruction, while secondary
 4884 //               and tertiary are often used for prefix options or addressing
 4885 //               modes.
 4886 // ins_encode -- A list of encode classes with parameters. The encode class
 4887 //               name must have been defined in an 'enc_class' specification
 4888 //               in the encode section of the architecture description.
 4889 
 4890 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 4891 // Load Float
 4892 instruct MoveF2VL(vlRegF dst, regF src) %{
 4893   match(Set dst src);
 4894   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4895   ins_encode %{
 4896     ShouldNotReachHere();
 4897   %}
 4898   ins_pipe( fpu_reg_reg );
 4899 %}
 4900 
 4901 // Load Float
 4902 instruct MoveF2LEG(legRegF dst, regF src) %{
 4903   match(Set dst src);
 4904   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4905   ins_encode %{
 4906     ShouldNotReachHere();
 4907   %}
 4908   ins_pipe( fpu_reg_reg );
 4909 %}
 4910 
 4911 // Load Float
 4912 instruct MoveVL2F(regF dst, vlRegF src) %{
 4913   match(Set dst src);
 4914   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 4915   ins_encode %{
 4916     ShouldNotReachHere();
 4917   %}
 4918   ins_pipe( fpu_reg_reg );
 4919 %}
 4920 
 4921 // Load Float
 4922 instruct MoveLEG2F(regF dst, legRegF src) %{
 4923   match(Set dst src);
 4924   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 4925   ins_encode %{
 4926     ShouldNotReachHere();
 4927   %}
 4928   ins_pipe( fpu_reg_reg );
 4929 %}
 4930 
 4931 // Load Double
 4932 instruct MoveD2VL(vlRegD dst, regD src) %{
 4933   match(Set dst src);
 4934   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4935   ins_encode %{
 4936     ShouldNotReachHere();
 4937   %}
 4938   ins_pipe( fpu_reg_reg );
 4939 %}
 4940 
 4941 // Load Double
 4942 instruct MoveD2LEG(legRegD dst, regD src) %{
 4943   match(Set dst src);
 4944   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4945   ins_encode %{
 4946     ShouldNotReachHere();
 4947   %}
 4948   ins_pipe( fpu_reg_reg );
 4949 %}
 4950 
 4951 // Load Double
 4952 instruct MoveVL2D(regD dst, vlRegD src) %{
 4953   match(Set dst src);
 4954   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 4955   ins_encode %{
 4956     ShouldNotReachHere();
 4957   %}
 4958   ins_pipe( fpu_reg_reg );
 4959 %}
 4960 
 4961 // Load Double
 4962 instruct MoveLEG2D(regD dst, legRegD src) %{
 4963   match(Set dst src);
 4964   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 4965   ins_encode %{
 4966     ShouldNotReachHere();
 4967   %}
 4968   ins_pipe( fpu_reg_reg );
 4969 %}
 4970 
 4971 //----------Load/Store/Move Instructions---------------------------------------
 4972 //----------Load Instructions--------------------------------------------------
 4973 
 4974 // Load Byte (8 bit signed)
 4975 instruct loadB(rRegI dst, memory mem)
 4976 %{
 4977   match(Set dst (LoadB mem));
 4978 
 4979   ins_cost(125);
 4980   format %{ "movsbl  $dst, $mem\t# byte" %}
 4981 
 4982   ins_encode %{
 4983     __ movsbl($dst$$Register, $mem$$Address);
 4984   %}
 4985 
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 // Load Byte (8 bit signed) into Long Register
 4990 instruct loadB2L(rRegL dst, memory mem)
 4991 %{
 4992   match(Set dst (ConvI2L (LoadB mem)));
 4993 
 4994   ins_cost(125);
 4995   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4996 
 4997   ins_encode %{
 4998     __ movsbq($dst$$Register, $mem$$Address);
 4999   %}
 5000 
 5001   ins_pipe(ialu_reg_mem);
 5002 %}
 5003 
 5004 // Load Unsigned Byte (8 bit UNsigned)
 5005 instruct loadUB(rRegI dst, memory mem)
 5006 %{
 5007   match(Set dst (LoadUB mem));
 5008 
 5009   ins_cost(125);
 5010   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 5011 
 5012   ins_encode %{
 5013     __ movzbl($dst$$Register, $mem$$Address);
 5014   %}
 5015 
 5016   ins_pipe(ialu_reg_mem);
 5017 %}
 5018 
 5019 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 5020 instruct loadUB2L(rRegL dst, memory mem)
 5021 %{
 5022   match(Set dst (ConvI2L (LoadUB mem)));
 5023 
 5024   ins_cost(125);
 5025   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 5026 
 5027   ins_encode %{
 5028     __ movzbq($dst$$Register, $mem$$Address);
 5029   %}
 5030 
 5031   ins_pipe(ialu_reg_mem);
 5032 %}
 5033 
 5034 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 5035 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5036   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 5037   effect(KILL cr);
 5038 
 5039   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 5040             "andl    $dst, right_n_bits($mask, 8)" %}
 5041   ins_encode %{
 5042     Register Rdst = $dst$$Register;
 5043     __ movzbq(Rdst, $mem$$Address);
 5044     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 5045   %}
 5046   ins_pipe(ialu_reg_mem);
 5047 %}
 5048 
 5049 // Load Short (16 bit signed)
 5050 instruct loadS(rRegI dst, memory mem)
 5051 %{
 5052   match(Set dst (LoadS mem));
 5053 
 5054   ins_cost(125);
 5055   format %{ "movswl $dst, $mem\t# short" %}
 5056 
 5057   ins_encode %{
 5058     __ movswl($dst$$Register, $mem$$Address);
 5059   %}
 5060 
 5061   ins_pipe(ialu_reg_mem);
 5062 %}
 5063 
 5064 // Load Short (16 bit signed) to Byte (8 bit signed)
 5065 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5066   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 5067 
 5068   ins_cost(125);
 5069   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 5070   ins_encode %{
 5071     __ movsbl($dst$$Register, $mem$$Address);
 5072   %}
 5073   ins_pipe(ialu_reg_mem);
 5074 %}
 5075 
 5076 // Load Short (16 bit signed) into Long Register
 5077 instruct loadS2L(rRegL dst, memory mem)
 5078 %{
 5079   match(Set dst (ConvI2L (LoadS mem)));
 5080 
 5081   ins_cost(125);
 5082   format %{ "movswq $dst, $mem\t# short -> long" %}
 5083 
 5084   ins_encode %{
 5085     __ movswq($dst$$Register, $mem$$Address);
 5086   %}
 5087 
 5088   ins_pipe(ialu_reg_mem);
 5089 %}
 5090 
 5091 // Load Unsigned Short/Char (16 bit UNsigned)
 5092 instruct loadUS(rRegI dst, memory mem)
 5093 %{
 5094   match(Set dst (LoadUS mem));
 5095 
 5096   ins_cost(125);
 5097   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 5098 
 5099   ins_encode %{
 5100     __ movzwl($dst$$Register, $mem$$Address);
 5101   %}
 5102 
 5103   ins_pipe(ialu_reg_mem);
 5104 %}
 5105 
 5106 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 5107 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5108   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 5109 
 5110   ins_cost(125);
 5111   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 5112   ins_encode %{
 5113     __ movsbl($dst$$Register, $mem$$Address);
 5114   %}
 5115   ins_pipe(ialu_reg_mem);
 5116 %}
 5117 
 5118 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 5119 instruct loadUS2L(rRegL dst, memory mem)
 5120 %{
 5121   match(Set dst (ConvI2L (LoadUS mem)));
 5122 
 5123   ins_cost(125);
 5124   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 5125 
 5126   ins_encode %{
 5127     __ movzwq($dst$$Register, $mem$$Address);
 5128   %}
 5129 
 5130   ins_pipe(ialu_reg_mem);
 5131 %}
 5132 
 5133 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 5134 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5135   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5136 
 5137   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 5138   ins_encode %{
 5139     __ movzbq($dst$$Register, $mem$$Address);
 5140   %}
 5141   ins_pipe(ialu_reg_mem);
 5142 %}
 5143 
 5144 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 5145 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 5146   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 5147   effect(KILL cr);
 5148 
 5149   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 5150             "andl    $dst, right_n_bits($mask, 16)" %}
 5151   ins_encode %{
 5152     Register Rdst = $dst$$Register;
 5153     __ movzwq(Rdst, $mem$$Address);
 5154     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 5155   %}
 5156   ins_pipe(ialu_reg_mem);
 5157 %}
 5158 
 5159 // Load Integer
 5160 instruct loadI(rRegI dst, memory mem)
 5161 %{
 5162   match(Set dst (LoadI mem));
 5163 
 5164   ins_cost(125);
 5165   format %{ "movl    $dst, $mem\t# int" %}
 5166 
 5167   ins_encode %{
 5168     __ movl($dst$$Register, $mem$$Address);
 5169   %}
 5170 
 5171   ins_pipe(ialu_reg_mem);
 5172 %}
 5173 
 5174 // Load Integer (32 bit signed) to Byte (8 bit signed)
 5175 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 5176   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 5177 
 5178   ins_cost(125);
 5179   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 5180   ins_encode %{
 5181     __ movsbl($dst$$Register, $mem$$Address);
 5182   %}
 5183   ins_pipe(ialu_reg_mem);
 5184 %}
 5185 
 5186 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 5187 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 5188   match(Set dst (AndI (LoadI mem) mask));
 5189 
 5190   ins_cost(125);
 5191   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 5192   ins_encode %{
 5193     __ movzbl($dst$$Register, $mem$$Address);
 5194   %}
 5195   ins_pipe(ialu_reg_mem);
 5196 %}
 5197 
 5198 // Load Integer (32 bit signed) to Short (16 bit signed)
 5199 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 5200   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 5201 
 5202   ins_cost(125);
 5203   format %{ "movswl  $dst, $mem\t# int -> short" %}
 5204   ins_encode %{
 5205     __ movswl($dst$$Register, $mem$$Address);
 5206   %}
 5207   ins_pipe(ialu_reg_mem);
 5208 %}
 5209 
 5210 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 5211 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 5212   match(Set dst (AndI (LoadI mem) mask));
 5213 
 5214   ins_cost(125);
 5215   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 5216   ins_encode %{
 5217     __ movzwl($dst$$Register, $mem$$Address);
 5218   %}
 5219   ins_pipe(ialu_reg_mem);
 5220 %}
 5221 
 5222 // Load Integer into Long Register
 5223 instruct loadI2L(rRegL dst, memory mem)
 5224 %{
 5225   match(Set dst (ConvI2L (LoadI mem)));
 5226 
 5227   ins_cost(125);
 5228   format %{ "movslq  $dst, $mem\t# int -> long" %}
 5229 
 5230   ins_encode %{
 5231     __ movslq($dst$$Register, $mem$$Address);
 5232   %}
 5233 
 5234   ins_pipe(ialu_reg_mem);
 5235 %}
 5236 
 5237 // Load Integer with mask 0xFF into Long Register
 5238 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 5239   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5240 
 5241   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 5242   ins_encode %{
 5243     __ movzbq($dst$$Register, $mem$$Address);
 5244   %}
 5245   ins_pipe(ialu_reg_mem);
 5246 %}
 5247 
 5248 // Load Integer with mask 0xFFFF into Long Register
 5249 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 5250   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5251 
 5252   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 5253   ins_encode %{
 5254     __ movzwq($dst$$Register, $mem$$Address);
 5255   %}
 5256   ins_pipe(ialu_reg_mem);
 5257 %}
 5258 
 5259 // Load Integer with a 31-bit mask into Long Register
 5260 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 5261   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 5262   effect(KILL cr);
 5263 
 5264   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 5265             "andl    $dst, $mask" %}
 5266   ins_encode %{
 5267     Register Rdst = $dst$$Register;
 5268     __ movl(Rdst, $mem$$Address);
 5269     __ andl(Rdst, $mask$$constant);
 5270   %}
 5271   ins_pipe(ialu_reg_mem);
 5272 %}
 5273 
 5274 // Load Unsigned Integer into Long Register
 5275 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 5276 %{
 5277   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 5278 
 5279   ins_cost(125);
 5280   format %{ "movl    $dst, $mem\t# uint -> long" %}
 5281 
 5282   ins_encode %{
 5283     __ movl($dst$$Register, $mem$$Address);
 5284   %}
 5285 
 5286   ins_pipe(ialu_reg_mem);
 5287 %}
 5288 
 5289 // Load Long
 5290 instruct loadL(rRegL dst, memory mem)
 5291 %{
 5292   match(Set dst (LoadL mem));
 5293 
 5294   ins_cost(125);
 5295   format %{ "movq    $dst, $mem\t# long" %}
 5296 
 5297   ins_encode %{
 5298     __ movq($dst$$Register, $mem$$Address);
 5299   %}
 5300 
 5301   ins_pipe(ialu_reg_mem); // XXX
 5302 %}
 5303 
 5304 // Load Range
 5305 instruct loadRange(rRegI dst, memory mem)
 5306 %{
 5307   match(Set dst (LoadRange mem));
 5308 
 5309   ins_cost(125); // XXX
 5310   format %{ "movl    $dst, $mem\t# range" %}
 5311   ins_encode %{
 5312     __ movl($dst$$Register, $mem$$Address);
 5313   %}
 5314   ins_pipe(ialu_reg_mem);
 5315 %}
 5316 
 5317 // Load Pointer
 5318 instruct loadP(rRegP dst, memory mem)
 5319 %{
 5320   match(Set dst (LoadP mem));
 5321   predicate(n->as_Load()->barrier_data() == 0);
 5322 
 5323   ins_cost(125); // XXX
 5324   format %{ "movq    $dst, $mem\t# ptr" %}
 5325   ins_encode %{
 5326     __ movq($dst$$Register, $mem$$Address);
 5327   %}
 5328   ins_pipe(ialu_reg_mem); // XXX
 5329 %}
 5330 
 5331 // Load Compressed Pointer
 5332 instruct loadN(rRegN dst, memory mem)
 5333 %{
 5334    match(Set dst (LoadN mem));
 5335 
 5336    ins_cost(125); // XXX
 5337    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 5338    ins_encode %{
 5339      __ movl($dst$$Register, $mem$$Address);
 5340    %}
 5341    ins_pipe(ialu_reg_mem); // XXX
 5342 %}
 5343 
 5344 
 5345 // Load Klass Pointer
 5346 instruct loadKlass(rRegP dst, memory mem)
 5347 %{
 5348   match(Set dst (LoadKlass mem));
 5349 
 5350   ins_cost(125); // XXX
 5351   format %{ "movq    $dst, $mem\t# class" %}
 5352   ins_encode %{
 5353     __ movq($dst$$Register, $mem$$Address);
 5354   %}
 5355   ins_pipe(ialu_reg_mem); // XXX
 5356 %}
 5357 
 5358 // Load narrow Klass Pointer
 5359 instruct loadNKlass(rRegN dst, memory mem)
 5360 %{
 5361   match(Set dst (LoadNKlass mem));
 5362 
 5363   ins_cost(125); // XXX
 5364   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 5365   ins_encode %{
 5366     __ movl($dst$$Register, $mem$$Address);
 5367   %}
 5368   ins_pipe(ialu_reg_mem); // XXX
 5369 %}
 5370 
 5371 // Load Float
 5372 instruct loadF(regF dst, memory mem)
 5373 %{
 5374   match(Set dst (LoadF mem));
 5375 
 5376   ins_cost(145); // XXX
 5377   format %{ "movss   $dst, $mem\t# float" %}
 5378   ins_encode %{
 5379     __ movflt($dst$$XMMRegister, $mem$$Address);
 5380   %}
 5381   ins_pipe(pipe_slow); // XXX
 5382 %}
 5383 
 5384 // Load Double
 5385 instruct loadD_partial(regD dst, memory mem)
 5386 %{
 5387   predicate(!UseXmmLoadAndClearUpper);
 5388   match(Set dst (LoadD mem));
 5389 
 5390   ins_cost(145); // XXX
 5391   format %{ "movlpd  $dst, $mem\t# double" %}
 5392   ins_encode %{
 5393     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5394   %}
 5395   ins_pipe(pipe_slow); // XXX
 5396 %}
 5397 
 5398 instruct loadD(regD dst, memory mem)
 5399 %{
 5400   predicate(UseXmmLoadAndClearUpper);
 5401   match(Set dst (LoadD mem));
 5402 
 5403   ins_cost(145); // XXX
 5404   format %{ "movsd   $dst, $mem\t# double" %}
 5405   ins_encode %{
 5406     __ movdbl($dst$$XMMRegister, $mem$$Address);
 5407   %}
 5408   ins_pipe(pipe_slow); // XXX
 5409 %}
 5410 
 5411 
 5412 // Following pseudo code describes the algorithm for max[FD]:
 5413 // Min algorithm is on similar lines
 5414 //  btmp = (b < +0.0) ? a : b
 5415 //  atmp = (b < +0.0) ? b : a
 5416 //  Tmp  = Max_Float(atmp , btmp)
 5417 //  Res  = (atmp == NaN) ? atmp : Tmp
 5418 
 5419 // max = java.lang.Math.max(float a, float b)
 5420 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5421   predicate(UseAVX > 0 && !n->is_reduction());
 5422   match(Set dst (MaxF a b));
 5423   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5424   format %{
 5425      "vblendvps        $btmp,$b,$a,$b           \n\t"
 5426      "vblendvps        $atmp,$a,$b,$b           \n\t"
 5427      "vmaxss           $tmp,$atmp,$btmp         \n\t"
 5428      "vcmpps.unordered $btmp,$atmp,$atmp        \n\t"
 5429      "vblendvps        $dst,$tmp,$atmp,$btmp    \n\t"
 5430   %}
 5431   ins_encode %{
 5432     int vector_len = Assembler::AVX_128bit;
 5433     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5434     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5435     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5436     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5437     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5438  %}
 5439   ins_pipe( pipe_slow );
 5440 %}
 5441 
 5442 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5443   predicate(UseAVX > 0 && n->is_reduction());
 5444   match(Set dst (MaxF a b));
 5445   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5446 
 5447   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
 5448   ins_encode %{
 5449     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5450                     false /*min*/, true /*single*/);
 5451   %}
 5452   ins_pipe( pipe_slow );
 5453 %}
 5454 
 5455 // max = java.lang.Math.max(double a, double b)
 5456 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5457   predicate(UseAVX > 0 && !n->is_reduction());
 5458   match(Set dst (MaxD a b));
 5459   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 5460   format %{
 5461      "vblendvpd        $btmp,$b,$a,$b            \n\t"
 5462      "vblendvpd        $atmp,$a,$b,$b            \n\t"
 5463      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
 5464      "vcmppd.unordered $btmp,$atmp,$atmp         \n\t"
 5465      "vblendvpd        $dst,$tmp,$atmp,$btmp     \n\t"
 5466   %}
 5467   ins_encode %{
 5468     int vector_len = Assembler::AVX_128bit;
 5469     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
 5470     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
 5471     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5472     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5473     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5474   %}
 5475   ins_pipe( pipe_slow );
 5476 %}
 5477 
 5478 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5479   predicate(UseAVX > 0 && n->is_reduction());
 5480   match(Set dst (MaxD a b));
 5481   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5482 
 5483   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
 5484   ins_encode %{
 5485     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5486                     false /*min*/, false /*single*/);
 5487   %}
 5488   ins_pipe( pipe_slow );
 5489 %}
 5490 
 5491 // min = java.lang.Math.min(float a, float b)
 5492 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 5493   predicate(UseAVX > 0 && !n->is_reduction());
 5494   match(Set dst (MinF a b));
 5495   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5496   format %{
 5497      "vblendvps        $atmp,$a,$b,$a             \n\t"
 5498      "vblendvps        $btmp,$b,$a,$a             \n\t"
 5499      "vminss           $tmp,$atmp,$btmp           \n\t"
 5500      "vcmpps.unordered $btmp,$atmp,$atmp          \n\t"
 5501      "vblendvps        $dst,$tmp,$atmp,$btmp      \n\t"
 5502   %}
 5503   ins_encode %{
 5504     int vector_len = Assembler::AVX_128bit;
 5505     __ vblendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5506     __ vblendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5507     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5508     __ vcmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5509     __ vblendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5510   %}
 5511   ins_pipe( pipe_slow );
 5512 %}
 5513 
 5514 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
 5515   predicate(UseAVX > 0 && n->is_reduction());
 5516   match(Set dst (MinF a b));
 5517   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5518 
 5519   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
 5520   ins_encode %{
 5521     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5522                     true /*min*/, true /*single*/);
 5523   %}
 5524   ins_pipe( pipe_slow );
 5525 %}
 5526 
 5527 // min = java.lang.Math.min(double a, double b)
 5528 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 5529   predicate(UseAVX > 0 && !n->is_reduction());
 5530   match(Set dst (MinD a b));
 5531   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 5532   format %{
 5533      "vblendvpd        $atmp,$a,$b,$a           \n\t"
 5534      "vblendvpd        $btmp,$b,$a,$a           \n\t"
 5535      "vminsd           $tmp,$atmp,$btmp         \n\t"
 5536      "vcmppd.unordered $btmp,$atmp,$atmp        \n\t"
 5537      "vblendvpd        $dst,$tmp,$atmp,$btmp    \n\t"
 5538   %}
 5539   ins_encode %{
 5540     int vector_len = Assembler::AVX_128bit;
 5541     __ vblendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
 5542     __ vblendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
 5543     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
 5544     __ vcmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
 5545     __ vblendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
 5546   %}
 5547   ins_pipe( pipe_slow );
 5548 %}
 5549 
 5550 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
 5551   predicate(UseAVX > 0 && n->is_reduction());
 5552   match(Set dst (MinD a b));
 5553   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
 5554 
 5555   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
 5556   ins_encode %{
 5557     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
 5558                     true /*min*/, false /*single*/);
 5559   %}
 5560   ins_pipe( pipe_slow );
 5561 %}
 5562 
 5563 // Load Effective Address
 5564 instruct leaP8(rRegP dst, indOffset8 mem)
 5565 %{
 5566   match(Set dst mem);
 5567 
 5568   ins_cost(110); // XXX
 5569   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 5570   ins_encode %{
 5571     __ leaq($dst$$Register, $mem$$Address);
 5572   %}
 5573   ins_pipe(ialu_reg_reg_fat);
 5574 %}
 5575 
 5576 instruct leaP32(rRegP dst, indOffset32 mem)
 5577 %{
 5578   match(Set dst mem);
 5579 
 5580   ins_cost(110);
 5581   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 5582   ins_encode %{
 5583     __ leaq($dst$$Register, $mem$$Address);
 5584   %}
 5585   ins_pipe(ialu_reg_reg_fat);
 5586 %}
 5587 
 5588 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 5589 %{
 5590   match(Set dst mem);
 5591 
 5592   ins_cost(110);
 5593   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 5594   ins_encode %{
 5595     __ leaq($dst$$Register, $mem$$Address);
 5596   %}
 5597   ins_pipe(ialu_reg_reg_fat);
 5598 %}
 5599 
 5600 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 5601 %{
 5602   match(Set dst mem);
 5603 
 5604   ins_cost(110);
 5605   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5606   ins_encode %{
 5607     __ leaq($dst$$Register, $mem$$Address);
 5608   %}
 5609   ins_pipe(ialu_reg_reg_fat);
 5610 %}
 5611 
 5612 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 5613 %{
 5614   match(Set dst mem);
 5615 
 5616   ins_cost(110);
 5617   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 5618   ins_encode %{
 5619     __ leaq($dst$$Register, $mem$$Address);
 5620   %}
 5621   ins_pipe(ialu_reg_reg_fat);
 5622 %}
 5623 
 5624 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 5625 %{
 5626   match(Set dst mem);
 5627 
 5628   ins_cost(110);
 5629   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 5630   ins_encode %{
 5631     __ leaq($dst$$Register, $mem$$Address);
 5632   %}
 5633   ins_pipe(ialu_reg_reg_fat);
 5634 %}
 5635 
 5636 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 5637 %{
 5638   match(Set dst mem);
 5639 
 5640   ins_cost(110);
 5641   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 5642   ins_encode %{
 5643     __ leaq($dst$$Register, $mem$$Address);
 5644   %}
 5645   ins_pipe(ialu_reg_reg_fat);
 5646 %}
 5647 
 5648 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 5649 %{
 5650   match(Set dst mem);
 5651 
 5652   ins_cost(110);
 5653   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 5654   ins_encode %{
 5655     __ leaq($dst$$Register, $mem$$Address);
 5656   %}
 5657   ins_pipe(ialu_reg_reg_fat);
 5658 %}
 5659 
 5660 // Load Effective Address which uses Narrow (32-bits) oop
 5661 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 5662 %{
 5663   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 5664   match(Set dst mem);
 5665 
 5666   ins_cost(110);
 5667   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 5668   ins_encode %{
 5669     __ leaq($dst$$Register, $mem$$Address);
 5670   %}
 5671   ins_pipe(ialu_reg_reg_fat);
 5672 %}
 5673 
 5674 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 5675 %{
 5676   predicate(CompressedOops::shift() == 0);
 5677   match(Set dst mem);
 5678 
 5679   ins_cost(110); // XXX
 5680   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 5681   ins_encode %{
 5682     __ leaq($dst$$Register, $mem$$Address);
 5683   %}
 5684   ins_pipe(ialu_reg_reg_fat);
 5685 %}
 5686 
 5687 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 5688 %{
 5689   predicate(CompressedOops::shift() == 0);
 5690   match(Set dst mem);
 5691 
 5692   ins_cost(110);
 5693   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 5694   ins_encode %{
 5695     __ leaq($dst$$Register, $mem$$Address);
 5696   %}
 5697   ins_pipe(ialu_reg_reg_fat);
 5698 %}
 5699 
 5700 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 5701 %{
 5702   predicate(CompressedOops::shift() == 0);
 5703   match(Set dst mem);
 5704 
 5705   ins_cost(110);
 5706   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 5707   ins_encode %{
 5708     __ leaq($dst$$Register, $mem$$Address);
 5709   %}
 5710   ins_pipe(ialu_reg_reg_fat);
 5711 %}
 5712 
 5713 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 5714 %{
 5715   predicate(CompressedOops::shift() == 0);
 5716   match(Set dst mem);
 5717 
 5718   ins_cost(110);
 5719   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 5720   ins_encode %{
 5721     __ leaq($dst$$Register, $mem$$Address);
 5722   %}
 5723   ins_pipe(ialu_reg_reg_fat);
 5724 %}
 5725 
 5726 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 5727 %{
 5728   predicate(CompressedOops::shift() == 0);
 5729   match(Set dst mem);
 5730 
 5731   ins_cost(110);
 5732   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 5733   ins_encode %{
 5734     __ leaq($dst$$Register, $mem$$Address);
 5735   %}
 5736   ins_pipe(ialu_reg_reg_fat);
 5737 %}
 5738 
 5739 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 5740 %{
 5741   predicate(CompressedOops::shift() == 0);
 5742   match(Set dst mem);
 5743 
 5744   ins_cost(110);
 5745   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 5746   ins_encode %{
 5747     __ leaq($dst$$Register, $mem$$Address);
 5748   %}
 5749   ins_pipe(ialu_reg_reg_fat);
 5750 %}
 5751 
 5752 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 5753 %{
 5754   predicate(CompressedOops::shift() == 0);
 5755   match(Set dst mem);
 5756 
 5757   ins_cost(110);
 5758   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 5759   ins_encode %{
 5760     __ leaq($dst$$Register, $mem$$Address);
 5761   %}
 5762   ins_pipe(ialu_reg_reg_fat);
 5763 %}
 5764 
 5765 instruct loadConI(rRegI dst, immI src)
 5766 %{
 5767   match(Set dst src);
 5768 
 5769   format %{ "movl    $dst, $src\t# int" %}
 5770   ins_encode %{
 5771     __ movl($dst$$Register, $src$$constant);
 5772   %}
 5773   ins_pipe(ialu_reg_fat); // XXX
 5774 %}
 5775 
 5776 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 5777 %{
 5778   match(Set dst src);
 5779   effect(KILL cr);
 5780 
 5781   ins_cost(50);
 5782   format %{ "xorl    $dst, $dst\t# int" %}
 5783   ins_encode %{
 5784     __ xorl($dst$$Register, $dst$$Register);
 5785   %}
 5786   ins_pipe(ialu_reg);
 5787 %}
 5788 
 5789 instruct loadConL(rRegL dst, immL src)
 5790 %{
 5791   match(Set dst src);
 5792 
 5793   ins_cost(150);
 5794   format %{ "movq    $dst, $src\t# long" %}
 5795   ins_encode %{
 5796     __ mov64($dst$$Register, $src$$constant);
 5797   %}
 5798   ins_pipe(ialu_reg);
 5799 %}
 5800 
 5801 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 5802 %{
 5803   match(Set dst src);
 5804   effect(KILL cr);
 5805 
 5806   ins_cost(50);
 5807   format %{ "xorl    $dst, $dst\t# long" %}
 5808   ins_encode %{
 5809     __ xorl($dst$$Register, $dst$$Register);
 5810   %}
 5811   ins_pipe(ialu_reg); // XXX
 5812 %}
 5813 
 5814 instruct loadConUL32(rRegL dst, immUL32 src)
 5815 %{
 5816   match(Set dst src);
 5817 
 5818   ins_cost(60);
 5819   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 5820   ins_encode %{
 5821     __ movl($dst$$Register, $src$$constant);
 5822   %}
 5823   ins_pipe(ialu_reg);
 5824 %}
 5825 
 5826 instruct loadConL32(rRegL dst, immL32 src)
 5827 %{
 5828   match(Set dst src);
 5829 
 5830   ins_cost(70);
 5831   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 5832   ins_encode %{
 5833     __ movq($dst$$Register, $src$$constant);
 5834   %}
 5835   ins_pipe(ialu_reg);
 5836 %}
 5837 
 5838 instruct loadConP(rRegP dst, immP con) %{
 5839   match(Set dst con);
 5840 
 5841   format %{ "movq    $dst, $con\t# ptr" %}
 5842   ins_encode %{
 5843     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 5844   %}
 5845   ins_pipe(ialu_reg_fat); // XXX
 5846 %}
 5847 
 5848 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 5849 %{
 5850   match(Set dst src);
 5851   effect(KILL cr);
 5852 
 5853   ins_cost(50);
 5854   format %{ "xorl    $dst, $dst\t# ptr" %}
 5855   ins_encode %{
 5856     __ xorl($dst$$Register, $dst$$Register);
 5857   %}
 5858   ins_pipe(ialu_reg);
 5859 %}
 5860 
 5861 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 5862 %{
 5863   match(Set dst src);
 5864   effect(KILL cr);
 5865 
 5866   ins_cost(60);
 5867   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 5868   ins_encode %{
 5869     __ movl($dst$$Register, $src$$constant);
 5870   %}
 5871   ins_pipe(ialu_reg);
 5872 %}
 5873 
 5874 instruct loadConF(regF dst, immF con) %{
 5875   match(Set dst con);
 5876   ins_cost(125);
 5877   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 5878   ins_encode %{
 5879     __ movflt($dst$$XMMRegister, $constantaddress($con));
 5880   %}
 5881   ins_pipe(pipe_slow);
 5882 %}
 5883 
 5884 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 5885   match(Set dst src);
 5886   effect(KILL cr);
 5887   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
 5888   ins_encode %{
 5889     __ xorq($dst$$Register, $dst$$Register);
 5890   %}
 5891   ins_pipe(ialu_reg);
 5892 %}
 5893 
 5894 instruct loadConN(rRegN dst, immN src) %{
 5895   match(Set dst src);
 5896 
 5897   ins_cost(125);
 5898   format %{ "movl    $dst, $src\t# compressed ptr" %}
 5899   ins_encode %{
 5900     address con = (address)$src$$constant;
 5901     if (con == NULL) {
 5902       ShouldNotReachHere();
 5903     } else {
 5904       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 5905     }
 5906   %}
 5907   ins_pipe(ialu_reg_fat); // XXX
 5908 %}
 5909 
 5910 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 5911   match(Set dst src);
 5912 
 5913   ins_cost(125);
 5914   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 5915   ins_encode %{
 5916     address con = (address)$src$$constant;
 5917     if (con == NULL) {
 5918       ShouldNotReachHere();
 5919     } else {
 5920       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 5921     }
 5922   %}
 5923   ins_pipe(ialu_reg_fat); // XXX
 5924 %}
 5925 
 5926 instruct loadConF0(regF dst, immF0 src)
 5927 %{
 5928   match(Set dst src);
 5929   ins_cost(100);
 5930 
 5931   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 5932   ins_encode %{
 5933     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 5934   %}
 5935   ins_pipe(pipe_slow);
 5936 %}
 5937 
 5938 // Use the same format since predicate() can not be used here.
 5939 instruct loadConD(regD dst, immD con) %{
 5940   match(Set dst con);
 5941   ins_cost(125);
 5942   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5943   ins_encode %{
 5944     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5945   %}
 5946   ins_pipe(pipe_slow);
 5947 %}
 5948 
 5949 instruct loadConD0(regD dst, immD0 src)
 5950 %{
 5951   match(Set dst src);
 5952   ins_cost(100);
 5953 
 5954   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5955   ins_encode %{
 5956     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
 5957   %}
 5958   ins_pipe(pipe_slow);
 5959 %}
 5960 
 5961 instruct loadSSI(rRegI dst, stackSlotI src)
 5962 %{
 5963   match(Set dst src);
 5964 
 5965   ins_cost(125);
 5966   format %{ "movl    $dst, $src\t# int stk" %}
 5967   opcode(0x8B);
 5968   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
 5969   ins_pipe(ialu_reg_mem);
 5970 %}
 5971 
 5972 instruct loadSSL(rRegL dst, stackSlotL src)
 5973 %{
 5974   match(Set dst src);
 5975 
 5976   ins_cost(125);
 5977   format %{ "movq    $dst, $src\t# long stk" %}
 5978   opcode(0x8B);
 5979   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5980   ins_pipe(ialu_reg_mem);
 5981 %}
 5982 
 5983 instruct loadSSP(rRegP dst, stackSlotP src)
 5984 %{
 5985   match(Set dst src);
 5986 
 5987   ins_cost(125);
 5988   format %{ "movq    $dst, $src\t# ptr stk" %}
 5989   opcode(0x8B);
 5990   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
 5991   ins_pipe(ialu_reg_mem);
 5992 %}
 5993 
 5994 instruct loadSSF(regF dst, stackSlotF src)
 5995 %{
 5996   match(Set dst src);
 5997 
 5998   ins_cost(125);
 5999   format %{ "movss   $dst, $src\t# float stk" %}
 6000   ins_encode %{
 6001     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 6002   %}
 6003   ins_pipe(pipe_slow); // XXX
 6004 %}
 6005 
 6006 // Use the same format since predicate() can not be used here.
 6007 instruct loadSSD(regD dst, stackSlotD src)
 6008 %{
 6009   match(Set dst src);
 6010 
 6011   ins_cost(125);
 6012   format %{ "movsd   $dst, $src\t# double stk" %}
 6013   ins_encode  %{
 6014     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 6015   %}
 6016   ins_pipe(pipe_slow); // XXX
 6017 %}
 6018 
 6019 // Prefetch instructions for allocation.
 6020 // Must be safe to execute with invalid address (cannot fault).
 6021 
 6022 instruct prefetchAlloc( memory mem ) %{
 6023   predicate(AllocatePrefetchInstr==3);
 6024   match(PrefetchAllocation mem);
 6025   ins_cost(125);
 6026 
 6027   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 6028   ins_encode %{
 6029     __ prefetchw($mem$$Address);
 6030   %}
 6031   ins_pipe(ialu_mem);
 6032 %}
 6033 
 6034 instruct prefetchAllocNTA( memory mem ) %{
 6035   predicate(AllocatePrefetchInstr==0);
 6036   match(PrefetchAllocation mem);
 6037   ins_cost(125);
 6038 
 6039   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 6040   ins_encode %{
 6041     __ prefetchnta($mem$$Address);
 6042   %}
 6043   ins_pipe(ialu_mem);
 6044 %}
 6045 
 6046 instruct prefetchAllocT0( memory mem ) %{
 6047   predicate(AllocatePrefetchInstr==1);
 6048   match(PrefetchAllocation mem);
 6049   ins_cost(125);
 6050 
 6051   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 6052   ins_encode %{
 6053     __ prefetcht0($mem$$Address);
 6054   %}
 6055   ins_pipe(ialu_mem);
 6056 %}
 6057 
 6058 instruct prefetchAllocT2( memory mem ) %{
 6059   predicate(AllocatePrefetchInstr==2);
 6060   match(PrefetchAllocation mem);
 6061   ins_cost(125);
 6062 
 6063   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 6064   ins_encode %{
 6065     __ prefetcht2($mem$$Address);
 6066   %}
 6067   ins_pipe(ialu_mem);
 6068 %}
 6069 
 6070 //----------Store Instructions-------------------------------------------------
 6071 
 6072 // Store Byte
 6073 instruct storeB(memory mem, rRegI src)
 6074 %{
 6075   match(Set mem (StoreB mem src));
 6076 
 6077   ins_cost(125); // XXX
 6078   format %{ "movb    $mem, $src\t# byte" %}
 6079   ins_encode %{
 6080     __ movb($mem$$Address, $src$$Register);
 6081   %}
 6082   ins_pipe(ialu_mem_reg);
 6083 %}
 6084 
 6085 // Store Char/Short
 6086 instruct storeC(memory mem, rRegI src)
 6087 %{
 6088   match(Set mem (StoreC mem src));
 6089 
 6090   ins_cost(125); // XXX
 6091   format %{ "movw    $mem, $src\t# char/short" %}
 6092   ins_encode %{
 6093     __ movw($mem$$Address, $src$$Register);
 6094   %}
 6095   ins_pipe(ialu_mem_reg);
 6096 %}
 6097 
 6098 // Store Integer
 6099 instruct storeI(memory mem, rRegI src)
 6100 %{
 6101   match(Set mem (StoreI mem src));
 6102 
 6103   ins_cost(125); // XXX
 6104   format %{ "movl    $mem, $src\t# int" %}
 6105   ins_encode %{
 6106     __ movl($mem$$Address, $src$$Register);
 6107   %}
 6108   ins_pipe(ialu_mem_reg);
 6109 %}
 6110 
 6111 // Store Long
 6112 instruct storeL(memory mem, rRegL src)
 6113 %{
 6114   match(Set mem (StoreL mem src));
 6115 
 6116   ins_cost(125); // XXX
 6117   format %{ "movq    $mem, $src\t# long" %}
 6118   ins_encode %{
 6119     __ movq($mem$$Address, $src$$Register);
 6120   %}
 6121   ins_pipe(ialu_mem_reg); // XXX
 6122 %}
 6123 
 6124 // Store Pointer
 6125 instruct storeP(memory mem, any_RegP src)
 6126 %{
 6127   match(Set mem (StoreP mem src));
 6128 
 6129   ins_cost(125); // XXX
 6130   format %{ "movq    $mem, $src\t# ptr" %}
 6131   ins_encode %{
 6132     __ movq($mem$$Address, $src$$Register);
 6133   %}
 6134   ins_pipe(ialu_mem_reg);
 6135 %}
 6136 
 6137 instruct storeImmP0(memory mem, immP0 zero)
 6138 %{
 6139   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6140   match(Set mem (StoreP mem zero));
 6141 
 6142   ins_cost(125); // XXX
 6143   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 6144   ins_encode %{
 6145     __ movq($mem$$Address, r12);
 6146   %}
 6147   ins_pipe(ialu_mem_reg);
 6148 %}
 6149 
 6150 // Store NULL Pointer, mark word, or other simple pointer constant.
 6151 instruct storeImmP(memory mem, immP31 src)
 6152 %{
 6153   match(Set mem (StoreP mem src));
 6154 
 6155   ins_cost(150); // XXX
 6156   format %{ "movq    $mem, $src\t# ptr" %}
 6157   ins_encode %{
 6158     __ movq($mem$$Address, $src$$constant);
 6159   %}
 6160   ins_pipe(ialu_mem_imm);
 6161 %}
 6162 
 6163 // Store Compressed Pointer
 6164 instruct storeN(memory mem, rRegN src)
 6165 %{
 6166   match(Set mem (StoreN mem src));
 6167 
 6168   ins_cost(125); // XXX
 6169   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6170   ins_encode %{
 6171     __ movl($mem$$Address, $src$$Register);
 6172   %}
 6173   ins_pipe(ialu_mem_reg);
 6174 %}
 6175 
 6176 instruct storeNKlass(memory mem, rRegN src)
 6177 %{
 6178   match(Set mem (StoreNKlass mem src));
 6179 
 6180   ins_cost(125); // XXX
 6181   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6182   ins_encode %{
 6183     __ movl($mem$$Address, $src$$Register);
 6184   %}
 6185   ins_pipe(ialu_mem_reg);
 6186 %}
 6187 
 6188 instruct storeImmN0(memory mem, immN0 zero)
 6189 %{
 6190   predicate(CompressedOops::base() == NULL);
 6191   match(Set mem (StoreN mem zero));
 6192 
 6193   ins_cost(125); // XXX
 6194   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 6195   ins_encode %{
 6196     __ movl($mem$$Address, r12);
 6197   %}
 6198   ins_pipe(ialu_mem_reg);
 6199 %}
 6200 
 6201 instruct storeImmN(memory mem, immN src)
 6202 %{
 6203   match(Set mem (StoreN mem src));
 6204 
 6205   ins_cost(150); // XXX
 6206   format %{ "movl    $mem, $src\t# compressed ptr" %}
 6207   ins_encode %{
 6208     address con = (address)$src$$constant;
 6209     if (con == NULL) {
 6210       __ movl($mem$$Address, (int32_t)0);
 6211     } else {
 6212       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 6213     }
 6214   %}
 6215   ins_pipe(ialu_mem_imm);
 6216 %}
 6217 
 6218 instruct storeImmNKlass(memory mem, immNKlass src)
 6219 %{
 6220   match(Set mem (StoreNKlass mem src));
 6221 
 6222   ins_cost(150); // XXX
 6223   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 6224   ins_encode %{
 6225     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 6226   %}
 6227   ins_pipe(ialu_mem_imm);
 6228 %}
 6229 
 6230 // Store Integer Immediate
 6231 instruct storeImmI0(memory mem, immI_0 zero)
 6232 %{
 6233   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6234   match(Set mem (StoreI mem zero));
 6235 
 6236   ins_cost(125); // XXX
 6237   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 6238   ins_encode %{
 6239     __ movl($mem$$Address, r12);
 6240   %}
 6241   ins_pipe(ialu_mem_reg);
 6242 %}
 6243 
 6244 instruct storeImmI(memory mem, immI src)
 6245 %{
 6246   match(Set mem (StoreI mem src));
 6247 
 6248   ins_cost(150);
 6249   format %{ "movl    $mem, $src\t# int" %}
 6250   ins_encode %{
 6251     __ movl($mem$$Address, $src$$constant);
 6252   %}
 6253   ins_pipe(ialu_mem_imm);
 6254 %}
 6255 
 6256 // Store Long Immediate
 6257 instruct storeImmL0(memory mem, immL0 zero)
 6258 %{
 6259   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6260   match(Set mem (StoreL mem zero));
 6261 
 6262   ins_cost(125); // XXX
 6263   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 6264   ins_encode %{
 6265     __ movq($mem$$Address, r12);
 6266   %}
 6267   ins_pipe(ialu_mem_reg);
 6268 %}
 6269 
 6270 instruct storeImmL(memory mem, immL32 src)
 6271 %{
 6272   match(Set mem (StoreL mem src));
 6273 
 6274   ins_cost(150);
 6275   format %{ "movq    $mem, $src\t# long" %}
 6276   ins_encode %{
 6277     __ movq($mem$$Address, $src$$constant);
 6278   %}
 6279   ins_pipe(ialu_mem_imm);
 6280 %}
 6281 
 6282 // Store Short/Char Immediate
 6283 instruct storeImmC0(memory mem, immI_0 zero)
 6284 %{
 6285   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6286   match(Set mem (StoreC mem zero));
 6287 
 6288   ins_cost(125); // XXX
 6289   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6290   ins_encode %{
 6291     __ movw($mem$$Address, r12);
 6292   %}
 6293   ins_pipe(ialu_mem_reg);
 6294 %}
 6295 
 6296 instruct storeImmI16(memory mem, immI16 src)
 6297 %{
 6298   predicate(UseStoreImmI16);
 6299   match(Set mem (StoreC mem src));
 6300 
 6301   ins_cost(150);
 6302   format %{ "movw    $mem, $src\t# short/char" %}
 6303   ins_encode %{
 6304     __ movw($mem$$Address, $src$$constant);
 6305   %}
 6306   ins_pipe(ialu_mem_imm);
 6307 %}
 6308 
 6309 // Store Byte Immediate
 6310 instruct storeImmB0(memory mem, immI_0 zero)
 6311 %{
 6312   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6313   match(Set mem (StoreB mem zero));
 6314 
 6315   ins_cost(125); // XXX
 6316   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 6317   ins_encode %{
 6318     __ movb($mem$$Address, r12);
 6319   %}
 6320   ins_pipe(ialu_mem_reg);
 6321 %}
 6322 
 6323 instruct storeImmB(memory mem, immI8 src)
 6324 %{
 6325   match(Set mem (StoreB mem src));
 6326 
 6327   ins_cost(150); // XXX
 6328   format %{ "movb    $mem, $src\t# byte" %}
 6329   ins_encode %{
 6330     __ movb($mem$$Address, $src$$constant);
 6331   %}
 6332   ins_pipe(ialu_mem_imm);
 6333 %}
 6334 
 6335 // Store CMS card-mark Immediate
 6336 instruct storeImmCM0_reg(memory mem, immI_0 zero)
 6337 %{
 6338   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6339   match(Set mem (StoreCM mem zero));
 6340 
 6341   ins_cost(125); // XXX
 6342   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
 6343   ins_encode %{
 6344     __ movb($mem$$Address, r12);
 6345   %}
 6346   ins_pipe(ialu_mem_reg);
 6347 %}
 6348 
 6349 instruct storeImmCM0(memory mem, immI_0 src)
 6350 %{
 6351   match(Set mem (StoreCM mem src));
 6352 
 6353   ins_cost(150); // XXX
 6354   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
 6355   ins_encode %{
 6356     __ movb($mem$$Address, $src$$constant);
 6357   %}
 6358   ins_pipe(ialu_mem_imm);
 6359 %}
 6360 
 6361 // Store Float
 6362 instruct storeF(memory mem, regF src)
 6363 %{
 6364   match(Set mem (StoreF mem src));
 6365 
 6366   ins_cost(95); // XXX
 6367   format %{ "movss   $mem, $src\t# float" %}
 6368   ins_encode %{
 6369     __ movflt($mem$$Address, $src$$XMMRegister);
 6370   %}
 6371   ins_pipe(pipe_slow); // XXX
 6372 %}
 6373 
 6374 // Store immediate Float value (it is faster than store from XMM register)
 6375 instruct storeF0(memory mem, immF0 zero)
 6376 %{
 6377   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6378   match(Set mem (StoreF mem zero));
 6379 
 6380   ins_cost(25); // XXX
 6381   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 6382   ins_encode %{
 6383     __ movl($mem$$Address, r12);
 6384   %}
 6385   ins_pipe(ialu_mem_reg);
 6386 %}
 6387 
 6388 instruct storeF_imm(memory mem, immF src)
 6389 %{
 6390   match(Set mem (StoreF mem src));
 6391 
 6392   ins_cost(50);
 6393   format %{ "movl    $mem, $src\t# float" %}
 6394   ins_encode %{
 6395     __ movl($mem$$Address, jint_cast($src$$constant));
 6396   %}
 6397   ins_pipe(ialu_mem_imm);
 6398 %}
 6399 
 6400 // Store Double
 6401 instruct storeD(memory mem, regD src)
 6402 %{
 6403   match(Set mem (StoreD mem src));
 6404 
 6405   ins_cost(95); // XXX
 6406   format %{ "movsd   $mem, $src\t# double" %}
 6407   ins_encode %{
 6408     __ movdbl($mem$$Address, $src$$XMMRegister);
 6409   %}
 6410   ins_pipe(pipe_slow); // XXX
 6411 %}
 6412 
 6413 // Store immediate double 0.0 (it is faster than store from XMM register)
 6414 instruct storeD0_imm(memory mem, immD0 src)
 6415 %{
 6416   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
 6417   match(Set mem (StoreD mem src));
 6418 
 6419   ins_cost(50);
 6420   format %{ "movq    $mem, $src\t# double 0." %}
 6421   ins_encode %{
 6422     __ movq($mem$$Address, $src$$constant);
 6423   %}
 6424   ins_pipe(ialu_mem_imm);
 6425 %}
 6426 
 6427 instruct storeD0(memory mem, immD0 zero)
 6428 %{
 6429   predicate(UseCompressedOops && (CompressedOops::base() == NULL));
 6430   match(Set mem (StoreD mem zero));
 6431 
 6432   ins_cost(25); // XXX
 6433   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 6434   ins_encode %{
 6435     __ movq($mem$$Address, r12);
 6436   %}
 6437   ins_pipe(ialu_mem_reg);
 6438 %}
 6439 
 6440 instruct storeSSI(stackSlotI dst, rRegI src)
 6441 %{
 6442   match(Set dst src);
 6443 
 6444   ins_cost(100);
 6445   format %{ "movl    $dst, $src\t# int stk" %}
 6446   opcode(0x89);
 6447   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
 6448   ins_pipe( ialu_mem_reg );
 6449 %}
 6450 
 6451 instruct storeSSL(stackSlotL dst, rRegL src)
 6452 %{
 6453   match(Set dst src);
 6454 
 6455   ins_cost(100);
 6456   format %{ "movq    $dst, $src\t# long stk" %}
 6457   opcode(0x89);
 6458   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6459   ins_pipe(ialu_mem_reg);
 6460 %}
 6461 
 6462 instruct storeSSP(stackSlotP dst, rRegP src)
 6463 %{
 6464   match(Set dst src);
 6465 
 6466   ins_cost(100);
 6467   format %{ "movq    $dst, $src\t# ptr stk" %}
 6468   opcode(0x89);
 6469   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
 6470   ins_pipe(ialu_mem_reg);
 6471 %}
 6472 
 6473 instruct storeSSF(stackSlotF dst, regF src)
 6474 %{
 6475   match(Set dst src);
 6476 
 6477   ins_cost(95); // XXX
 6478   format %{ "movss   $dst, $src\t# float stk" %}
 6479   ins_encode %{
 6480     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6481   %}
 6482   ins_pipe(pipe_slow); // XXX
 6483 %}
 6484 
 6485 instruct storeSSD(stackSlotD dst, regD src)
 6486 %{
 6487   match(Set dst src);
 6488 
 6489   ins_cost(95); // XXX
 6490   format %{ "movsd   $dst, $src\t# double stk" %}
 6491   ins_encode %{
 6492     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 6493   %}
 6494   ins_pipe(pipe_slow); // XXX
 6495 %}
 6496 
 6497 instruct cacheWB(indirect addr)
 6498 %{
 6499   predicate(VM_Version::supports_data_cache_line_flush());
 6500   match(CacheWB addr);
 6501 
 6502   ins_cost(100);
 6503   format %{"cache wb $addr" %}
 6504   ins_encode %{
 6505     assert($addr->index_position() < 0, "should be");
 6506     assert($addr$$disp == 0, "should be");
 6507     __ cache_wb(Address($addr$$base$$Register, 0));
 6508   %}
 6509   ins_pipe(pipe_slow); // XXX
 6510 %}
 6511 
 6512 instruct cacheWBPreSync()
 6513 %{
 6514   predicate(VM_Version::supports_data_cache_line_flush());
 6515   match(CacheWBPreSync);
 6516 
 6517   ins_cost(100);
 6518   format %{"cache wb presync" %}
 6519   ins_encode %{
 6520     __ cache_wbsync(true);
 6521   %}
 6522   ins_pipe(pipe_slow); // XXX
 6523 %}
 6524 
 6525 instruct cacheWBPostSync()
 6526 %{
 6527   predicate(VM_Version::supports_data_cache_line_flush());
 6528   match(CacheWBPostSync);
 6529 
 6530   ins_cost(100);
 6531   format %{"cache wb postsync" %}
 6532   ins_encode %{
 6533     __ cache_wbsync(false);
 6534   %}
 6535   ins_pipe(pipe_slow); // XXX
 6536 %}
 6537 
 6538 //----------BSWAP Instructions-------------------------------------------------
 6539 instruct bytes_reverse_int(rRegI dst) %{
 6540   match(Set dst (ReverseBytesI dst));
 6541 
 6542   format %{ "bswapl  $dst" %}
 6543   ins_encode %{
 6544     __ bswapl($dst$$Register);
 6545   %}
 6546   ins_pipe( ialu_reg );
 6547 %}
 6548 
 6549 instruct bytes_reverse_long(rRegL dst) %{
 6550   match(Set dst (ReverseBytesL dst));
 6551 
 6552   format %{ "bswapq  $dst" %}
 6553   ins_encode %{
 6554     __ bswapq($dst$$Register);
 6555   %}
 6556   ins_pipe( ialu_reg);
 6557 %}
 6558 
 6559 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 6560   match(Set dst (ReverseBytesUS dst));
 6561   effect(KILL cr);
 6562 
 6563   format %{ "bswapl  $dst\n\t"
 6564             "shrl    $dst,16\n\t" %}
 6565   ins_encode %{
 6566     __ bswapl($dst$$Register);
 6567     __ shrl($dst$$Register, 16);
 6568   %}
 6569   ins_pipe( ialu_reg );
 6570 %}
 6571 
 6572 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 6573   match(Set dst (ReverseBytesS dst));
 6574   effect(KILL cr);
 6575 
 6576   format %{ "bswapl  $dst\n\t"
 6577             "sar     $dst,16\n\t" %}
 6578   ins_encode %{
 6579     __ bswapl($dst$$Register);
 6580     __ sarl($dst$$Register, 16);
 6581   %}
 6582   ins_pipe( ialu_reg );
 6583 %}
 6584 
 6585 //---------- Zeros Count Instructions ------------------------------------------
 6586 
 6587 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6588   predicate(UseCountLeadingZerosInstruction);
 6589   match(Set dst (CountLeadingZerosI src));
 6590   effect(KILL cr);
 6591 
 6592   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 6593   ins_encode %{
 6594     __ lzcntl($dst$$Register, $src$$Register);
 6595   %}
 6596   ins_pipe(ialu_reg);
 6597 %}
 6598 
 6599 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 6600   predicate(!UseCountLeadingZerosInstruction);
 6601   match(Set dst (CountLeadingZerosI src));
 6602   effect(KILL cr);
 6603 
 6604   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 6605             "jnz     skip\n\t"
 6606             "movl    $dst, -1\n"
 6607       "skip:\n\t"
 6608             "negl    $dst\n\t"
 6609             "addl    $dst, 31" %}
 6610   ins_encode %{
 6611     Register Rdst = $dst$$Register;
 6612     Register Rsrc = $src$$Register;
 6613     Label skip;
 6614     __ bsrl(Rdst, Rsrc);
 6615     __ jccb(Assembler::notZero, skip);
 6616     __ movl(Rdst, -1);
 6617     __ bind(skip);
 6618     __ negl(Rdst);
 6619     __ addl(Rdst, BitsPerInt - 1);
 6620   %}
 6621   ins_pipe(ialu_reg);
 6622 %}
 6623 
 6624 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6625   predicate(UseCountLeadingZerosInstruction);
 6626   match(Set dst (CountLeadingZerosL src));
 6627   effect(KILL cr);
 6628 
 6629   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 6630   ins_encode %{
 6631     __ lzcntq($dst$$Register, $src$$Register);
 6632   %}
 6633   ins_pipe(ialu_reg);
 6634 %}
 6635 
 6636 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 6637   predicate(!UseCountLeadingZerosInstruction);
 6638   match(Set dst (CountLeadingZerosL src));
 6639   effect(KILL cr);
 6640 
 6641   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 6642             "jnz     skip\n\t"
 6643             "movl    $dst, -1\n"
 6644       "skip:\n\t"
 6645             "negl    $dst\n\t"
 6646             "addl    $dst, 63" %}
 6647   ins_encode %{
 6648     Register Rdst = $dst$$Register;
 6649     Register Rsrc = $src$$Register;
 6650     Label skip;
 6651     __ bsrq(Rdst, Rsrc);
 6652     __ jccb(Assembler::notZero, skip);
 6653     __ movl(Rdst, -1);
 6654     __ bind(skip);
 6655     __ negl(Rdst);
 6656     __ addl(Rdst, BitsPerLong - 1);
 6657   %}
 6658   ins_pipe(ialu_reg);
 6659 %}
 6660 
 6661 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6662   predicate(UseCountTrailingZerosInstruction);
 6663   match(Set dst (CountTrailingZerosI src));
 6664   effect(KILL cr);
 6665 
 6666   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 6667   ins_encode %{
 6668     __ tzcntl($dst$$Register, $src$$Register);
 6669   %}
 6670   ins_pipe(ialu_reg);
 6671 %}
 6672 
 6673 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 6674   predicate(!UseCountTrailingZerosInstruction);
 6675   match(Set dst (CountTrailingZerosI src));
 6676   effect(KILL cr);
 6677 
 6678   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 6679             "jnz     done\n\t"
 6680             "movl    $dst, 32\n"
 6681       "done:" %}
 6682   ins_encode %{
 6683     Register Rdst = $dst$$Register;
 6684     Label done;
 6685     __ bsfl(Rdst, $src$$Register);
 6686     __ jccb(Assembler::notZero, done);
 6687     __ movl(Rdst, BitsPerInt);
 6688     __ bind(done);
 6689   %}
 6690   ins_pipe(ialu_reg);
 6691 %}
 6692 
 6693 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6694   predicate(UseCountTrailingZerosInstruction);
 6695   match(Set dst (CountTrailingZerosL src));
 6696   effect(KILL cr);
 6697 
 6698   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 6699   ins_encode %{
 6700     __ tzcntq($dst$$Register, $src$$Register);
 6701   %}
 6702   ins_pipe(ialu_reg);
 6703 %}
 6704 
 6705 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 6706   predicate(!UseCountTrailingZerosInstruction);
 6707   match(Set dst (CountTrailingZerosL src));
 6708   effect(KILL cr);
 6709 
 6710   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 6711             "jnz     done\n\t"
 6712             "movl    $dst, 64\n"
 6713       "done:" %}
 6714   ins_encode %{
 6715     Register Rdst = $dst$$Register;
 6716     Label done;
 6717     __ bsfq(Rdst, $src$$Register);
 6718     __ jccb(Assembler::notZero, done);
 6719     __ movl(Rdst, BitsPerLong);
 6720     __ bind(done);
 6721   %}
 6722   ins_pipe(ialu_reg);
 6723 %}
 6724 
 6725 
 6726 //---------- Population Count Instructions -------------------------------------
 6727 
 6728 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 6729   predicate(UsePopCountInstruction);
 6730   match(Set dst (PopCountI src));
 6731   effect(KILL cr);
 6732 
 6733   format %{ "popcnt  $dst, $src" %}
 6734   ins_encode %{
 6735     __ popcntl($dst$$Register, $src$$Register);
 6736   %}
 6737   ins_pipe(ialu_reg);
 6738 %}
 6739 
 6740 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6741   predicate(UsePopCountInstruction);
 6742   match(Set dst (PopCountI (LoadI mem)));
 6743   effect(KILL cr);
 6744 
 6745   format %{ "popcnt  $dst, $mem" %}
 6746   ins_encode %{
 6747     __ popcntl($dst$$Register, $mem$$Address);
 6748   %}
 6749   ins_pipe(ialu_reg);
 6750 %}
 6751 
 6752 // Note: Long.bitCount(long) returns an int.
 6753 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 6754   predicate(UsePopCountInstruction);
 6755   match(Set dst (PopCountL src));
 6756   effect(KILL cr);
 6757 
 6758   format %{ "popcnt  $dst, $src" %}
 6759   ins_encode %{
 6760     __ popcntq($dst$$Register, $src$$Register);
 6761   %}
 6762   ins_pipe(ialu_reg);
 6763 %}
 6764 
 6765 // Note: Long.bitCount(long) returns an int.
 6766 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 6767   predicate(UsePopCountInstruction);
 6768   match(Set dst (PopCountL (LoadL mem)));
 6769   effect(KILL cr);
 6770 
 6771   format %{ "popcnt  $dst, $mem" %}
 6772   ins_encode %{
 6773     __ popcntq($dst$$Register, $mem$$Address);
 6774   %}
 6775   ins_pipe(ialu_reg);
 6776 %}
 6777 
 6778 
 6779 //----------MemBar Instructions-----------------------------------------------
 6780 // Memory barrier flavors
 6781 
 6782 instruct membar_acquire()
 6783 %{
 6784   match(MemBarAcquire);
 6785   match(LoadFence);
 6786   ins_cost(0);
 6787 
 6788   size(0);
 6789   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 6790   ins_encode();
 6791   ins_pipe(empty);
 6792 %}
 6793 
 6794 instruct membar_acquire_lock()
 6795 %{
 6796   match(MemBarAcquireLock);
 6797   ins_cost(0);
 6798 
 6799   size(0);
 6800   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 6801   ins_encode();
 6802   ins_pipe(empty);
 6803 %}
 6804 
 6805 instruct membar_release()
 6806 %{
 6807   match(MemBarRelease);
 6808   match(StoreFence);
 6809   ins_cost(0);
 6810 
 6811   size(0);
 6812   format %{ "MEMBAR-release ! (empty encoding)" %}
 6813   ins_encode();
 6814   ins_pipe(empty);
 6815 %}
 6816 
 6817 instruct membar_release_lock()
 6818 %{
 6819   match(MemBarReleaseLock);
 6820   ins_cost(0);
 6821 
 6822   size(0);
 6823   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 6824   ins_encode();
 6825   ins_pipe(empty);
 6826 %}
 6827 
 6828 instruct membar_volatile(rFlagsReg cr) %{
 6829   match(MemBarVolatile);
 6830   effect(KILL cr);
 6831   ins_cost(400);
 6832 
 6833   format %{
 6834     $$template
 6835     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 6836   %}
 6837   ins_encode %{
 6838     __ membar(Assembler::StoreLoad);
 6839   %}
 6840   ins_pipe(pipe_slow);
 6841 %}
 6842 
 6843 instruct unnecessary_membar_volatile()
 6844 %{
 6845   match(MemBarVolatile);
 6846   predicate(Matcher::post_store_load_barrier(n));
 6847   ins_cost(0);
 6848 
 6849   size(0);
 6850   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 6851   ins_encode();
 6852   ins_pipe(empty);
 6853 %}
 6854 
 6855 instruct membar_storestore() %{
 6856   match(MemBarStoreStore);
 6857   match(StoreStoreFence);
 6858   ins_cost(0);
 6859 
 6860   size(0);
 6861   format %{ "MEMBAR-storestore (empty encoding)" %}
 6862   ins_encode( );
 6863   ins_pipe(empty);
 6864 %}
 6865 
 6866 //----------Move Instructions--------------------------------------------------
 6867 
 6868 instruct castX2P(rRegP dst, rRegL src)
 6869 %{
 6870   match(Set dst (CastX2P src));
 6871 
 6872   format %{ "movq    $dst, $src\t# long->ptr" %}
 6873   ins_encode %{
 6874     if ($dst$$reg != $src$$reg) {
 6875       __ movptr($dst$$Register, $src$$Register);
 6876     }
 6877   %}
 6878   ins_pipe(ialu_reg_reg); // XXX
 6879 %}
 6880 
 6881 instruct castN2X(rRegL dst, rRegN src)
 6882 %{
 6883   match(Set dst (CastP2X src));
 6884 
 6885   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6886   ins_encode %{
 6887     if ($dst$$reg != $src$$reg) {
 6888       __ movptr($dst$$Register, $src$$Register);
 6889     }
 6890   %}
 6891   ins_pipe(ialu_reg_reg); // XXX
 6892 %}
 6893 
 6894 instruct castP2X(rRegL dst, rRegP src)
 6895 %{
 6896   match(Set dst (CastP2X src));
 6897 
 6898   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6899   ins_encode %{
 6900     if ($dst$$reg != $src$$reg) {
 6901       __ movptr($dst$$Register, $src$$Register);
 6902     }
 6903   %}
 6904   ins_pipe(ialu_reg_reg); // XXX
 6905 %}
 6906 
 6907 // Convert oop into int for vectors alignment masking
 6908 instruct convP2I(rRegI dst, rRegP src)
 6909 %{
 6910   match(Set dst (ConvL2I (CastP2X src)));
 6911 
 6912   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6913   ins_encode %{
 6914     __ movl($dst$$Register, $src$$Register);
 6915   %}
 6916   ins_pipe(ialu_reg_reg); // XXX
 6917 %}
 6918 
 6919 // Convert compressed oop into int for vectors alignment masking
 6920 // in case of 32bit oops (heap < 4Gb).
 6921 instruct convN2I(rRegI dst, rRegN src)
 6922 %{
 6923   predicate(CompressedOops::shift() == 0);
 6924   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6925 
 6926   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6927   ins_encode %{
 6928     __ movl($dst$$Register, $src$$Register);
 6929   %}
 6930   ins_pipe(ialu_reg_reg); // XXX
 6931 %}
 6932 
 6933 // Convert oop pointer into compressed form
 6934 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6935   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6936   match(Set dst (EncodeP src));
 6937   effect(KILL cr);
 6938   format %{ "encode_heap_oop $dst,$src" %}
 6939   ins_encode %{
 6940     Register s = $src$$Register;
 6941     Register d = $dst$$Register;
 6942     if (s != d) {
 6943       __ movq(d, s);
 6944     }
 6945     __ encode_heap_oop(d);
 6946   %}
 6947   ins_pipe(ialu_reg_long);
 6948 %}
 6949 
 6950 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6951   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6952   match(Set dst (EncodeP src));
 6953   effect(KILL cr);
 6954   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6955   ins_encode %{
 6956     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6957   %}
 6958   ins_pipe(ialu_reg_long);
 6959 %}
 6960 
 6961 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6962   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6963             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6964   match(Set dst (DecodeN src));
 6965   effect(KILL cr);
 6966   format %{ "decode_heap_oop $dst,$src" %}
 6967   ins_encode %{
 6968     Register s = $src$$Register;
 6969     Register d = $dst$$Register;
 6970     if (s != d) {
 6971       __ movq(d, s);
 6972     }
 6973     __ decode_heap_oop(d);
 6974   %}
 6975   ins_pipe(ialu_reg_long);
 6976 %}
 6977 
 6978 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6979   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6980             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6981   match(Set dst (DecodeN src));
 6982   effect(KILL cr);
 6983   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6984   ins_encode %{
 6985     Register s = $src$$Register;
 6986     Register d = $dst$$Register;
 6987     if (s != d) {
 6988       __ decode_heap_oop_not_null(d, s);
 6989     } else {
 6990       __ decode_heap_oop_not_null(d);
 6991     }
 6992   %}
 6993   ins_pipe(ialu_reg_long);
 6994 %}
 6995 
 6996 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6997   match(Set dst (EncodePKlass src));
 6998   effect(TEMP dst, KILL cr);
 6999   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 7000   ins_encode %{
 7001     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7002   %}
 7003   ins_pipe(ialu_reg_long);
 7004 %}
 7005 
 7006 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 7007   match(Set dst (DecodeNKlass src));
 7008   effect(TEMP dst, KILL cr);
 7009   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 7010   ins_encode %{
 7011     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 7012   %}
 7013   ins_pipe(ialu_reg_long);
 7014 %}
 7015 
 7016 //----------Conditional Move---------------------------------------------------
 7017 // Jump
 7018 // dummy instruction for generating temp registers
 7019 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 7020   match(Jump (LShiftL switch_val shift));
 7021   ins_cost(350);
 7022   predicate(false);
 7023   effect(TEMP dest);
 7024 
 7025   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7026             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 7027   ins_encode %{
 7028     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7029     // to do that and the compiler is using that register as one it can allocate.
 7030     // So we build it all by hand.
 7031     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 7032     // ArrayAddress dispatch(table, index);
 7033     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 7034     __ lea($dest$$Register, $constantaddress);
 7035     __ jmp(dispatch);
 7036   %}
 7037   ins_pipe(pipe_jmp);
 7038 %}
 7039 
 7040 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 7041   match(Jump (AddL (LShiftL switch_val shift) offset));
 7042   ins_cost(350);
 7043   effect(TEMP dest);
 7044 
 7045   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7046             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 7047   ins_encode %{
 7048     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7049     // to do that and the compiler is using that register as one it can allocate.
 7050     // So we build it all by hand.
 7051     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7052     // ArrayAddress dispatch(table, index);
 7053     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 7054     __ lea($dest$$Register, $constantaddress);
 7055     __ jmp(dispatch);
 7056   %}
 7057   ins_pipe(pipe_jmp);
 7058 %}
 7059 
 7060 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 7061   match(Jump switch_val);
 7062   ins_cost(350);
 7063   effect(TEMP dest);
 7064 
 7065   format %{ "leaq    $dest, [$constantaddress]\n\t"
 7066             "jmp     [$dest + $switch_val]\n\t" %}
 7067   ins_encode %{
 7068     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 7069     // to do that and the compiler is using that register as one it can allocate.
 7070     // So we build it all by hand.
 7071     // Address index(noreg, switch_reg, Address::times_1);
 7072     // ArrayAddress dispatch(table, index);
 7073     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 7074     __ lea($dest$$Register, $constantaddress);
 7075     __ jmp(dispatch);
 7076   %}
 7077   ins_pipe(pipe_jmp);
 7078 %}
 7079 
 7080 // Conditional move
 7081 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 7082 %{
 7083   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7084 
 7085   ins_cost(200); // XXX
 7086   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7087   ins_encode %{
 7088     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7089   %}
 7090   ins_pipe(pipe_cmov_reg);
 7091 %}
 7092 
 7093 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 7094   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7095 
 7096   ins_cost(200); // XXX
 7097   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7098   ins_encode %{
 7099     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7100   %}
 7101   ins_pipe(pipe_cmov_reg);
 7102 %}
 7103 
 7104 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 7105   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 7106   ins_cost(200);
 7107   expand %{
 7108     cmovI_regU(cop, cr, dst, src);
 7109   %}
 7110 %}
 7111 
 7112 // Conditional move
 7113 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 7114   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7115 
 7116   ins_cost(250); // XXX
 7117   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 7118   ins_encode %{
 7119     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7120   %}
 7121   ins_pipe(pipe_cmov_mem);
 7122 %}
 7123 
 7124 // Conditional move
 7125 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 7126 %{
 7127   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7128 
 7129   ins_cost(250); // XXX
 7130   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 7131   ins_encode %{
 7132     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7133   %}
 7134   ins_pipe(pipe_cmov_mem);
 7135 %}
 7136 
 7137 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 7138   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 7139   ins_cost(250);
 7140   expand %{
 7141     cmovI_memU(cop, cr, dst, src);
 7142   %}
 7143 %}
 7144 
 7145 // Conditional move
 7146 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 7147 %{
 7148   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7149 
 7150   ins_cost(200); // XXX
 7151   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 7152   ins_encode %{
 7153     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7154   %}
 7155   ins_pipe(pipe_cmov_reg);
 7156 %}
 7157 
 7158 // Conditional move
 7159 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 7160 %{
 7161   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7162 
 7163   ins_cost(200); // XXX
 7164   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 7165   ins_encode %{
 7166     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7167   %}
 7168   ins_pipe(pipe_cmov_reg);
 7169 %}
 7170 
 7171 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 7172   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 7173   ins_cost(200);
 7174   expand %{
 7175     cmovN_regU(cop, cr, dst, src);
 7176   %}
 7177 %}
 7178 
 7179 // Conditional move
 7180 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 7181 %{
 7182   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7183 
 7184   ins_cost(200); // XXX
 7185   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 7186   ins_encode %{
 7187     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7188   %}
 7189   ins_pipe(pipe_cmov_reg);  // XXX
 7190 %}
 7191 
 7192 // Conditional move
 7193 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 7194 %{
 7195   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7196 
 7197   ins_cost(200); // XXX
 7198   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 7199   ins_encode %{
 7200     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7201   %}
 7202   ins_pipe(pipe_cmov_reg); // XXX
 7203 %}
 7204 
 7205 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 7206   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 7207   ins_cost(200);
 7208   expand %{
 7209     cmovP_regU(cop, cr, dst, src);
 7210   %}
 7211 %}
 7212 
 7213 // DISABLED: Requires the ADLC to emit a bottom_type call that
 7214 // correctly meets the two pointer arguments; one is an incoming
 7215 // register but the other is a memory operand.  ALSO appears to
 7216 // be buggy with implicit null checks.
 7217 //
 7218 //// Conditional move
 7219 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
 7220 //%{
 7221 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7222 //  ins_cost(250);
 7223 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7224 //  opcode(0x0F,0x40);
 7225 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7226 //  ins_pipe( pipe_cmov_mem );
 7227 //%}
 7228 //
 7229 //// Conditional move
 7230 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
 7231 //%{
 7232 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
 7233 //  ins_cost(250);
 7234 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
 7235 //  opcode(0x0F,0x40);
 7236 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
 7237 //  ins_pipe( pipe_cmov_mem );
 7238 //%}
 7239 
 7240 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 7241 %{
 7242   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7243 
 7244   ins_cost(200); // XXX
 7245   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7246   ins_encode %{
 7247     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7248   %}
 7249   ins_pipe(pipe_cmov_reg);  // XXX
 7250 %}
 7251 
 7252 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 7253 %{
 7254   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7255 
 7256   ins_cost(200); // XXX
 7257   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 7258   ins_encode %{
 7259     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7260   %}
 7261   ins_pipe(pipe_cmov_mem);  // XXX
 7262 %}
 7263 
 7264 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 7265 %{
 7266   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7267 
 7268   ins_cost(200); // XXX
 7269   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7270   ins_encode %{
 7271     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 7272   %}
 7273   ins_pipe(pipe_cmov_reg); // XXX
 7274 %}
 7275 
 7276 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 7277   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 7278   ins_cost(200);
 7279   expand %{
 7280     cmovL_regU(cop, cr, dst, src);
 7281   %}
 7282 %}
 7283 
 7284 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 7285 %{
 7286   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7287 
 7288   ins_cost(200); // XXX
 7289   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 7290   ins_encode %{
 7291     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 7292   %}
 7293   ins_pipe(pipe_cmov_mem); // XXX
 7294 %}
 7295 
 7296 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 7297   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 7298   ins_cost(200);
 7299   expand %{
 7300     cmovL_memU(cop, cr, dst, src);
 7301   %}
 7302 %}
 7303 
 7304 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 7305 %{
 7306   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7307 
 7308   ins_cost(200); // XXX
 7309   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7310             "movss     $dst, $src\n"
 7311     "skip:" %}
 7312   ins_encode %{
 7313     Label Lskip;
 7314     // Invert sense of branch from sense of CMOV
 7315     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7316     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7317     __ bind(Lskip);
 7318   %}
 7319   ins_pipe(pipe_slow);
 7320 %}
 7321 
 7322 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 7323 // %{
 7324 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
 7325 
 7326 //   ins_cost(200); // XXX
 7327 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 7328 //             "movss     $dst, $src\n"
 7329 //     "skip:" %}
 7330 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
 7331 //   ins_pipe(pipe_slow);
 7332 // %}
 7333 
 7334 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 7335 %{
 7336   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7337 
 7338   ins_cost(200); // XXX
 7339   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 7340             "movss     $dst, $src\n"
 7341     "skip:" %}
 7342   ins_encode %{
 7343     Label Lskip;
 7344     // Invert sense of branch from sense of CMOV
 7345     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7346     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 7347     __ bind(Lskip);
 7348   %}
 7349   ins_pipe(pipe_slow);
 7350 %}
 7351 
 7352 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 7353   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 7354   ins_cost(200);
 7355   expand %{
 7356     cmovF_regU(cop, cr, dst, src);
 7357   %}
 7358 %}
 7359 
 7360 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 7361 %{
 7362   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7363 
 7364   ins_cost(200); // XXX
 7365   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 7366             "movsd     $dst, $src\n"
 7367     "skip:" %}
 7368   ins_encode %{
 7369     Label Lskip;
 7370     // Invert sense of branch from sense of CMOV
 7371     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7372     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7373     __ bind(Lskip);
 7374   %}
 7375   ins_pipe(pipe_slow);
 7376 %}
 7377 
 7378 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7379 %{
 7380   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7381 
 7382   ins_cost(200); // XXX
 7383   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7384             "movsd     $dst, $src\n"
 7385     "skip:" %}
 7386   ins_encode %{
 7387     Label Lskip;
 7388     // Invert sense of branch from sense of CMOV
 7389     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7390     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7391     __ bind(Lskip);
 7392   %}
 7393   ins_pipe(pipe_slow);
 7394 %}
 7395 
 7396 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7397   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7398   ins_cost(200);
 7399   expand %{
 7400     cmovD_regU(cop, cr, dst, src);
 7401   %}
 7402 %}
 7403 
 7404 //----------Arithmetic Instructions--------------------------------------------
 7405 //----------Addition Instructions----------------------------------------------
 7406 
 7407 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7408 %{
 7409   match(Set dst (AddI dst src));
 7410   effect(KILL cr);
 7411 
 7412   format %{ "addl    $dst, $src\t# int" %}
 7413   ins_encode %{
 7414     __ addl($dst$$Register, $src$$Register);
 7415   %}
 7416   ins_pipe(ialu_reg_reg);
 7417 %}
 7418 
 7419 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7420 %{
 7421   match(Set dst (AddI dst src));
 7422   effect(KILL cr);
 7423 
 7424   format %{ "addl    $dst, $src\t# int" %}
 7425   ins_encode %{
 7426     __ addl($dst$$Register, $src$$constant);
 7427   %}
 7428   ins_pipe( ialu_reg );
 7429 %}
 7430 
 7431 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7432 %{
 7433   match(Set dst (AddI dst (LoadI src)));
 7434   effect(KILL cr);
 7435 
 7436   ins_cost(125); // XXX
 7437   format %{ "addl    $dst, $src\t# int" %}
 7438   ins_encode %{
 7439     __ addl($dst$$Register, $src$$Address);
 7440   %}
 7441   ins_pipe(ialu_reg_mem);
 7442 %}
 7443 
 7444 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7445 %{
 7446   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7447   effect(KILL cr);
 7448 
 7449   ins_cost(150); // XXX
 7450   format %{ "addl    $dst, $src\t# int" %}
 7451   ins_encode %{
 7452     __ addl($dst$$Address, $src$$Register);
 7453   %}
 7454   ins_pipe(ialu_mem_reg);
 7455 %}
 7456 
 7457 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7458 %{
 7459   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7460   effect(KILL cr);
 7461 
 7462   ins_cost(125); // XXX
 7463   format %{ "addl    $dst, $src\t# int" %}
 7464   ins_encode %{
 7465     __ addl($dst$$Address, $src$$constant);
 7466   %}
 7467   ins_pipe(ialu_mem_imm);
 7468 %}
 7469 
 7470 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7471 %{
 7472   predicate(UseIncDec);
 7473   match(Set dst (AddI dst src));
 7474   effect(KILL cr);
 7475 
 7476   format %{ "incl    $dst\t# int" %}
 7477   ins_encode %{
 7478     __ incrementl($dst$$Register);
 7479   %}
 7480   ins_pipe(ialu_reg);
 7481 %}
 7482 
 7483 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7484 %{
 7485   predicate(UseIncDec);
 7486   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7487   effect(KILL cr);
 7488 
 7489   ins_cost(125); // XXX
 7490   format %{ "incl    $dst\t# int" %}
 7491   ins_encode %{
 7492     __ incrementl($dst$$Address);
 7493   %}
 7494   ins_pipe(ialu_mem_imm);
 7495 %}
 7496 
 7497 // XXX why does that use AddI
 7498 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7499 %{
 7500   predicate(UseIncDec);
 7501   match(Set dst (AddI dst src));
 7502   effect(KILL cr);
 7503 
 7504   format %{ "decl    $dst\t# int" %}
 7505   ins_encode %{
 7506     __ decrementl($dst$$Register);
 7507   %}
 7508   ins_pipe(ialu_reg);
 7509 %}
 7510 
 7511 // XXX why does that use AddI
 7512 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7513 %{
 7514   predicate(UseIncDec);
 7515   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7516   effect(KILL cr);
 7517 
 7518   ins_cost(125); // XXX
 7519   format %{ "decl    $dst\t# int" %}
 7520   ins_encode %{
 7521     __ decrementl($dst$$Address);
 7522   %}
 7523   ins_pipe(ialu_mem_imm);
 7524 %}
 7525 
 7526 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7527 %{
 7528   predicate(VM_Version::supports_fast_2op_lea());
 7529   match(Set dst (AddI (LShiftI index scale) disp));
 7530 
 7531   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7532   ins_encode %{
 7533     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7534     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7535   %}
 7536   ins_pipe(ialu_reg_reg);
 7537 %}
 7538 
 7539 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7540 %{
 7541   predicate(VM_Version::supports_fast_3op_lea());
 7542   match(Set dst (AddI (AddI base index) disp));
 7543 
 7544   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7545   ins_encode %{
 7546     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7547   %}
 7548   ins_pipe(ialu_reg_reg);
 7549 %}
 7550 
 7551 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7552 %{
 7553   predicate(VM_Version::supports_fast_2op_lea());
 7554   match(Set dst (AddI base (LShiftI index scale)));
 7555 
 7556   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7557   ins_encode %{
 7558     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7559     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7560   %}
 7561   ins_pipe(ialu_reg_reg);
 7562 %}
 7563 
 7564 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7565 %{
 7566   predicate(VM_Version::supports_fast_3op_lea());
 7567   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7568 
 7569   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7570   ins_encode %{
 7571     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7572     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7573   %}
 7574   ins_pipe(ialu_reg_reg);
 7575 %}
 7576 
 7577 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7578 %{
 7579   match(Set dst (AddL dst src));
 7580   effect(KILL cr);
 7581 
 7582   format %{ "addq    $dst, $src\t# long" %}
 7583   ins_encode %{
 7584     __ addq($dst$$Register, $src$$Register);
 7585   %}
 7586   ins_pipe(ialu_reg_reg);
 7587 %}
 7588 
 7589 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7590 %{
 7591   match(Set dst (AddL dst src));
 7592   effect(KILL cr);
 7593 
 7594   format %{ "addq    $dst, $src\t# long" %}
 7595   ins_encode %{
 7596     __ addq($dst$$Register, $src$$constant);
 7597   %}
 7598   ins_pipe( ialu_reg );
 7599 %}
 7600 
 7601 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7602 %{
 7603   match(Set dst (AddL dst (LoadL src)));
 7604   effect(KILL cr);
 7605 
 7606   ins_cost(125); // XXX
 7607   format %{ "addq    $dst, $src\t# long" %}
 7608   ins_encode %{
 7609     __ addq($dst$$Register, $src$$Address);
 7610   %}
 7611   ins_pipe(ialu_reg_mem);
 7612 %}
 7613 
 7614 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7615 %{
 7616   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7617   effect(KILL cr);
 7618 
 7619   ins_cost(150); // XXX
 7620   format %{ "addq    $dst, $src\t# long" %}
 7621   ins_encode %{
 7622     __ addq($dst$$Address, $src$$Register);
 7623   %}
 7624   ins_pipe(ialu_mem_reg);
 7625 %}
 7626 
 7627 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7628 %{
 7629   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7630   effect(KILL cr);
 7631 
 7632   ins_cost(125); // XXX
 7633   format %{ "addq    $dst, $src\t# long" %}
 7634   ins_encode %{
 7635     __ addq($dst$$Address, $src$$constant);
 7636   %}
 7637   ins_pipe(ialu_mem_imm);
 7638 %}
 7639 
 7640 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7641 %{
 7642   predicate(UseIncDec);
 7643   match(Set dst (AddL dst src));
 7644   effect(KILL cr);
 7645 
 7646   format %{ "incq    $dst\t# long" %}
 7647   ins_encode %{
 7648     __ incrementq($dst$$Register);
 7649   %}
 7650   ins_pipe(ialu_reg);
 7651 %}
 7652 
 7653 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7654 %{
 7655   predicate(UseIncDec);
 7656   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7657   effect(KILL cr);
 7658 
 7659   ins_cost(125); // XXX
 7660   format %{ "incq    $dst\t# long" %}
 7661   ins_encode %{
 7662     __ incrementq($dst$$Address);
 7663   %}
 7664   ins_pipe(ialu_mem_imm);
 7665 %}
 7666 
 7667 // XXX why does that use AddL
 7668 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7669 %{
 7670   predicate(UseIncDec);
 7671   match(Set dst (AddL dst src));
 7672   effect(KILL cr);
 7673 
 7674   format %{ "decq    $dst\t# long" %}
 7675   ins_encode %{
 7676     __ decrementq($dst$$Register);
 7677   %}
 7678   ins_pipe(ialu_reg);
 7679 %}
 7680 
 7681 // XXX why does that use AddL
 7682 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7683 %{
 7684   predicate(UseIncDec);
 7685   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7686   effect(KILL cr);
 7687 
 7688   ins_cost(125); // XXX
 7689   format %{ "decq    $dst\t# long" %}
 7690   ins_encode %{
 7691     __ decrementq($dst$$Address);
 7692   %}
 7693   ins_pipe(ialu_mem_imm);
 7694 %}
 7695 
 7696 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7697 %{
 7698   predicate(VM_Version::supports_fast_2op_lea());
 7699   match(Set dst (AddL (LShiftL index scale) disp));
 7700 
 7701   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7702   ins_encode %{
 7703     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7704     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7705   %}
 7706   ins_pipe(ialu_reg_reg);
 7707 %}
 7708 
 7709 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7710 %{
 7711   predicate(VM_Version::supports_fast_3op_lea());
 7712   match(Set dst (AddL (AddL base index) disp));
 7713 
 7714   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7715   ins_encode %{
 7716     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7717   %}
 7718   ins_pipe(ialu_reg_reg);
 7719 %}
 7720 
 7721 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7722 %{
 7723   predicate(VM_Version::supports_fast_2op_lea());
 7724   match(Set dst (AddL base (LShiftL index scale)));
 7725 
 7726   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7727   ins_encode %{
 7728     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7729     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7730   %}
 7731   ins_pipe(ialu_reg_reg);
 7732 %}
 7733 
 7734 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7735 %{
 7736   predicate(VM_Version::supports_fast_3op_lea());
 7737   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7738 
 7739   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7740   ins_encode %{
 7741     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7742     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7743   %}
 7744   ins_pipe(ialu_reg_reg);
 7745 %}
 7746 
 7747 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7748 %{
 7749   match(Set dst (AddP dst src));
 7750   effect(KILL cr);
 7751 
 7752   format %{ "addq    $dst, $src\t# ptr" %}
 7753   ins_encode %{
 7754     __ addq($dst$$Register, $src$$Register);
 7755   %}
 7756   ins_pipe(ialu_reg_reg);
 7757 %}
 7758 
 7759 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7760 %{
 7761   match(Set dst (AddP dst src));
 7762   effect(KILL cr);
 7763 
 7764   format %{ "addq    $dst, $src\t# ptr" %}
 7765   ins_encode %{
 7766     __ addq($dst$$Register, $src$$constant);
 7767   %}
 7768   ins_pipe( ialu_reg );
 7769 %}
 7770 
 7771 // XXX addP mem ops ????
 7772 
 7773 instruct checkCastPP(rRegP dst)
 7774 %{
 7775   match(Set dst (CheckCastPP dst));
 7776 
 7777   size(0);
 7778   format %{ "# checkcastPP of $dst" %}
 7779   ins_encode(/* empty encoding */);
 7780   ins_pipe(empty);
 7781 %}
 7782 
 7783 instruct castPP(rRegP dst)
 7784 %{
 7785   match(Set dst (CastPP dst));
 7786 
 7787   size(0);
 7788   format %{ "# castPP of $dst" %}
 7789   ins_encode(/* empty encoding */);
 7790   ins_pipe(empty);
 7791 %}
 7792 
 7793 instruct castII(rRegI dst)
 7794 %{
 7795   match(Set dst (CastII dst));
 7796 
 7797   size(0);
 7798   format %{ "# castII of $dst" %}
 7799   ins_encode(/* empty encoding */);
 7800   ins_cost(0);
 7801   ins_pipe(empty);
 7802 %}
 7803 
 7804 instruct castLL(rRegL dst)
 7805 %{
 7806   match(Set dst (CastLL dst));
 7807 
 7808   size(0);
 7809   format %{ "# castLL of $dst" %}
 7810   ins_encode(/* empty encoding */);
 7811   ins_cost(0);
 7812   ins_pipe(empty);
 7813 %}
 7814 
 7815 instruct castFF(regF dst)
 7816 %{
 7817   match(Set dst (CastFF dst));
 7818 
 7819   size(0);
 7820   format %{ "# castFF of $dst" %}
 7821   ins_encode(/* empty encoding */);
 7822   ins_cost(0);
 7823   ins_pipe(empty);
 7824 %}
 7825 
 7826 instruct castDD(regD dst)
 7827 %{
 7828   match(Set dst (CastDD dst));
 7829 
 7830   size(0);
 7831   format %{ "# castDD of $dst" %}
 7832   ins_encode(/* empty encoding */);
 7833   ins_cost(0);
 7834   ins_pipe(empty);
 7835 %}
 7836 
 7837 // LoadP-locked same as a regular LoadP when used with compare-swap
 7838 instruct loadPLocked(rRegP dst, memory mem)
 7839 %{
 7840   match(Set dst (LoadPLocked mem));
 7841 
 7842   ins_cost(125); // XXX
 7843   format %{ "movq    $dst, $mem\t# ptr locked" %}
 7844   ins_encode %{
 7845     __ movq($dst$$Register, $mem$$Address);
 7846   %}
 7847   ins_pipe(ialu_reg_mem); // XXX
 7848 %}
 7849 
 7850 // Conditional-store of the updated heap-top.
 7851 // Used during allocation of the shared heap.
 7852 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
 7853 
 7854 instruct storePConditional(memory heap_top_ptr,
 7855                            rax_RegP oldval, rRegP newval,
 7856                            rFlagsReg cr)
 7857 %{
 7858   predicate(n->as_LoadStore()->barrier_data() == 0);
 7859   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
 7860 
 7861   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
 7862             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
 7863   ins_encode %{
 7864     __ lock();
 7865     __ cmpxchgq($newval$$Register, $heap_top_ptr$$Address);
 7866   %}
 7867   ins_pipe(pipe_cmpxchg);
 7868 %}
 7869 
 7870 // Conditional-store of an int value.
 7871 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7872 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
 7873 %{
 7874   match(Set cr (StoreIConditional mem (Binary oldval newval)));
 7875   effect(KILL oldval);
 7876 
 7877   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7878   opcode(0x0F, 0xB1);
 7879   ins_encode(lock_prefix,
 7880              REX_reg_mem(newval, mem),
 7881              OpcP, OpcS,
 7882              reg_mem(newval, mem));
 7883   ins_pipe(pipe_cmpxchg);
 7884 %}
 7885 
 7886 // Conditional-store of a long value.
 7887 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
 7888 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
 7889 %{
 7890   match(Set cr (StoreLConditional mem (Binary oldval newval)));
 7891   effect(KILL oldval);
 7892 
 7893   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
 7894   ins_encode %{
 7895     __ lock();
 7896     __ cmpxchgq($newval$$Register, $mem$$Address);
 7897   %}
 7898   ins_pipe(pipe_cmpxchg);
 7899 %}
 7900 
 7901 
 7902 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7903 instruct compareAndSwapP(rRegI res,
 7904                          memory mem_ptr,
 7905                          rax_RegP oldval, rRegP newval,
 7906                          rFlagsReg cr)
 7907 %{
 7908   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 7909   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7910   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7911   effect(KILL cr, KILL oldval);
 7912 
 7913   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7914             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7915             "sete    $res\n\t"
 7916             "movzbl  $res, $res" %}
 7917   ins_encode %{
 7918     __ lock();
 7919     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7920     __ sete($res$$Register);
 7921     __ movzbl($res$$Register, $res$$Register);
 7922   %}
 7923   ins_pipe( pipe_cmpxchg );
 7924 %}
 7925 
 7926 instruct compareAndSwapL(rRegI res,
 7927                          memory mem_ptr,
 7928                          rax_RegL oldval, rRegL newval,
 7929                          rFlagsReg cr)
 7930 %{
 7931   predicate(VM_Version::supports_cx8());
 7932   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7933   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7934   effect(KILL cr, KILL oldval);
 7935 
 7936   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7937             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7938             "sete    $res\n\t"
 7939             "movzbl  $res, $res" %}
 7940   ins_encode %{
 7941     __ lock();
 7942     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7943     __ sete($res$$Register);
 7944     __ movzbl($res$$Register, $res$$Register);
 7945   %}
 7946   ins_pipe( pipe_cmpxchg );
 7947 %}
 7948 
 7949 instruct compareAndSwapI(rRegI res,
 7950                          memory mem_ptr,
 7951                          rax_RegI oldval, rRegI newval,
 7952                          rFlagsReg cr)
 7953 %{
 7954   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7955   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7956   effect(KILL cr, KILL oldval);
 7957 
 7958   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7959             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7960             "sete    $res\n\t"
 7961             "movzbl  $res, $res" %}
 7962   ins_encode %{
 7963     __ lock();
 7964     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7965     __ sete($res$$Register);
 7966     __ movzbl($res$$Register, $res$$Register);
 7967   %}
 7968   ins_pipe( pipe_cmpxchg );
 7969 %}
 7970 
 7971 instruct compareAndSwapB(rRegI res,
 7972                          memory mem_ptr,
 7973                          rax_RegI oldval, rRegI newval,
 7974                          rFlagsReg cr)
 7975 %{
 7976   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7977   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7978   effect(KILL cr, KILL oldval);
 7979 
 7980   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7981             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7982             "sete    $res\n\t"
 7983             "movzbl  $res, $res" %}
 7984   ins_encode %{
 7985     __ lock();
 7986     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7987     __ sete($res$$Register);
 7988     __ movzbl($res$$Register, $res$$Register);
 7989   %}
 7990   ins_pipe( pipe_cmpxchg );
 7991 %}
 7992 
 7993 instruct compareAndSwapS(rRegI res,
 7994                          memory mem_ptr,
 7995                          rax_RegI oldval, rRegI newval,
 7996                          rFlagsReg cr)
 7997 %{
 7998   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7999   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 8000   effect(KILL cr, KILL oldval);
 8001 
 8002   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8003             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8004             "sete    $res\n\t"
 8005             "movzbl  $res, $res" %}
 8006   ins_encode %{
 8007     __ lock();
 8008     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8009     __ sete($res$$Register);
 8010     __ movzbl($res$$Register, $res$$Register);
 8011   %}
 8012   ins_pipe( pipe_cmpxchg );
 8013 %}
 8014 
 8015 instruct compareAndSwapN(rRegI res,
 8016                           memory mem_ptr,
 8017                           rax_RegN oldval, rRegN newval,
 8018                           rFlagsReg cr) %{
 8019   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 8020   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 8021   effect(KILL cr, KILL oldval);
 8022 
 8023   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8024             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 8025             "sete    $res\n\t"
 8026             "movzbl  $res, $res" %}
 8027   ins_encode %{
 8028     __ lock();
 8029     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8030     __ sete($res$$Register);
 8031     __ movzbl($res$$Register, $res$$Register);
 8032   %}
 8033   ins_pipe( pipe_cmpxchg );
 8034 %}
 8035 
 8036 instruct compareAndExchangeB(
 8037                          memory mem_ptr,
 8038                          rax_RegI oldval, rRegI newval,
 8039                          rFlagsReg cr)
 8040 %{
 8041   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 8042   effect(KILL cr);
 8043 
 8044   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 8045             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8046   ins_encode %{
 8047     __ lock();
 8048     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 8049   %}
 8050   ins_pipe( pipe_cmpxchg );
 8051 %}
 8052 
 8053 instruct compareAndExchangeS(
 8054                          memory mem_ptr,
 8055                          rax_RegI oldval, rRegI newval,
 8056                          rFlagsReg cr)
 8057 %{
 8058   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 8059   effect(KILL cr);
 8060 
 8061   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 8062             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8063   ins_encode %{
 8064     __ lock();
 8065     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 8066   %}
 8067   ins_pipe( pipe_cmpxchg );
 8068 %}
 8069 
 8070 instruct compareAndExchangeI(
 8071                          memory mem_ptr,
 8072                          rax_RegI oldval, rRegI newval,
 8073                          rFlagsReg cr)
 8074 %{
 8075   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 8076   effect(KILL cr);
 8077 
 8078   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8079             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8080   ins_encode %{
 8081     __ lock();
 8082     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8083   %}
 8084   ins_pipe( pipe_cmpxchg );
 8085 %}
 8086 
 8087 instruct compareAndExchangeL(
 8088                          memory mem_ptr,
 8089                          rax_RegL oldval, rRegL newval,
 8090                          rFlagsReg cr)
 8091 %{
 8092   predicate(VM_Version::supports_cx8());
 8093   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 8094   effect(KILL cr);
 8095 
 8096   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8097             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 8098   ins_encode %{
 8099     __ lock();
 8100     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8101   %}
 8102   ins_pipe( pipe_cmpxchg );
 8103 %}
 8104 
 8105 instruct compareAndExchangeN(
 8106                           memory mem_ptr,
 8107                           rax_RegN oldval, rRegN newval,
 8108                           rFlagsReg cr) %{
 8109   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 8110   effect(KILL cr);
 8111 
 8112   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 8113             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8114   ins_encode %{
 8115     __ lock();
 8116     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 8117   %}
 8118   ins_pipe( pipe_cmpxchg );
 8119 %}
 8120 
 8121 instruct compareAndExchangeP(
 8122                          memory mem_ptr,
 8123                          rax_RegP oldval, rRegP newval,
 8124                          rFlagsReg cr)
 8125 %{
 8126   predicate(VM_Version::supports_cx8() && n->as_LoadStore()->barrier_data() == 0);
 8127   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 8128   effect(KILL cr);
 8129 
 8130   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 8131             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 8132   ins_encode %{
 8133     __ lock();
 8134     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 8135   %}
 8136   ins_pipe( pipe_cmpxchg );
 8137 %}
 8138 
 8139 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8140   predicate(n->as_LoadStore()->result_not_used());
 8141   match(Set dummy (GetAndAddB mem add));
 8142   effect(KILL cr);
 8143   format %{ "ADDB  [$mem],$add" %}
 8144   ins_encode %{
 8145     __ lock();
 8146     __ addb($mem$$Address, $add$$constant);
 8147   %}
 8148   ins_pipe( pipe_cmpxchg );
 8149 %}
 8150 
 8151 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
 8152   match(Set newval (GetAndAddB mem newval));
 8153   effect(KILL cr);
 8154   format %{ "XADDB  [$mem],$newval" %}
 8155   ins_encode %{
 8156     __ lock();
 8157     __ xaddb($mem$$Address, $newval$$Register);
 8158   %}
 8159   ins_pipe( pipe_cmpxchg );
 8160 %}
 8161 
 8162 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8163   predicate(n->as_LoadStore()->result_not_used());
 8164   match(Set dummy (GetAndAddS mem add));
 8165   effect(KILL cr);
 8166   format %{ "ADDW  [$mem],$add" %}
 8167   ins_encode %{
 8168     __ lock();
 8169     __ addw($mem$$Address, $add$$constant);
 8170   %}
 8171   ins_pipe( pipe_cmpxchg );
 8172 %}
 8173 
 8174 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
 8175   match(Set newval (GetAndAddS mem newval));
 8176   effect(KILL cr);
 8177   format %{ "XADDW  [$mem],$newval" %}
 8178   ins_encode %{
 8179     __ lock();
 8180     __ xaddw($mem$$Address, $newval$$Register);
 8181   %}
 8182   ins_pipe( pipe_cmpxchg );
 8183 %}
 8184 
 8185 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8186   predicate(n->as_LoadStore()->result_not_used());
 8187   match(Set dummy (GetAndAddI mem add));
 8188   effect(KILL cr);
 8189   format %{ "ADDL  [$mem],$add" %}
 8190   ins_encode %{
 8191     __ lock();
 8192     __ addl($mem$$Address, $add$$constant);
 8193   %}
 8194   ins_pipe( pipe_cmpxchg );
 8195 %}
 8196 
 8197 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
 8198   match(Set newval (GetAndAddI mem newval));
 8199   effect(KILL cr);
 8200   format %{ "XADDL  [$mem],$newval" %}
 8201   ins_encode %{
 8202     __ lock();
 8203     __ xaddl($mem$$Address, $newval$$Register);
 8204   %}
 8205   ins_pipe( pipe_cmpxchg );
 8206 %}
 8207 
 8208 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8209   predicate(n->as_LoadStore()->result_not_used());
 8210   match(Set dummy (GetAndAddL mem add));
 8211   effect(KILL cr);
 8212   format %{ "ADDQ  [$mem],$add" %}
 8213   ins_encode %{
 8214     __ lock();
 8215     __ addq($mem$$Address, $add$$constant);
 8216   %}
 8217   ins_pipe( pipe_cmpxchg );
 8218 %}
 8219 
 8220 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
 8221   match(Set newval (GetAndAddL mem newval));
 8222   effect(KILL cr);
 8223   format %{ "XADDQ  [$mem],$newval" %}
 8224   ins_encode %{
 8225     __ lock();
 8226     __ xaddq($mem$$Address, $newval$$Register);
 8227   %}
 8228   ins_pipe( pipe_cmpxchg );
 8229 %}
 8230 
 8231 instruct xchgB( memory mem, rRegI newval) %{
 8232   match(Set newval (GetAndSetB mem newval));
 8233   format %{ "XCHGB  $newval,[$mem]" %}
 8234   ins_encode %{
 8235     __ xchgb($newval$$Register, $mem$$Address);
 8236   %}
 8237   ins_pipe( pipe_cmpxchg );
 8238 %}
 8239 
 8240 instruct xchgS( memory mem, rRegI newval) %{
 8241   match(Set newval (GetAndSetS mem newval));
 8242   format %{ "XCHGW  $newval,[$mem]" %}
 8243   ins_encode %{
 8244     __ xchgw($newval$$Register, $mem$$Address);
 8245   %}
 8246   ins_pipe( pipe_cmpxchg );
 8247 %}
 8248 
 8249 instruct xchgI( memory mem, rRegI newval) %{
 8250   match(Set newval (GetAndSetI mem newval));
 8251   format %{ "XCHGL  $newval,[$mem]" %}
 8252   ins_encode %{
 8253     __ xchgl($newval$$Register, $mem$$Address);
 8254   %}
 8255   ins_pipe( pipe_cmpxchg );
 8256 %}
 8257 
 8258 instruct xchgL( memory mem, rRegL newval) %{
 8259   match(Set newval (GetAndSetL mem newval));
 8260   format %{ "XCHGL  $newval,[$mem]" %}
 8261   ins_encode %{
 8262     __ xchgq($newval$$Register, $mem$$Address);
 8263   %}
 8264   ins_pipe( pipe_cmpxchg );
 8265 %}
 8266 
 8267 instruct xchgP( memory mem, rRegP newval) %{
 8268   match(Set newval (GetAndSetP mem newval));
 8269   predicate(n->as_LoadStore()->barrier_data() == 0);
 8270   format %{ "XCHGQ  $newval,[$mem]" %}
 8271   ins_encode %{
 8272     __ xchgq($newval$$Register, $mem$$Address);
 8273   %}
 8274   ins_pipe( pipe_cmpxchg );
 8275 %}
 8276 
 8277 instruct xchgN( memory mem, rRegN newval) %{
 8278   match(Set newval (GetAndSetN mem newval));
 8279   format %{ "XCHGL  $newval,$mem]" %}
 8280   ins_encode %{
 8281     __ xchgl($newval$$Register, $mem$$Address);
 8282   %}
 8283   ins_pipe( pipe_cmpxchg );
 8284 %}
 8285 
 8286 //----------Abs Instructions-------------------------------------------
 8287 
 8288 // Integer Absolute Instructions
 8289 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
 8290 %{
 8291   match(Set dst (AbsI src));
 8292   effect(TEMP dst, TEMP tmp, KILL cr);
 8293   format %{ "movl $tmp, $src\n\t"
 8294             "sarl $tmp, 31\n\t"
 8295             "movl $dst, $src\n\t"
 8296             "xorl $dst, $tmp\n\t"
 8297             "subl $dst, $tmp\n"
 8298           %}
 8299   ins_encode %{
 8300     __ movl($tmp$$Register, $src$$Register);
 8301     __ sarl($tmp$$Register, 31);
 8302     __ movl($dst$$Register, $src$$Register);
 8303     __ xorl($dst$$Register, $tmp$$Register);
 8304     __ subl($dst$$Register, $tmp$$Register);
 8305   %}
 8306 
 8307   ins_pipe(ialu_reg_reg);
 8308 %}
 8309 
 8310 // Long Absolute Instructions
 8311 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
 8312 %{
 8313   match(Set dst (AbsL src));
 8314   effect(TEMP dst, TEMP tmp, KILL cr);
 8315   format %{ "movq $tmp, $src\n\t"
 8316             "sarq $tmp, 63\n\t"
 8317             "movq $dst, $src\n\t"
 8318             "xorq $dst, $tmp\n\t"
 8319             "subq $dst, $tmp\n"
 8320           %}
 8321   ins_encode %{
 8322     __ movq($tmp$$Register, $src$$Register);
 8323     __ sarq($tmp$$Register, 63);
 8324     __ movq($dst$$Register, $src$$Register);
 8325     __ xorq($dst$$Register, $tmp$$Register);
 8326     __ subq($dst$$Register, $tmp$$Register);
 8327   %}
 8328 
 8329   ins_pipe(ialu_reg_reg);
 8330 %}
 8331 
 8332 //----------Subtraction Instructions-------------------------------------------
 8333 
 8334 // Integer Subtraction Instructions
 8335 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8336 %{
 8337   match(Set dst (SubI dst src));
 8338   effect(KILL cr);
 8339 
 8340   format %{ "subl    $dst, $src\t# int" %}
 8341   ins_encode %{
 8342     __ subl($dst$$Register, $src$$Register);
 8343   %}
 8344   ins_pipe(ialu_reg_reg);
 8345 %}
 8346 
 8347 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8348 %{
 8349   match(Set dst (SubI dst src));
 8350   effect(KILL cr);
 8351 
 8352   format %{ "subl    $dst, $src\t# int" %}
 8353   ins_encode %{
 8354     __ subl($dst$$Register, $src$$constant);
 8355   %}
 8356   ins_pipe(ialu_reg);
 8357 %}
 8358 
 8359 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8360 %{
 8361   match(Set dst (SubI dst (LoadI src)));
 8362   effect(KILL cr);
 8363 
 8364   ins_cost(125);
 8365   format %{ "subl    $dst, $src\t# int" %}
 8366   ins_encode %{
 8367     __ subl($dst$$Register, $src$$Address);
 8368   %}
 8369   ins_pipe(ialu_reg_mem);
 8370 %}
 8371 
 8372 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8373 %{
 8374   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8375   effect(KILL cr);
 8376 
 8377   ins_cost(150);
 8378   format %{ "subl    $dst, $src\t# int" %}
 8379   ins_encode %{
 8380     __ subl($dst$$Address, $src$$Register);
 8381   %}
 8382   ins_pipe(ialu_mem_reg);
 8383 %}
 8384 
 8385 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
 8386 %{
 8387   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8388   effect(KILL cr);
 8389 
 8390   ins_cost(125); // XXX
 8391   format %{ "subl    $dst, $src\t# int" %}
 8392   ins_encode %{
 8393     __ subl($dst$$Address, $src$$constant);
 8394   %}
 8395   ins_pipe(ialu_mem_imm);
 8396 %}
 8397 
 8398 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8399 %{
 8400   match(Set dst (SubL dst src));
 8401   effect(KILL cr);
 8402 
 8403   format %{ "subq    $dst, $src\t# long" %}
 8404   ins_encode %{
 8405     __ subq($dst$$Register, $src$$Register);
 8406   %}
 8407   ins_pipe(ialu_reg_reg);
 8408 %}
 8409 
 8410 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
 8411 %{
 8412   match(Set dst (SubL dst src));
 8413   effect(KILL cr);
 8414 
 8415   format %{ "subq    $dst, $src\t# long" %}
 8416   ins_encode %{
 8417     __ subq($dst$$Register, $src$$constant);
 8418   %}
 8419   ins_pipe(ialu_reg);
 8420 %}
 8421 
 8422 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8423 %{
 8424   match(Set dst (SubL dst (LoadL src)));
 8425   effect(KILL cr);
 8426 
 8427   ins_cost(125);
 8428   format %{ "subq    $dst, $src\t# long" %}
 8429   ins_encode %{
 8430     __ subq($dst$$Register, $src$$Address);
 8431   %}
 8432   ins_pipe(ialu_reg_mem);
 8433 %}
 8434 
 8435 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8436 %{
 8437   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8438   effect(KILL cr);
 8439 
 8440   ins_cost(150);
 8441   format %{ "subq    $dst, $src\t# long" %}
 8442   ins_encode %{
 8443     __ subq($dst$$Address, $src$$Register);
 8444   %}
 8445   ins_pipe(ialu_mem_reg);
 8446 %}
 8447 
 8448 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 8449 %{
 8450   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8451   effect(KILL cr);
 8452 
 8453   ins_cost(125); // XXX
 8454   format %{ "subq    $dst, $src\t# long" %}
 8455   ins_encode %{
 8456     __ subq($dst$$Address, $src$$constant);
 8457   %}
 8458   ins_pipe(ialu_mem_imm);
 8459 %}
 8460 
 8461 // Subtract from a pointer
 8462 // XXX hmpf???
 8463 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8464 %{
 8465   match(Set dst (AddP dst (SubI zero src)));
 8466   effect(KILL cr);
 8467 
 8468   format %{ "subq    $dst, $src\t# ptr - int" %}
 8469   opcode(0x2B);
 8470   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
 8471   ins_pipe(ialu_reg_reg);
 8472 %}
 8473 
 8474 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8475 %{
 8476   match(Set dst (SubI zero dst));
 8477   effect(KILL cr);
 8478 
 8479   format %{ "negl    $dst\t# int" %}
 8480   ins_encode %{
 8481     __ negl($dst$$Register);
 8482   %}
 8483   ins_pipe(ialu_reg);
 8484 %}
 8485 
 8486 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8487 %{
 8488   match(Set dst (NegI dst));
 8489   effect(KILL cr);
 8490 
 8491   format %{ "negl    $dst\t# int" %}
 8492   ins_encode %{
 8493     __ negl($dst$$Register);
 8494   %}
 8495   ins_pipe(ialu_reg);
 8496 %}
 8497 
 8498 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8499 %{
 8500   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8501   effect(KILL cr);
 8502 
 8503   format %{ "negl    $dst\t# int" %}
 8504   ins_encode %{
 8505     __ negl($dst$$Address);
 8506   %}
 8507   ins_pipe(ialu_reg);
 8508 %}
 8509 
 8510 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8511 %{
 8512   match(Set dst (SubL zero dst));
 8513   effect(KILL cr);
 8514 
 8515   format %{ "negq    $dst\t# long" %}
 8516   ins_encode %{
 8517     __ negq($dst$$Register);
 8518   %}
 8519   ins_pipe(ialu_reg);
 8520 %}
 8521 
 8522 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8523 %{
 8524   match(Set dst (NegL dst));
 8525   effect(KILL cr);
 8526 
 8527   format %{ "negq    $dst\t# int" %}
 8528   ins_encode %{
 8529     __ negq($dst$$Register);
 8530   %}
 8531   ins_pipe(ialu_reg);
 8532 %}
 8533 
 8534 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8535 %{
 8536   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8537   effect(KILL cr);
 8538 
 8539   format %{ "negq    $dst\t# long" %}
 8540   ins_encode %{
 8541     __ negq($dst$$Address);
 8542   %}
 8543   ins_pipe(ialu_reg);
 8544 %}
 8545 
 8546 //----------Multiplication/Division Instructions-------------------------------
 8547 // Integer Multiplication Instructions
 8548 // Multiply Register
 8549 
 8550 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8551 %{
 8552   match(Set dst (MulI dst src));
 8553   effect(KILL cr);
 8554 
 8555   ins_cost(300);
 8556   format %{ "imull   $dst, $src\t# int" %}
 8557   ins_encode %{
 8558     __ imull($dst$$Register, $src$$Register);
 8559   %}
 8560   ins_pipe(ialu_reg_reg_alu0);
 8561 %}
 8562 
 8563 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8564 %{
 8565   match(Set dst (MulI src imm));
 8566   effect(KILL cr);
 8567 
 8568   ins_cost(300);
 8569   format %{ "imull   $dst, $src, $imm\t# int" %}
 8570   ins_encode %{
 8571     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8572   %}
 8573   ins_pipe(ialu_reg_reg_alu0);
 8574 %}
 8575 
 8576 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8577 %{
 8578   match(Set dst (MulI dst (LoadI src)));
 8579   effect(KILL cr);
 8580 
 8581   ins_cost(350);
 8582   format %{ "imull   $dst, $src\t# int" %}
 8583   ins_encode %{
 8584     __ imull($dst$$Register, $src$$Address);
 8585   %}
 8586   ins_pipe(ialu_reg_mem_alu0);
 8587 %}
 8588 
 8589 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8590 %{
 8591   match(Set dst (MulI (LoadI src) imm));
 8592   effect(KILL cr);
 8593 
 8594   ins_cost(300);
 8595   format %{ "imull   $dst, $src, $imm\t# int" %}
 8596   ins_encode %{
 8597     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8598   %}
 8599   ins_pipe(ialu_reg_mem_alu0);
 8600 %}
 8601 
 8602 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8603 %{
 8604   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8605   effect(KILL cr, KILL src2);
 8606 
 8607   expand %{ mulI_rReg(dst, src1, cr);
 8608            mulI_rReg(src2, src3, cr);
 8609            addI_rReg(dst, src2, cr); %}
 8610 %}
 8611 
 8612 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8613 %{
 8614   match(Set dst (MulL dst src));
 8615   effect(KILL cr);
 8616 
 8617   ins_cost(300);
 8618   format %{ "imulq   $dst, $src\t# long" %}
 8619   ins_encode %{
 8620     __ imulq($dst$$Register, $src$$Register);
 8621   %}
 8622   ins_pipe(ialu_reg_reg_alu0);
 8623 %}
 8624 
 8625 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8626 %{
 8627   match(Set dst (MulL src imm));
 8628   effect(KILL cr);
 8629 
 8630   ins_cost(300);
 8631   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8632   ins_encode %{
 8633     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8634   %}
 8635   ins_pipe(ialu_reg_reg_alu0);
 8636 %}
 8637 
 8638 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8639 %{
 8640   match(Set dst (MulL dst (LoadL src)));
 8641   effect(KILL cr);
 8642 
 8643   ins_cost(350);
 8644   format %{ "imulq   $dst, $src\t# long" %}
 8645   ins_encode %{
 8646     __ imulq($dst$$Register, $src$$Address);
 8647   %}
 8648   ins_pipe(ialu_reg_mem_alu0);
 8649 %}
 8650 
 8651 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8652 %{
 8653   match(Set dst (MulL (LoadL src) imm));
 8654   effect(KILL cr);
 8655 
 8656   ins_cost(300);
 8657   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8658   ins_encode %{
 8659     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8660   %}
 8661   ins_pipe(ialu_reg_mem_alu0);
 8662 %}
 8663 
 8664 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8665 %{
 8666   match(Set dst (MulHiL src rax));
 8667   effect(USE_KILL rax, KILL cr);
 8668 
 8669   ins_cost(300);
 8670   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8671   ins_encode %{
 8672     __ imulq($src$$Register);
 8673   %}
 8674   ins_pipe(ialu_reg_reg_alu0);
 8675 %}
 8676 
 8677 instruct umulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8678 %{
 8679   match(Set dst (UMulHiL src rax));
 8680   effect(USE_KILL rax, KILL cr);
 8681 
 8682   ins_cost(300);
 8683   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8684   ins_encode %{
 8685     __ mulq($src$$Register);
 8686   %}
 8687   ins_pipe(ialu_reg_reg_alu0);
 8688 %}
 8689 
 8690 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8691                    rFlagsReg cr)
 8692 %{
 8693   match(Set rax (DivI rax div));
 8694   effect(KILL rdx, KILL cr);
 8695 
 8696   ins_cost(30*100+10*100); // XXX
 8697   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8698             "jne,s   normal\n\t"
 8699             "xorl    rdx, rdx\n\t"
 8700             "cmpl    $div, -1\n\t"
 8701             "je,s    done\n"
 8702     "normal: cdql\n\t"
 8703             "idivl   $div\n"
 8704     "done:"        %}
 8705   ins_encode(cdql_enc(div));
 8706   ins_pipe(ialu_reg_reg_alu0);
 8707 %}
 8708 
 8709 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8710                    rFlagsReg cr)
 8711 %{
 8712   match(Set rax (DivL rax div));
 8713   effect(KILL rdx, KILL cr);
 8714 
 8715   ins_cost(30*100+10*100); // XXX
 8716   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8717             "cmpq    rax, rdx\n\t"
 8718             "jne,s   normal\n\t"
 8719             "xorl    rdx, rdx\n\t"
 8720             "cmpq    $div, -1\n\t"
 8721             "je,s    done\n"
 8722     "normal: cdqq\n\t"
 8723             "idivq   $div\n"
 8724     "done:"        %}
 8725   ins_encode(cdqq_enc(div));
 8726   ins_pipe(ialu_reg_reg_alu0);
 8727 %}
 8728 
 8729 // Integer DIVMOD with Register, both quotient and mod results
 8730 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8731                              rFlagsReg cr)
 8732 %{
 8733   match(DivModI rax div);
 8734   effect(KILL cr);
 8735 
 8736   ins_cost(30*100+10*100); // XXX
 8737   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8738             "jne,s   normal\n\t"
 8739             "xorl    rdx, rdx\n\t"
 8740             "cmpl    $div, -1\n\t"
 8741             "je,s    done\n"
 8742     "normal: cdql\n\t"
 8743             "idivl   $div\n"
 8744     "done:"        %}
 8745   ins_encode(cdql_enc(div));
 8746   ins_pipe(pipe_slow);
 8747 %}
 8748 
 8749 // Long DIVMOD with Register, both quotient and mod results
 8750 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8751                              rFlagsReg cr)
 8752 %{
 8753   match(DivModL rax div);
 8754   effect(KILL cr);
 8755 
 8756   ins_cost(30*100+10*100); // XXX
 8757   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8758             "cmpq    rax, rdx\n\t"
 8759             "jne,s   normal\n\t"
 8760             "xorl    rdx, rdx\n\t"
 8761             "cmpq    $div, -1\n\t"
 8762             "je,s    done\n"
 8763     "normal: cdqq\n\t"
 8764             "idivq   $div\n"
 8765     "done:"        %}
 8766   ins_encode(cdqq_enc(div));
 8767   ins_pipe(pipe_slow);
 8768 %}
 8769 
 8770 //----------- DivL-By-Constant-Expansions--------------------------------------
 8771 // DivI cases are handled by the compiler
 8772 
 8773 // Magic constant, reciprocal of 10
 8774 instruct loadConL_0x6666666666666667(rRegL dst)
 8775 %{
 8776   effect(DEF dst);
 8777 
 8778   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
 8779   ins_encode(load_immL(dst, 0x6666666666666667));
 8780   ins_pipe(ialu_reg);
 8781 %}
 8782 
 8783 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
 8784 %{
 8785   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
 8786 
 8787   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
 8788   ins_encode %{
 8789     __ imulq($src$$Register);
 8790   %}
 8791   ins_pipe(ialu_reg_reg_alu0);
 8792 %}
 8793 
 8794 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
 8795 %{
 8796   effect(USE_DEF dst, KILL cr);
 8797 
 8798   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
 8799   ins_encode %{
 8800     __ sarq($dst$$Register, 63);
 8801   %}
 8802   ins_pipe(ialu_reg);
 8803 %}
 8804 
 8805 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
 8806 %{
 8807   effect(USE_DEF dst, KILL cr);
 8808 
 8809   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
 8810   ins_encode %{
 8811     __ sarq($dst$$Register, 2);
 8812   %}
 8813   ins_pipe(ialu_reg);
 8814 %}
 8815 
 8816 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
 8817 %{
 8818   match(Set dst (DivL src div));
 8819 
 8820   ins_cost((5+8)*100);
 8821   expand %{
 8822     rax_RegL rax;                     // Killed temp
 8823     rFlagsReg cr;                     // Killed
 8824     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
 8825     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
 8826     sarL_rReg_63(src, cr);            // sarq  src, 63
 8827     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
 8828     subL_rReg(dst, src, cr);          // subl  rdx, src
 8829   %}
 8830 %}
 8831 
 8832 //-----------------------------------------------------------------------------
 8833 
 8834 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8835                    rFlagsReg cr)
 8836 %{
 8837   match(Set rdx (ModI rax div));
 8838   effect(KILL rax, KILL cr);
 8839 
 8840   ins_cost(300); // XXX
 8841   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8842             "jne,s   normal\n\t"
 8843             "xorl    rdx, rdx\n\t"
 8844             "cmpl    $div, -1\n\t"
 8845             "je,s    done\n"
 8846     "normal: cdql\n\t"
 8847             "idivl   $div\n"
 8848     "done:"        %}
 8849   ins_encode(cdql_enc(div));
 8850   ins_pipe(ialu_reg_reg_alu0);
 8851 %}
 8852 
 8853 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8854                    rFlagsReg cr)
 8855 %{
 8856   match(Set rdx (ModL rax div));
 8857   effect(KILL rax, KILL cr);
 8858 
 8859   ins_cost(300); // XXX
 8860   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8861             "cmpq    rax, rdx\n\t"
 8862             "jne,s   normal\n\t"
 8863             "xorl    rdx, rdx\n\t"
 8864             "cmpq    $div, -1\n\t"
 8865             "je,s    done\n"
 8866     "normal: cdqq\n\t"
 8867             "idivq   $div\n"
 8868     "done:"        %}
 8869   ins_encode(cdqq_enc(div));
 8870   ins_pipe(ialu_reg_reg_alu0);
 8871 %}
 8872 
 8873 // Integer Shift Instructions
 8874 // Shift Left by one
 8875 instruct salI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8876 %{
 8877   match(Set dst (LShiftI dst shift));
 8878   effect(KILL cr);
 8879 
 8880   format %{ "sall    $dst, $shift" %}
 8881   ins_encode %{
 8882     __ sall($dst$$Register, $shift$$constant);
 8883   %}
 8884   ins_pipe(ialu_reg);
 8885 %}
 8886 
 8887 // Shift Left by one
 8888 instruct salI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8889 %{
 8890   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8891   effect(KILL cr);
 8892 
 8893   format %{ "sall    $dst, $shift\t" %}
 8894   ins_encode %{
 8895     __ sall($dst$$Address, $shift$$constant);
 8896   %}
 8897   ins_pipe(ialu_mem_imm);
 8898 %}
 8899 
 8900 // Shift Left by 8-bit immediate
 8901 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8902 %{
 8903   match(Set dst (LShiftI dst shift));
 8904   effect(KILL cr);
 8905 
 8906   format %{ "sall    $dst, $shift" %}
 8907   ins_encode %{
 8908     __ sall($dst$$Register, $shift$$constant);
 8909   %}
 8910   ins_pipe(ialu_reg);
 8911 %}
 8912 
 8913 // Shift Left by 8-bit immediate
 8914 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8915 %{
 8916   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8917   effect(KILL cr);
 8918 
 8919   format %{ "sall    $dst, $shift" %}
 8920   ins_encode %{
 8921     __ sall($dst$$Address, $shift$$constant);
 8922   %}
 8923   ins_pipe(ialu_mem_imm);
 8924 %}
 8925 
 8926 // Shift Left by variable
 8927 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8928 %{
 8929   match(Set dst (LShiftI dst shift));
 8930   effect(KILL cr);
 8931 
 8932   format %{ "sall    $dst, $shift" %}
 8933   ins_encode %{
 8934     __ sall($dst$$Register);
 8935   %}
 8936   ins_pipe(ialu_reg_reg);
 8937 %}
 8938 
 8939 // Shift Left by variable
 8940 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8941 %{
 8942   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8943   effect(KILL cr);
 8944 
 8945   format %{ "sall    $dst, $shift" %}
 8946   ins_encode %{
 8947     __ sall($dst$$Address);
 8948   %}
 8949   ins_pipe(ialu_mem_reg);
 8950 %}
 8951 
 8952 // Arithmetic shift right by one
 8953 instruct sarI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 8954 %{
 8955   match(Set dst (RShiftI dst shift));
 8956   effect(KILL cr);
 8957 
 8958   format %{ "sarl    $dst, $shift" %}
 8959   ins_encode %{
 8960     __ sarl($dst$$Register, $shift$$constant);
 8961   %}
 8962   ins_pipe(ialu_reg);
 8963 %}
 8964 
 8965 // Arithmetic shift right by one
 8966 instruct sarI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 8967 %{
 8968   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8969   effect(KILL cr);
 8970 
 8971   format %{ "sarl    $dst, $shift" %}
 8972   ins_encode %{
 8973     __ sarl($dst$$Address, $shift$$constant);
 8974   %}
 8975   ins_pipe(ialu_mem_imm);
 8976 %}
 8977 
 8978 // Arithmetic Shift Right by 8-bit immediate
 8979 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8980 %{
 8981   match(Set dst (RShiftI dst shift));
 8982   effect(KILL cr);
 8983 
 8984   format %{ "sarl    $dst, $shift" %}
 8985   ins_encode %{
 8986     __ sarl($dst$$Register, $shift$$constant);
 8987   %}
 8988   ins_pipe(ialu_mem_imm);
 8989 %}
 8990 
 8991 // Arithmetic Shift Right by 8-bit immediate
 8992 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8993 %{
 8994   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8995   effect(KILL cr);
 8996 
 8997   format %{ "sarl    $dst, $shift" %}
 8998   ins_encode %{
 8999     __ sarl($dst$$Address, $shift$$constant);
 9000   %}
 9001   ins_pipe(ialu_mem_imm);
 9002 %}
 9003 
 9004 // Arithmetic Shift Right by variable
 9005 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9006 %{
 9007   match(Set dst (RShiftI dst shift));
 9008   effect(KILL cr);
 9009   format %{ "sarl    $dst, $shift" %}
 9010   ins_encode %{
 9011     __ sarl($dst$$Register);
 9012   %}
 9013   ins_pipe(ialu_reg_reg);
 9014 %}
 9015 
 9016 // Arithmetic Shift Right by variable
 9017 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9018 %{
 9019   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9020   effect(KILL cr);
 9021 
 9022   format %{ "sarl    $dst, $shift" %}
 9023   ins_encode %{
 9024     __ sarl($dst$$Address);
 9025   %}
 9026   ins_pipe(ialu_mem_reg);
 9027 %}
 9028 
 9029 // Logical shift right by one
 9030 instruct shrI_rReg_1(rRegI dst, immI_1 shift, rFlagsReg cr)
 9031 %{
 9032   match(Set dst (URShiftI dst shift));
 9033   effect(KILL cr);
 9034 
 9035   format %{ "shrl    $dst, $shift" %}
 9036   ins_encode %{
 9037     __ shrl($dst$$Register, $shift$$constant);
 9038   %}
 9039   ins_pipe(ialu_reg);
 9040 %}
 9041 
 9042 // Logical shift right by one
 9043 instruct shrI_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9044 %{
 9045   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9046   effect(KILL cr);
 9047 
 9048   format %{ "shrl    $dst, $shift" %}
 9049   ins_encode %{
 9050     __ shrl($dst$$Address, $shift$$constant);
 9051   %}
 9052   ins_pipe(ialu_mem_imm);
 9053 %}
 9054 
 9055 // Logical Shift Right by 8-bit immediate
 9056 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9057 %{
 9058   match(Set dst (URShiftI dst shift));
 9059   effect(KILL cr);
 9060 
 9061   format %{ "shrl    $dst, $shift" %}
 9062   ins_encode %{
 9063     __ shrl($dst$$Register, $shift$$constant);
 9064   %}
 9065   ins_pipe(ialu_reg);
 9066 %}
 9067 
 9068 // Logical Shift Right by 8-bit immediate
 9069 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9070 %{
 9071   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9072   effect(KILL cr);
 9073 
 9074   format %{ "shrl    $dst, $shift" %}
 9075   ins_encode %{
 9076     __ shrl($dst$$Address, $shift$$constant);
 9077   %}
 9078   ins_pipe(ialu_mem_imm);
 9079 %}
 9080 
 9081 // Logical Shift Right by variable
 9082 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9083 %{
 9084   match(Set dst (URShiftI dst shift));
 9085   effect(KILL cr);
 9086 
 9087   format %{ "shrl    $dst, $shift" %}
 9088   ins_encode %{
 9089     __ shrl($dst$$Register);
 9090   %}
 9091   ins_pipe(ialu_reg_reg);
 9092 %}
 9093 
 9094 // Logical Shift Right by variable
 9095 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9096 %{
 9097   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9098   effect(KILL cr);
 9099 
 9100   format %{ "shrl    $dst, $shift" %}
 9101   ins_encode %{
 9102     __ shrl($dst$$Address);
 9103   %}
 9104   ins_pipe(ialu_mem_reg);
 9105 %}
 9106 
 9107 // Long Shift Instructions
 9108 // Shift Left by one
 9109 instruct salL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9110 %{
 9111   match(Set dst (LShiftL dst shift));
 9112   effect(KILL cr);
 9113 
 9114   format %{ "salq    $dst, $shift" %}
 9115   ins_encode %{
 9116     __ salq($dst$$Register, $shift$$constant);
 9117   %}
 9118   ins_pipe(ialu_reg);
 9119 %}
 9120 
 9121 // Shift Left by one
 9122 instruct salL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9123 %{
 9124   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9125   effect(KILL cr);
 9126 
 9127   format %{ "salq    $dst, $shift" %}
 9128   ins_encode %{
 9129     __ salq($dst$$Address, $shift$$constant);
 9130   %}
 9131   ins_pipe(ialu_mem_imm);
 9132 %}
 9133 
 9134 // Shift Left by 8-bit immediate
 9135 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9136 %{
 9137   match(Set dst (LShiftL dst shift));
 9138   effect(KILL cr);
 9139 
 9140   format %{ "salq    $dst, $shift" %}
 9141   ins_encode %{
 9142     __ salq($dst$$Register, $shift$$constant);
 9143   %}
 9144   ins_pipe(ialu_reg);
 9145 %}
 9146 
 9147 // Shift Left by 8-bit immediate
 9148 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9149 %{
 9150   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9151   effect(KILL cr);
 9152 
 9153   format %{ "salq    $dst, $shift" %}
 9154   ins_encode %{
 9155     __ salq($dst$$Address, $shift$$constant);
 9156   %}
 9157   ins_pipe(ialu_mem_imm);
 9158 %}
 9159 
 9160 // Shift Left by variable
 9161 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9162 %{
 9163   match(Set dst (LShiftL dst shift));
 9164   effect(KILL cr);
 9165 
 9166   format %{ "salq    $dst, $shift" %}
 9167   ins_encode %{
 9168     __ salq($dst$$Register);
 9169   %}
 9170   ins_pipe(ialu_reg_reg);
 9171 %}
 9172 
 9173 // Shift Left by variable
 9174 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9175 %{
 9176   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9177   effect(KILL cr);
 9178 
 9179   format %{ "salq    $dst, $shift" %}
 9180   ins_encode %{
 9181     __ salq($dst$$Address);
 9182   %}
 9183   ins_pipe(ialu_mem_reg);
 9184 %}
 9185 
 9186 // Arithmetic shift right by one
 9187 instruct sarL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9188 %{
 9189   match(Set dst (RShiftL dst shift));
 9190   effect(KILL cr);
 9191 
 9192   format %{ "sarq    $dst, $shift" %}
 9193   ins_encode %{
 9194     __ sarq($dst$$Register, $shift$$constant);
 9195   %}
 9196   ins_pipe(ialu_reg);
 9197 %}
 9198 
 9199 // Arithmetic shift right by one
 9200 instruct sarL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9201 %{
 9202   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9203   effect(KILL cr);
 9204 
 9205   format %{ "sarq    $dst, $shift" %}
 9206   ins_encode %{
 9207     __ sarq($dst$$Address, $shift$$constant);
 9208   %}
 9209   ins_pipe(ialu_mem_imm);
 9210 %}
 9211 
 9212 // Arithmetic Shift Right by 8-bit immediate
 9213 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9214 %{
 9215   match(Set dst (RShiftL dst shift));
 9216   effect(KILL cr);
 9217 
 9218   format %{ "sarq    $dst, $shift" %}
 9219   ins_encode %{
 9220     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9221   %}
 9222   ins_pipe(ialu_mem_imm);
 9223 %}
 9224 
 9225 // Arithmetic Shift Right by 8-bit immediate
 9226 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9227 %{
 9228   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9229   effect(KILL cr);
 9230 
 9231   format %{ "sarq    $dst, $shift" %}
 9232   ins_encode %{
 9233     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9234   %}
 9235   ins_pipe(ialu_mem_imm);
 9236 %}
 9237 
 9238 // Arithmetic Shift Right by variable
 9239 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9240 %{
 9241   match(Set dst (RShiftL dst shift));
 9242   effect(KILL cr);
 9243 
 9244   format %{ "sarq    $dst, $shift" %}
 9245   ins_encode %{
 9246     __ sarq($dst$$Register);
 9247   %}
 9248   ins_pipe(ialu_reg_reg);
 9249 %}
 9250 
 9251 // Arithmetic Shift Right by variable
 9252 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9253 %{
 9254   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9255   effect(KILL cr);
 9256 
 9257   format %{ "sarq    $dst, $shift" %}
 9258   ins_encode %{
 9259     __ sarq($dst$$Address);
 9260   %}
 9261   ins_pipe(ialu_mem_reg);
 9262 %}
 9263 
 9264 // Logical shift right by one
 9265 instruct shrL_rReg_1(rRegL dst, immI_1 shift, rFlagsReg cr)
 9266 %{
 9267   match(Set dst (URShiftL dst shift));
 9268   effect(KILL cr);
 9269 
 9270   format %{ "shrq    $dst, $shift" %}
 9271   ins_encode %{
 9272     __ shrq($dst$$Register, $shift$$constant);
 9273   %}
 9274   ins_pipe(ialu_reg);
 9275 %}
 9276 
 9277 // Logical shift right by one
 9278 instruct shrL_mem_1(memory dst, immI_1 shift, rFlagsReg cr)
 9279 %{
 9280   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9281   effect(KILL cr);
 9282 
 9283   format %{ "shrq    $dst, $shift" %}
 9284   ins_encode %{
 9285     __ shrq($dst$$Address, $shift$$constant);
 9286   %}
 9287   ins_pipe(ialu_mem_imm);
 9288 %}
 9289 
 9290 // Logical Shift Right by 8-bit immediate
 9291 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9292 %{
 9293   match(Set dst (URShiftL dst shift));
 9294   effect(KILL cr);
 9295 
 9296   format %{ "shrq    $dst, $shift" %}
 9297   ins_encode %{
 9298     __ shrq($dst$$Register, $shift$$constant);
 9299   %}
 9300   ins_pipe(ialu_reg);
 9301 %}
 9302 
 9303 // Logical Shift Right by 8-bit immediate
 9304 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9305 %{
 9306   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9307   effect(KILL cr);
 9308 
 9309   format %{ "shrq    $dst, $shift" %}
 9310   ins_encode %{
 9311     __ shrq($dst$$Address, $shift$$constant);
 9312   %}
 9313   ins_pipe(ialu_mem_imm);
 9314 %}
 9315 
 9316 // Logical Shift Right by variable
 9317 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9318 %{
 9319   match(Set dst (URShiftL dst shift));
 9320   effect(KILL cr);
 9321 
 9322   format %{ "shrq    $dst, $shift" %}
 9323   ins_encode %{
 9324     __ shrq($dst$$Register);
 9325   %}
 9326   ins_pipe(ialu_reg_reg);
 9327 %}
 9328 
 9329 // Logical Shift Right by variable
 9330 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9331 %{
 9332   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9333   effect(KILL cr);
 9334 
 9335   format %{ "shrq    $dst, $shift" %}
 9336   ins_encode %{
 9337     __ shrq($dst$$Address);
 9338   %}
 9339   ins_pipe(ialu_mem_reg);
 9340 %}
 9341 
 9342 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9343 // This idiom is used by the compiler for the i2b bytecode.
 9344 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9345 %{
 9346   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9347 
 9348   format %{ "movsbl  $dst, $src\t# i2b" %}
 9349   ins_encode %{
 9350     __ movsbl($dst$$Register, $src$$Register);
 9351   %}
 9352   ins_pipe(ialu_reg_reg);
 9353 %}
 9354 
 9355 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9356 // This idiom is used by the compiler the i2s bytecode.
 9357 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9358 %{
 9359   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9360 
 9361   format %{ "movswl  $dst, $src\t# i2s" %}
 9362   ins_encode %{
 9363     __ movswl($dst$$Register, $src$$Register);
 9364   %}
 9365   ins_pipe(ialu_reg_reg);
 9366 %}
 9367 
 9368 // ROL/ROR instructions
 9369 
 9370 // Rotate left by constant.
 9371 instruct rolI_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9372 %{
 9373   predicate(n->bottom_type()->basic_type() == T_INT);
 9374   match(Set dst (RotateLeft dst shift));
 9375   effect(KILL cr);
 9376   format %{ "roll    $dst, $shift" %}
 9377   ins_encode %{
 9378     __ roll($dst$$Register, $shift$$constant);
 9379   %}
 9380   ins_pipe(ialu_reg);
 9381 %}
 9382 
 9383 // Rotate Left by variable
 9384 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9385 %{
 9386   predicate(n->bottom_type()->basic_type() == T_INT);
 9387   match(Set dst (RotateLeft dst shift));
 9388   effect(KILL cr);
 9389   format %{ "roll    $dst, $shift" %}
 9390   ins_encode %{
 9391     __ roll($dst$$Register);
 9392   %}
 9393   ins_pipe(ialu_reg_reg);
 9394 %}
 9395 
 9396 // Rotate Right by constant.
 9397 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9398 %{
 9399   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9400   match(Set dst (RotateRight dst shift));
 9401   effect(KILL cr);
 9402   format %{ "rorl    $dst, $shift" %}
 9403   ins_encode %{
 9404     __ rorl($dst$$Register, $shift$$constant);
 9405   %}
 9406   ins_pipe(ialu_reg);
 9407 %}
 9408 
 9409 // Rotate Right by constant.
 9410 instruct rorI_immI8(rRegI dst, immI8 shift)
 9411 %{
 9412   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9413   match(Set dst (RotateRight dst shift));
 9414   format %{ "rorxd     $dst, $shift" %}
 9415   ins_encode %{
 9416     __ rorxd($dst$$Register, $dst$$Register, $shift$$constant);
 9417   %}
 9418   ins_pipe(ialu_reg_reg);
 9419 %}
 9420 
 9421 // Rotate Right by variable
 9422 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9423 %{
 9424   predicate(n->bottom_type()->basic_type() == T_INT);
 9425   match(Set dst (RotateRight dst shift));
 9426   effect(KILL cr);
 9427   format %{ "rorl    $dst, $shift" %}
 9428   ins_encode %{
 9429     __ rorl($dst$$Register);
 9430   %}
 9431   ins_pipe(ialu_reg_reg);
 9432 %}
 9433 
 9434 
 9435 // Rotate Left by constant.
 9436 instruct rolL_immI8(rRegL dst, immI8 shift, rFlagsReg cr)
 9437 %{
 9438   predicate(n->bottom_type()->basic_type() == T_LONG);
 9439   match(Set dst (RotateLeft dst shift));
 9440   effect(KILL cr);
 9441   format %{ "rolq    $dst, $shift" %}
 9442   ins_encode %{
 9443     __ rolq($dst$$Register, $shift$$constant);
 9444   %}
 9445   ins_pipe(ialu_reg);
 9446 %}
 9447 
 9448 // Rotate Left by variable
 9449 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9450 %{
 9451   predicate(n->bottom_type()->basic_type() == T_LONG);
 9452   match(Set dst (RotateLeft dst shift));
 9453   effect(KILL cr);
 9454   format %{ "rolq    $dst, $shift" %}
 9455   ins_encode %{
 9456     __ rolq($dst$$Register);
 9457   %}
 9458   ins_pipe(ialu_reg_reg);
 9459 %}
 9460 
 9461 
 9462 // Rotate Right by constant.
 9463 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9464 %{
 9465   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9466   match(Set dst (RotateRight dst shift));
 9467   effect(KILL cr);
 9468   format %{ "rorq    $dst, $shift" %}
 9469   ins_encode %{
 9470     __ rorq($dst$$Register, $shift$$constant);
 9471   %}
 9472   ins_pipe(ialu_reg);
 9473 %}
 9474 
 9475 
 9476 // Rotate Right by constant
 9477 instruct rorL_immI8(rRegL dst, immI8 shift)
 9478 %{
 9479   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9480   match(Set dst (RotateRight dst shift));
 9481   format %{ "rorxq    $dst, $shift" %}
 9482   ins_encode %{
 9483     __ rorxq($dst$$Register, $dst$$Register, $shift$$constant);
 9484   %}
 9485   ins_pipe(ialu_reg_reg);
 9486 %}
 9487 
 9488 // Rotate Right by variable
 9489 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9490 %{
 9491   predicate(n->bottom_type()->basic_type() == T_LONG);
 9492   match(Set dst (RotateRight dst shift));
 9493   effect(KILL cr);
 9494   format %{ "rorq    $dst, $shift" %}
 9495   ins_encode %{
 9496     __ rorq($dst$$Register);
 9497   %}
 9498   ins_pipe(ialu_reg_reg);
 9499 %}
 9500 
 9501 
 9502 // Logical Instructions
 9503 
 9504 // Integer Logical Instructions
 9505 
 9506 // And Instructions
 9507 // And Register with Register
 9508 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9509 %{
 9510   match(Set dst (AndI dst src));
 9511   effect(KILL cr);
 9512 
 9513   format %{ "andl    $dst, $src\t# int" %}
 9514   ins_encode %{
 9515     __ andl($dst$$Register, $src$$Register);
 9516   %}
 9517   ins_pipe(ialu_reg_reg);
 9518 %}
 9519 
 9520 // And Register with Immediate 255
 9521 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
 9522 %{
 9523   match(Set dst (AndI dst src));
 9524 
 9525   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
 9526   ins_encode %{
 9527     __ movzbl($dst$$Register, $dst$$Register);
 9528   %}
 9529   ins_pipe(ialu_reg);
 9530 %}
 9531 
 9532 // And Register with Immediate 255 and promote to long
 9533 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 9534 %{
 9535   match(Set dst (ConvI2L (AndI src mask)));
 9536 
 9537   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 9538   ins_encode %{
 9539     __ movzbl($dst$$Register, $src$$Register);
 9540   %}
 9541   ins_pipe(ialu_reg);
 9542 %}
 9543 
 9544 // And Register with Immediate 65535
 9545 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
 9546 %{
 9547   match(Set dst (AndI dst src));
 9548 
 9549   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
 9550   ins_encode %{
 9551     __ movzwl($dst$$Register, $dst$$Register);
 9552   %}
 9553   ins_pipe(ialu_reg);
 9554 %}
 9555 
 9556 // And Register with Immediate 65535 and promote to long
 9557 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 9558 %{
 9559   match(Set dst (ConvI2L (AndI src mask)));
 9560 
 9561   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 9562   ins_encode %{
 9563     __ movzwl($dst$$Register, $src$$Register);
 9564   %}
 9565   ins_pipe(ialu_reg);
 9566 %}
 9567 
 9568 // Can skip int2long conversions after AND with small bitmask
 9569 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 9570 %{
 9571   predicate(VM_Version::supports_bmi2());
 9572   ins_cost(125);
 9573   effect(TEMP tmp, KILL cr);
 9574   match(Set dst (ConvI2L (AndI src mask)));
 9575   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 9576   ins_encode %{
 9577     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 9578     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 9579   %}
 9580   ins_pipe(ialu_reg_reg);
 9581 %}
 9582 
 9583 // And Register with Immediate
 9584 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9585 %{
 9586   match(Set dst (AndI dst src));
 9587   effect(KILL cr);
 9588 
 9589   format %{ "andl    $dst, $src\t# int" %}
 9590   ins_encode %{
 9591     __ andl($dst$$Register, $src$$constant);
 9592   %}
 9593   ins_pipe(ialu_reg);
 9594 %}
 9595 
 9596 // And Register with Memory
 9597 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9598 %{
 9599   match(Set dst (AndI dst (LoadI src)));
 9600   effect(KILL cr);
 9601 
 9602   ins_cost(125);
 9603   format %{ "andl    $dst, $src\t# int" %}
 9604   ins_encode %{
 9605     __ andl($dst$$Register, $src$$Address);
 9606   %}
 9607   ins_pipe(ialu_reg_mem);
 9608 %}
 9609 
 9610 // And Memory with Register
 9611 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9612 %{
 9613   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 9614   effect(KILL cr);
 9615 
 9616   ins_cost(150);
 9617   format %{ "andb    $dst, $src\t# byte" %}
 9618   ins_encode %{
 9619     __ andb($dst$$Address, $src$$Register);
 9620   %}
 9621   ins_pipe(ialu_mem_reg);
 9622 %}
 9623 
 9624 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9625 %{
 9626   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9627   effect(KILL cr);
 9628 
 9629   ins_cost(150);
 9630   format %{ "andl    $dst, $src\t# int" %}
 9631   ins_encode %{
 9632     __ andl($dst$$Address, $src$$Register);
 9633   %}
 9634   ins_pipe(ialu_mem_reg);
 9635 %}
 9636 
 9637 // And Memory with Immediate
 9638 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9639 %{
 9640   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9641   effect(KILL cr);
 9642 
 9643   ins_cost(125);
 9644   format %{ "andl    $dst, $src\t# int" %}
 9645   ins_encode %{
 9646     __ andl($dst$$Address, $src$$constant);
 9647   %}
 9648   ins_pipe(ialu_mem_imm);
 9649 %}
 9650 
 9651 // BMI1 instructions
 9652 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9653   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9654   predicate(UseBMI1Instructions);
 9655   effect(KILL cr);
 9656 
 9657   ins_cost(125);
 9658   format %{ "andnl  $dst, $src1, $src2" %}
 9659 
 9660   ins_encode %{
 9661     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9662   %}
 9663   ins_pipe(ialu_reg_mem);
 9664 %}
 9665 
 9666 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9667   match(Set dst (AndI (XorI src1 minus_1) src2));
 9668   predicate(UseBMI1Instructions);
 9669   effect(KILL cr);
 9670 
 9671   format %{ "andnl  $dst, $src1, $src2" %}
 9672 
 9673   ins_encode %{
 9674     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9675   %}
 9676   ins_pipe(ialu_reg);
 9677 %}
 9678 
 9679 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9680   match(Set dst (AndI (SubI imm_zero src) src));
 9681   predicate(UseBMI1Instructions);
 9682   effect(KILL cr);
 9683 
 9684   format %{ "blsil  $dst, $src" %}
 9685 
 9686   ins_encode %{
 9687     __ blsil($dst$$Register, $src$$Register);
 9688   %}
 9689   ins_pipe(ialu_reg);
 9690 %}
 9691 
 9692 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9693   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9694   predicate(UseBMI1Instructions);
 9695   effect(KILL cr);
 9696 
 9697   ins_cost(125);
 9698   format %{ "blsil  $dst, $src" %}
 9699 
 9700   ins_encode %{
 9701     __ blsil($dst$$Register, $src$$Address);
 9702   %}
 9703   ins_pipe(ialu_reg_mem);
 9704 %}
 9705 
 9706 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9707 %{
 9708   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9709   predicate(UseBMI1Instructions);
 9710   effect(KILL cr);
 9711 
 9712   ins_cost(125);
 9713   format %{ "blsmskl $dst, $src" %}
 9714 
 9715   ins_encode %{
 9716     __ blsmskl($dst$$Register, $src$$Address);
 9717   %}
 9718   ins_pipe(ialu_reg_mem);
 9719 %}
 9720 
 9721 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9722 %{
 9723   match(Set dst (XorI (AddI src minus_1) src));
 9724   predicate(UseBMI1Instructions);
 9725   effect(KILL cr);
 9726 
 9727   format %{ "blsmskl $dst, $src" %}
 9728 
 9729   ins_encode %{
 9730     __ blsmskl($dst$$Register, $src$$Register);
 9731   %}
 9732 
 9733   ins_pipe(ialu_reg);
 9734 %}
 9735 
 9736 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9737 %{
 9738   match(Set dst (AndI (AddI src minus_1) src) );
 9739   predicate(UseBMI1Instructions);
 9740   effect(KILL cr);
 9741 
 9742   format %{ "blsrl  $dst, $src" %}
 9743 
 9744   ins_encode %{
 9745     __ blsrl($dst$$Register, $src$$Register);
 9746   %}
 9747 
 9748   ins_pipe(ialu_reg_mem);
 9749 %}
 9750 
 9751 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9752 %{
 9753   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9754   predicate(UseBMI1Instructions);
 9755   effect(KILL cr);
 9756 
 9757   ins_cost(125);
 9758   format %{ "blsrl  $dst, $src" %}
 9759 
 9760   ins_encode %{
 9761     __ blsrl($dst$$Register, $src$$Address);
 9762   %}
 9763 
 9764   ins_pipe(ialu_reg);
 9765 %}
 9766 
 9767 // Or Instructions
 9768 // Or Register with Register
 9769 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9770 %{
 9771   match(Set dst (OrI dst src));
 9772   effect(KILL cr);
 9773 
 9774   format %{ "orl     $dst, $src\t# int" %}
 9775   ins_encode %{
 9776     __ orl($dst$$Register, $src$$Register);
 9777   %}
 9778   ins_pipe(ialu_reg_reg);
 9779 %}
 9780 
 9781 // Or Register with Immediate
 9782 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9783 %{
 9784   match(Set dst (OrI dst src));
 9785   effect(KILL cr);
 9786 
 9787   format %{ "orl     $dst, $src\t# int" %}
 9788   ins_encode %{
 9789     __ orl($dst$$Register, $src$$constant);
 9790   %}
 9791   ins_pipe(ialu_reg);
 9792 %}
 9793 
 9794 // Or Register with Memory
 9795 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9796 %{
 9797   match(Set dst (OrI dst (LoadI src)));
 9798   effect(KILL cr);
 9799 
 9800   ins_cost(125);
 9801   format %{ "orl     $dst, $src\t# int" %}
 9802   ins_encode %{
 9803     __ orl($dst$$Register, $src$$Address);
 9804   %}
 9805   ins_pipe(ialu_reg_mem);
 9806 %}
 9807 
 9808 // Or Memory with Register
 9809 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9810 %{
 9811   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9812   effect(KILL cr);
 9813 
 9814   ins_cost(150);
 9815   format %{ "orb    $dst, $src\t# byte" %}
 9816   ins_encode %{
 9817     __ orb($dst$$Address, $src$$Register);
 9818   %}
 9819   ins_pipe(ialu_mem_reg);
 9820 %}
 9821 
 9822 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9823 %{
 9824   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9825   effect(KILL cr);
 9826 
 9827   ins_cost(150);
 9828   format %{ "orl     $dst, $src\t# int" %}
 9829   ins_encode %{
 9830     __ orl($dst$$Address, $src$$Register);
 9831   %}
 9832   ins_pipe(ialu_mem_reg);
 9833 %}
 9834 
 9835 // Or Memory with Immediate
 9836 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9837 %{
 9838   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9839   effect(KILL cr);
 9840 
 9841   ins_cost(125);
 9842   format %{ "orl     $dst, $src\t# int" %}
 9843   ins_encode %{
 9844     __ orl($dst$$Address, $src$$constant);
 9845   %}
 9846   ins_pipe(ialu_mem_imm);
 9847 %}
 9848 
 9849 // Xor Instructions
 9850 // Xor Register with Register
 9851 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9852 %{
 9853   match(Set dst (XorI dst src));
 9854   effect(KILL cr);
 9855 
 9856   format %{ "xorl    $dst, $src\t# int" %}
 9857   ins_encode %{
 9858     __ xorl($dst$$Register, $src$$Register);
 9859   %}
 9860   ins_pipe(ialu_reg_reg);
 9861 %}
 9862 
 9863 // Xor Register with Immediate -1
 9864 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9865   match(Set dst (XorI dst imm));
 9866 
 9867   format %{ "not    $dst" %}
 9868   ins_encode %{
 9869      __ notl($dst$$Register);
 9870   %}
 9871   ins_pipe(ialu_reg);
 9872 %}
 9873 
 9874 // Xor Register with Immediate
 9875 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9876 %{
 9877   match(Set dst (XorI dst src));
 9878   effect(KILL cr);
 9879 
 9880   format %{ "xorl    $dst, $src\t# int" %}
 9881   ins_encode %{
 9882     __ xorl($dst$$Register, $src$$constant);
 9883   %}
 9884   ins_pipe(ialu_reg);
 9885 %}
 9886 
 9887 // Xor Register with Memory
 9888 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9889 %{
 9890   match(Set dst (XorI dst (LoadI src)));
 9891   effect(KILL cr);
 9892 
 9893   ins_cost(125);
 9894   format %{ "xorl    $dst, $src\t# int" %}
 9895   ins_encode %{
 9896     __ xorl($dst$$Register, $src$$Address);
 9897   %}
 9898   ins_pipe(ialu_reg_mem);
 9899 %}
 9900 
 9901 // Xor Memory with Register
 9902 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9903 %{
 9904   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9905   effect(KILL cr);
 9906 
 9907   ins_cost(150);
 9908   format %{ "xorb    $dst, $src\t# byte" %}
 9909   ins_encode %{
 9910     __ xorb($dst$$Address, $src$$Register);
 9911   %}
 9912   ins_pipe(ialu_mem_reg);
 9913 %}
 9914 
 9915 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9916 %{
 9917   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9918   effect(KILL cr);
 9919 
 9920   ins_cost(150);
 9921   format %{ "xorl    $dst, $src\t# int" %}
 9922   ins_encode %{
 9923     __ xorl($dst$$Address, $src$$Register);
 9924   %}
 9925   ins_pipe(ialu_mem_reg);
 9926 %}
 9927 
 9928 // Xor Memory with Immediate
 9929 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9930 %{
 9931   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9932   effect(KILL cr);
 9933 
 9934   ins_cost(125);
 9935   format %{ "xorl    $dst, $src\t# int" %}
 9936   ins_encode %{
 9937     __ xorl($dst$$Address, $src$$constant);
 9938   %}
 9939   ins_pipe(ialu_mem_imm);
 9940 %}
 9941 
 9942 
 9943 // Long Logical Instructions
 9944 
 9945 // And Instructions
 9946 // And Register with Register
 9947 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9948 %{
 9949   match(Set dst (AndL dst src));
 9950   effect(KILL cr);
 9951 
 9952   format %{ "andq    $dst, $src\t# long" %}
 9953   ins_encode %{
 9954     __ andq($dst$$Register, $src$$Register);
 9955   %}
 9956   ins_pipe(ialu_reg_reg);
 9957 %}
 9958 
 9959 // And Register with Immediate 255
 9960 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
 9961 %{
 9962   match(Set dst (AndL dst src));
 9963 
 9964   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
 9965   ins_encode %{
 9966     __ movzbq($dst$$Register, $dst$$Register);
 9967   %}
 9968   ins_pipe(ialu_reg);
 9969 %}
 9970 
 9971 // And Register with Immediate 65535
 9972 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
 9973 %{
 9974   match(Set dst (AndL dst src));
 9975 
 9976   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
 9977   ins_encode %{
 9978     __ movzwq($dst$$Register, $dst$$Register);
 9979   %}
 9980   ins_pipe(ialu_reg);
 9981 %}
 9982 
 9983 // And Register with Immediate
 9984 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9985 %{
 9986   match(Set dst (AndL dst src));
 9987   effect(KILL cr);
 9988 
 9989   format %{ "andq    $dst, $src\t# long" %}
 9990   ins_encode %{
 9991     __ andq($dst$$Register, $src$$constant);
 9992   %}
 9993   ins_pipe(ialu_reg);
 9994 %}
 9995 
 9996 // And Register with Memory
 9997 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9998 %{
 9999   match(Set dst (AndL dst (LoadL src)));
10000   effect(KILL cr);
10001 
10002   ins_cost(125);
10003   format %{ "andq    $dst, $src\t# long" %}
10004   ins_encode %{
10005     __ andq($dst$$Register, $src$$Address);
10006   %}
10007   ins_pipe(ialu_reg_mem);
10008 %}
10009 
10010 // And Memory with Register
10011 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10012 %{
10013   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10014   effect(KILL cr);
10015 
10016   ins_cost(150);
10017   format %{ "andq    $dst, $src\t# long" %}
10018   ins_encode %{
10019     __ andq($dst$$Address, $src$$Register);
10020   %}
10021   ins_pipe(ialu_mem_reg);
10022 %}
10023 
10024 // And Memory with Immediate
10025 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10026 %{
10027   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10028   effect(KILL cr);
10029 
10030   ins_cost(125);
10031   format %{ "andq    $dst, $src\t# long" %}
10032   ins_encode %{
10033     __ andq($dst$$Address, $src$$constant);
10034   %}
10035   ins_pipe(ialu_mem_imm);
10036 %}
10037 
10038 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10039 %{
10040   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10041   // because AND/OR works well enough for 8/32-bit values.
10042   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10043 
10044   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10045   effect(KILL cr);
10046 
10047   ins_cost(125);
10048   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10049   ins_encode %{
10050     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10051   %}
10052   ins_pipe(ialu_mem_imm);
10053 %}
10054 
10055 // BMI1 instructions
10056 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10057   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10058   predicate(UseBMI1Instructions);
10059   effect(KILL cr);
10060 
10061   ins_cost(125);
10062   format %{ "andnq  $dst, $src1, $src2" %}
10063 
10064   ins_encode %{
10065     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10066   %}
10067   ins_pipe(ialu_reg_mem);
10068 %}
10069 
10070 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10071   match(Set dst (AndL (XorL src1 minus_1) src2));
10072   predicate(UseBMI1Instructions);
10073   effect(KILL cr);
10074 
10075   format %{ "andnq  $dst, $src1, $src2" %}
10076 
10077   ins_encode %{
10078   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10079   %}
10080   ins_pipe(ialu_reg_mem);
10081 %}
10082 
10083 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10084   match(Set dst (AndL (SubL imm_zero src) src));
10085   predicate(UseBMI1Instructions);
10086   effect(KILL cr);
10087 
10088   format %{ "blsiq  $dst, $src" %}
10089 
10090   ins_encode %{
10091     __ blsiq($dst$$Register, $src$$Register);
10092   %}
10093   ins_pipe(ialu_reg);
10094 %}
10095 
10096 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10097   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10098   predicate(UseBMI1Instructions);
10099   effect(KILL cr);
10100 
10101   ins_cost(125);
10102   format %{ "blsiq  $dst, $src" %}
10103 
10104   ins_encode %{
10105     __ blsiq($dst$$Register, $src$$Address);
10106   %}
10107   ins_pipe(ialu_reg_mem);
10108 %}
10109 
10110 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10111 %{
10112   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10113   predicate(UseBMI1Instructions);
10114   effect(KILL cr);
10115 
10116   ins_cost(125);
10117   format %{ "blsmskq $dst, $src" %}
10118 
10119   ins_encode %{
10120     __ blsmskq($dst$$Register, $src$$Address);
10121   %}
10122   ins_pipe(ialu_reg_mem);
10123 %}
10124 
10125 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10126 %{
10127   match(Set dst (XorL (AddL src minus_1) src));
10128   predicate(UseBMI1Instructions);
10129   effect(KILL cr);
10130 
10131   format %{ "blsmskq $dst, $src" %}
10132 
10133   ins_encode %{
10134     __ blsmskq($dst$$Register, $src$$Register);
10135   %}
10136 
10137   ins_pipe(ialu_reg);
10138 %}
10139 
10140 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10141 %{
10142   match(Set dst (AndL (AddL src minus_1) src) );
10143   predicate(UseBMI1Instructions);
10144   effect(KILL cr);
10145 
10146   format %{ "blsrq  $dst, $src" %}
10147 
10148   ins_encode %{
10149     __ blsrq($dst$$Register, $src$$Register);
10150   %}
10151 
10152   ins_pipe(ialu_reg);
10153 %}
10154 
10155 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10156 %{
10157   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10158   predicate(UseBMI1Instructions);
10159   effect(KILL cr);
10160 
10161   ins_cost(125);
10162   format %{ "blsrq  $dst, $src" %}
10163 
10164   ins_encode %{
10165     __ blsrq($dst$$Register, $src$$Address);
10166   %}
10167 
10168   ins_pipe(ialu_reg);
10169 %}
10170 
10171 // Or Instructions
10172 // Or Register with Register
10173 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10174 %{
10175   match(Set dst (OrL dst src));
10176   effect(KILL cr);
10177 
10178   format %{ "orq     $dst, $src\t# long" %}
10179   ins_encode %{
10180     __ orq($dst$$Register, $src$$Register);
10181   %}
10182   ins_pipe(ialu_reg_reg);
10183 %}
10184 
10185 // Use any_RegP to match R15 (TLS register) without spilling.
10186 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10187   match(Set dst (OrL dst (CastP2X src)));
10188   effect(KILL cr);
10189 
10190   format %{ "orq     $dst, $src\t# long" %}
10191   ins_encode %{
10192     __ orq($dst$$Register, $src$$Register);
10193   %}
10194   ins_pipe(ialu_reg_reg);
10195 %}
10196 
10197 
10198 // Or Register with Immediate
10199 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10200 %{
10201   match(Set dst (OrL dst src));
10202   effect(KILL cr);
10203 
10204   format %{ "orq     $dst, $src\t# long" %}
10205   ins_encode %{
10206     __ orq($dst$$Register, $src$$constant);
10207   %}
10208   ins_pipe(ialu_reg);
10209 %}
10210 
10211 // Or Register with Memory
10212 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10213 %{
10214   match(Set dst (OrL dst (LoadL src)));
10215   effect(KILL cr);
10216 
10217   ins_cost(125);
10218   format %{ "orq     $dst, $src\t# long" %}
10219   ins_encode %{
10220     __ orq($dst$$Register, $src$$Address);
10221   %}
10222   ins_pipe(ialu_reg_mem);
10223 %}
10224 
10225 // Or Memory with Register
10226 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10227 %{
10228   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10229   effect(KILL cr);
10230 
10231   ins_cost(150);
10232   format %{ "orq     $dst, $src\t# long" %}
10233   ins_encode %{
10234     __ orq($dst$$Address, $src$$Register);
10235   %}
10236   ins_pipe(ialu_mem_reg);
10237 %}
10238 
10239 // Or Memory with Immediate
10240 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10241 %{
10242   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10243   effect(KILL cr);
10244 
10245   ins_cost(125);
10246   format %{ "orq     $dst, $src\t# long" %}
10247   ins_encode %{
10248     __ orq($dst$$Address, $src$$constant);
10249   %}
10250   ins_pipe(ialu_mem_imm);
10251 %}
10252 
10253 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
10254 %{
10255   // con should be a pure 64-bit power of 2 immediate
10256   // because AND/OR works well enough for 8/32-bit values.
10257   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
10258 
10259   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
10260   effect(KILL cr);
10261 
10262   ins_cost(125);
10263   format %{ "btsq    $dst, log2($con)\t# long" %}
10264   ins_encode %{
10265     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
10266   %}
10267   ins_pipe(ialu_mem_imm);
10268 %}
10269 
10270 // Xor Instructions
10271 // Xor Register with Register
10272 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10273 %{
10274   match(Set dst (XorL dst src));
10275   effect(KILL cr);
10276 
10277   format %{ "xorq    $dst, $src\t# long" %}
10278   ins_encode %{
10279     __ xorq($dst$$Register, $src$$Register);
10280   %}
10281   ins_pipe(ialu_reg_reg);
10282 %}
10283 
10284 // Xor Register with Immediate -1
10285 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10286   match(Set dst (XorL dst imm));
10287 
10288   format %{ "notq   $dst" %}
10289   ins_encode %{
10290      __ notq($dst$$Register);
10291   %}
10292   ins_pipe(ialu_reg);
10293 %}
10294 
10295 // Xor Register with Immediate
10296 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10297 %{
10298   match(Set dst (XorL dst src));
10299   effect(KILL cr);
10300 
10301   format %{ "xorq    $dst, $src\t# long" %}
10302   ins_encode %{
10303     __ xorq($dst$$Register, $src$$constant);
10304   %}
10305   ins_pipe(ialu_reg);
10306 %}
10307 
10308 // Xor Register with Memory
10309 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10310 %{
10311   match(Set dst (XorL dst (LoadL src)));
10312   effect(KILL cr);
10313 
10314   ins_cost(125);
10315   format %{ "xorq    $dst, $src\t# long" %}
10316   ins_encode %{
10317     __ xorq($dst$$Register, $src$$Address);
10318   %}
10319   ins_pipe(ialu_reg_mem);
10320 %}
10321 
10322 // Xor Memory with Register
10323 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10324 %{
10325   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10326   effect(KILL cr);
10327 
10328   ins_cost(150);
10329   format %{ "xorq    $dst, $src\t# long" %}
10330   ins_encode %{
10331     __ xorq($dst$$Address, $src$$Register);
10332   %}
10333   ins_pipe(ialu_mem_reg);
10334 %}
10335 
10336 // Xor Memory with Immediate
10337 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10338 %{
10339   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10340   effect(KILL cr);
10341 
10342   ins_cost(125);
10343   format %{ "xorq    $dst, $src\t# long" %}
10344   ins_encode %{
10345     __ xorq($dst$$Address, $src$$constant);
10346   %}
10347   ins_pipe(ialu_mem_imm);
10348 %}
10349 
10350 // Convert Int to Boolean
10351 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10352 %{
10353   match(Set dst (Conv2B src));
10354   effect(KILL cr);
10355 
10356   format %{ "testl   $src, $src\t# ci2b\n\t"
10357             "setnz   $dst\n\t"
10358             "movzbl  $dst, $dst" %}
10359   ins_encode %{
10360     __ testl($src$$Register, $src$$Register);
10361     __ set_byte_if_not_zero($dst$$Register);
10362     __ movzbl($dst$$Register, $dst$$Register);
10363   %}
10364   ins_pipe(pipe_slow); // XXX
10365 %}
10366 
10367 // Convert Pointer to Boolean
10368 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10369 %{
10370   match(Set dst (Conv2B src));
10371   effect(KILL cr);
10372 
10373   format %{ "testq   $src, $src\t# cp2b\n\t"
10374             "setnz   $dst\n\t"
10375             "movzbl  $dst, $dst" %}
10376   ins_encode %{
10377     __ testq($src$$Register, $src$$Register);
10378     __ set_byte_if_not_zero($dst$$Register);
10379     __ movzbl($dst$$Register, $dst$$Register);
10380   %}
10381   ins_pipe(pipe_slow); // XXX
10382 %}
10383 
10384 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10385 %{
10386   match(Set dst (CmpLTMask p q));
10387   effect(KILL cr);
10388 
10389   ins_cost(400);
10390   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10391             "setlt   $dst\n\t"
10392             "movzbl  $dst, $dst\n\t"
10393             "negl    $dst" %}
10394   ins_encode %{
10395     __ cmpl($p$$Register, $q$$Register);
10396     __ setl($dst$$Register);
10397     __ movzbl($dst$$Register, $dst$$Register);
10398     __ negl($dst$$Register);
10399   %}
10400   ins_pipe(pipe_slow);
10401 %}
10402 
10403 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
10404 %{
10405   match(Set dst (CmpLTMask dst zero));
10406   effect(KILL cr);
10407 
10408   ins_cost(100);
10409   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10410   ins_encode %{
10411     __ sarl($dst$$Register, 31);
10412   %}
10413   ins_pipe(ialu_reg);
10414 %}
10415 
10416 /* Better to save a register than avoid a branch */
10417 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10418 %{
10419   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10420   effect(KILL cr);
10421   ins_cost(300);
10422   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10423             "jge     done\n\t"
10424             "addl    $p,$y\n"
10425             "done:   " %}
10426   ins_encode %{
10427     Register Rp = $p$$Register;
10428     Register Rq = $q$$Register;
10429     Register Ry = $y$$Register;
10430     Label done;
10431     __ subl(Rp, Rq);
10432     __ jccb(Assembler::greaterEqual, done);
10433     __ addl(Rp, Ry);
10434     __ bind(done);
10435   %}
10436   ins_pipe(pipe_cmplt);
10437 %}
10438 
10439 /* Better to save a register than avoid a branch */
10440 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10441 %{
10442   match(Set y (AndI (CmpLTMask p q) y));
10443   effect(KILL cr);
10444 
10445   ins_cost(300);
10446 
10447   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10448             "jlt     done\n\t"
10449             "xorl    $y, $y\n"
10450             "done:   " %}
10451   ins_encode %{
10452     Register Rp = $p$$Register;
10453     Register Rq = $q$$Register;
10454     Register Ry = $y$$Register;
10455     Label done;
10456     __ cmpl(Rp, Rq);
10457     __ jccb(Assembler::less, done);
10458     __ xorl(Ry, Ry);
10459     __ bind(done);
10460   %}
10461   ins_pipe(pipe_cmplt);
10462 %}
10463 
10464 
10465 //---------- FP Instructions------------------------------------------------
10466 
10467 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10468 %{
10469   match(Set cr (CmpF src1 src2));
10470 
10471   ins_cost(145);
10472   format %{ "ucomiss $src1, $src2\n\t"
10473             "jnp,s   exit\n\t"
10474             "pushfq\t# saw NaN, set CF\n\t"
10475             "andq    [rsp], #0xffffff2b\n\t"
10476             "popfq\n"
10477     "exit:" %}
10478   ins_encode %{
10479     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10480     emit_cmpfp_fixup(_masm);
10481   %}
10482   ins_pipe(pipe_slow);
10483 %}
10484 
10485 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10486   match(Set cr (CmpF src1 src2));
10487 
10488   ins_cost(100);
10489   format %{ "ucomiss $src1, $src2" %}
10490   ins_encode %{
10491     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10492   %}
10493   ins_pipe(pipe_slow);
10494 %}
10495 
10496 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10497 %{
10498   match(Set cr (CmpF src1 (LoadF src2)));
10499 
10500   ins_cost(145);
10501   format %{ "ucomiss $src1, $src2\n\t"
10502             "jnp,s   exit\n\t"
10503             "pushfq\t# saw NaN, set CF\n\t"
10504             "andq    [rsp], #0xffffff2b\n\t"
10505             "popfq\n"
10506     "exit:" %}
10507   ins_encode %{
10508     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10509     emit_cmpfp_fixup(_masm);
10510   %}
10511   ins_pipe(pipe_slow);
10512 %}
10513 
10514 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10515   match(Set cr (CmpF src1 (LoadF src2)));
10516 
10517   ins_cost(100);
10518   format %{ "ucomiss $src1, $src2" %}
10519   ins_encode %{
10520     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10521   %}
10522   ins_pipe(pipe_slow);
10523 %}
10524 
10525 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10526   match(Set cr (CmpF src con));
10527 
10528   ins_cost(145);
10529   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10530             "jnp,s   exit\n\t"
10531             "pushfq\t# saw NaN, set CF\n\t"
10532             "andq    [rsp], #0xffffff2b\n\t"
10533             "popfq\n"
10534     "exit:" %}
10535   ins_encode %{
10536     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10537     emit_cmpfp_fixup(_masm);
10538   %}
10539   ins_pipe(pipe_slow);
10540 %}
10541 
10542 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10543   match(Set cr (CmpF src con));
10544   ins_cost(100);
10545   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10546   ins_encode %{
10547     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10548   %}
10549   ins_pipe(pipe_slow);
10550 %}
10551 
10552 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10553 %{
10554   match(Set cr (CmpD src1 src2));
10555 
10556   ins_cost(145);
10557   format %{ "ucomisd $src1, $src2\n\t"
10558             "jnp,s   exit\n\t"
10559             "pushfq\t# saw NaN, set CF\n\t"
10560             "andq    [rsp], #0xffffff2b\n\t"
10561             "popfq\n"
10562     "exit:" %}
10563   ins_encode %{
10564     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10565     emit_cmpfp_fixup(_masm);
10566   %}
10567   ins_pipe(pipe_slow);
10568 %}
10569 
10570 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10571   match(Set cr (CmpD src1 src2));
10572 
10573   ins_cost(100);
10574   format %{ "ucomisd $src1, $src2 test" %}
10575   ins_encode %{
10576     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10577   %}
10578   ins_pipe(pipe_slow);
10579 %}
10580 
10581 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10582 %{
10583   match(Set cr (CmpD src1 (LoadD src2)));
10584 
10585   ins_cost(145);
10586   format %{ "ucomisd $src1, $src2\n\t"
10587             "jnp,s   exit\n\t"
10588             "pushfq\t# saw NaN, set CF\n\t"
10589             "andq    [rsp], #0xffffff2b\n\t"
10590             "popfq\n"
10591     "exit:" %}
10592   ins_encode %{
10593     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10594     emit_cmpfp_fixup(_masm);
10595   %}
10596   ins_pipe(pipe_slow);
10597 %}
10598 
10599 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10600   match(Set cr (CmpD src1 (LoadD src2)));
10601 
10602   ins_cost(100);
10603   format %{ "ucomisd $src1, $src2" %}
10604   ins_encode %{
10605     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10606   %}
10607   ins_pipe(pipe_slow);
10608 %}
10609 
10610 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10611   match(Set cr (CmpD src con));
10612 
10613   ins_cost(145);
10614   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10615             "jnp,s   exit\n\t"
10616             "pushfq\t# saw NaN, set CF\n\t"
10617             "andq    [rsp], #0xffffff2b\n\t"
10618             "popfq\n"
10619     "exit:" %}
10620   ins_encode %{
10621     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10622     emit_cmpfp_fixup(_masm);
10623   %}
10624   ins_pipe(pipe_slow);
10625 %}
10626 
10627 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10628   match(Set cr (CmpD src con));
10629   ins_cost(100);
10630   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10631   ins_encode %{
10632     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10633   %}
10634   ins_pipe(pipe_slow);
10635 %}
10636 
10637 // Compare into -1,0,1
10638 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10639 %{
10640   match(Set dst (CmpF3 src1 src2));
10641   effect(KILL cr);
10642 
10643   ins_cost(275);
10644   format %{ "ucomiss $src1, $src2\n\t"
10645             "movl    $dst, #-1\n\t"
10646             "jp,s    done\n\t"
10647             "jb,s    done\n\t"
10648             "setne   $dst\n\t"
10649             "movzbl  $dst, $dst\n"
10650     "done:" %}
10651   ins_encode %{
10652     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10653     emit_cmpfp3(_masm, $dst$$Register);
10654   %}
10655   ins_pipe(pipe_slow);
10656 %}
10657 
10658 // Compare into -1,0,1
10659 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10660 %{
10661   match(Set dst (CmpF3 src1 (LoadF src2)));
10662   effect(KILL cr);
10663 
10664   ins_cost(275);
10665   format %{ "ucomiss $src1, $src2\n\t"
10666             "movl    $dst, #-1\n\t"
10667             "jp,s    done\n\t"
10668             "jb,s    done\n\t"
10669             "setne   $dst\n\t"
10670             "movzbl  $dst, $dst\n"
10671     "done:" %}
10672   ins_encode %{
10673     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10674     emit_cmpfp3(_masm, $dst$$Register);
10675   %}
10676   ins_pipe(pipe_slow);
10677 %}
10678 
10679 // Compare into -1,0,1
10680 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10681   match(Set dst (CmpF3 src con));
10682   effect(KILL cr);
10683 
10684   ins_cost(275);
10685   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10686             "movl    $dst, #-1\n\t"
10687             "jp,s    done\n\t"
10688             "jb,s    done\n\t"
10689             "setne   $dst\n\t"
10690             "movzbl  $dst, $dst\n"
10691     "done:" %}
10692   ins_encode %{
10693     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10694     emit_cmpfp3(_masm, $dst$$Register);
10695   %}
10696   ins_pipe(pipe_slow);
10697 %}
10698 
10699 // Compare into -1,0,1
10700 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10701 %{
10702   match(Set dst (CmpD3 src1 src2));
10703   effect(KILL cr);
10704 
10705   ins_cost(275);
10706   format %{ "ucomisd $src1, $src2\n\t"
10707             "movl    $dst, #-1\n\t"
10708             "jp,s    done\n\t"
10709             "jb,s    done\n\t"
10710             "setne   $dst\n\t"
10711             "movzbl  $dst, $dst\n"
10712     "done:" %}
10713   ins_encode %{
10714     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10715     emit_cmpfp3(_masm, $dst$$Register);
10716   %}
10717   ins_pipe(pipe_slow);
10718 %}
10719 
10720 // Compare into -1,0,1
10721 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10722 %{
10723   match(Set dst (CmpD3 src1 (LoadD src2)));
10724   effect(KILL cr);
10725 
10726   ins_cost(275);
10727   format %{ "ucomisd $src1, $src2\n\t"
10728             "movl    $dst, #-1\n\t"
10729             "jp,s    done\n\t"
10730             "jb,s    done\n\t"
10731             "setne   $dst\n\t"
10732             "movzbl  $dst, $dst\n"
10733     "done:" %}
10734   ins_encode %{
10735     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10736     emit_cmpfp3(_masm, $dst$$Register);
10737   %}
10738   ins_pipe(pipe_slow);
10739 %}
10740 
10741 // Compare into -1,0,1
10742 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10743   match(Set dst (CmpD3 src con));
10744   effect(KILL cr);
10745 
10746   ins_cost(275);
10747   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10748             "movl    $dst, #-1\n\t"
10749             "jp,s    done\n\t"
10750             "jb,s    done\n\t"
10751             "setne   $dst\n\t"
10752             "movzbl  $dst, $dst\n"
10753     "done:" %}
10754   ins_encode %{
10755     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10756     emit_cmpfp3(_masm, $dst$$Register);
10757   %}
10758   ins_pipe(pipe_slow);
10759 %}
10760 
10761 //----------Arithmetic Conversion Instructions---------------------------------
10762 
10763 instruct convF2D_reg_reg(regD dst, regF src)
10764 %{
10765   match(Set dst (ConvF2D src));
10766 
10767   format %{ "cvtss2sd $dst, $src" %}
10768   ins_encode %{
10769     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10770   %}
10771   ins_pipe(pipe_slow); // XXX
10772 %}
10773 
10774 instruct convF2D_reg_mem(regD dst, memory src)
10775 %{
10776   match(Set dst (ConvF2D (LoadF src)));
10777 
10778   format %{ "cvtss2sd $dst, $src" %}
10779   ins_encode %{
10780     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10781   %}
10782   ins_pipe(pipe_slow); // XXX
10783 %}
10784 
10785 instruct convD2F_reg_reg(regF dst, regD src)
10786 %{
10787   match(Set dst (ConvD2F src));
10788 
10789   format %{ "cvtsd2ss $dst, $src" %}
10790   ins_encode %{
10791     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10792   %}
10793   ins_pipe(pipe_slow); // XXX
10794 %}
10795 
10796 instruct convD2F_reg_mem(regF dst, memory src)
10797 %{
10798   match(Set dst (ConvD2F (LoadD src)));
10799 
10800   format %{ "cvtsd2ss $dst, $src" %}
10801   ins_encode %{
10802     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10803   %}
10804   ins_pipe(pipe_slow); // XXX
10805 %}
10806 
10807 // XXX do mem variants
10808 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10809 %{
10810   match(Set dst (ConvF2I src));
10811   effect(KILL cr);
10812   format %{ "convert_f2i $dst,$src" %}
10813   ins_encode %{
10814     __ convert_f2i($dst$$Register, $src$$XMMRegister);
10815   %}
10816   ins_pipe(pipe_slow);
10817 %}
10818 
10819 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10820 %{
10821   match(Set dst (ConvF2L src));
10822   effect(KILL cr);
10823   format %{ "convert_f2l $dst,$src"%}
10824   ins_encode %{
10825     __ convert_f2l($dst$$Register, $src$$XMMRegister);
10826   %}
10827   ins_pipe(pipe_slow);
10828 %}
10829 
10830 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10831 %{
10832   match(Set dst (ConvD2I src));
10833   effect(KILL cr);
10834   format %{ "convert_d2i $dst,$src"%}
10835   ins_encode %{
10836     __ convert_d2i($dst$$Register, $src$$XMMRegister);
10837   %}
10838   ins_pipe(pipe_slow);
10839 %}
10840 
10841 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10842 %{
10843   match(Set dst (ConvD2L src));
10844   effect(KILL cr);
10845   format %{ "convert_d2l $dst,$src"%}
10846   ins_encode %{
10847     __ convert_d2l($dst$$Register, $src$$XMMRegister);
10848   %}
10849   ins_pipe(pipe_slow);
10850 %}
10851 
10852 instruct convI2F_reg_reg(regF dst, rRegI src)
10853 %{
10854   predicate(!UseXmmI2F);
10855   match(Set dst (ConvI2F src));
10856 
10857   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10858   ins_encode %{
10859     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10860   %}
10861   ins_pipe(pipe_slow); // XXX
10862 %}
10863 
10864 instruct convI2F_reg_mem(regF dst, memory src)
10865 %{
10866   match(Set dst (ConvI2F (LoadI src)));
10867 
10868   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10869   ins_encode %{
10870     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10871   %}
10872   ins_pipe(pipe_slow); // XXX
10873 %}
10874 
10875 instruct convI2D_reg_reg(regD dst, rRegI src)
10876 %{
10877   predicate(!UseXmmI2D);
10878   match(Set dst (ConvI2D src));
10879 
10880   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10881   ins_encode %{
10882     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10883   %}
10884   ins_pipe(pipe_slow); // XXX
10885 %}
10886 
10887 instruct convI2D_reg_mem(regD dst, memory src)
10888 %{
10889   match(Set dst (ConvI2D (LoadI src)));
10890 
10891   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10892   ins_encode %{
10893     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10894   %}
10895   ins_pipe(pipe_slow); // XXX
10896 %}
10897 
10898 instruct convXI2F_reg(regF dst, rRegI src)
10899 %{
10900   predicate(UseXmmI2F);
10901   match(Set dst (ConvI2F src));
10902 
10903   format %{ "movdl $dst, $src\n\t"
10904             "cvtdq2psl $dst, $dst\t# i2f" %}
10905   ins_encode %{
10906     __ movdl($dst$$XMMRegister, $src$$Register);
10907     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10908   %}
10909   ins_pipe(pipe_slow); // XXX
10910 %}
10911 
10912 instruct convXI2D_reg(regD dst, rRegI src)
10913 %{
10914   predicate(UseXmmI2D);
10915   match(Set dst (ConvI2D src));
10916 
10917   format %{ "movdl $dst, $src\n\t"
10918             "cvtdq2pdl $dst, $dst\t# i2d" %}
10919   ins_encode %{
10920     __ movdl($dst$$XMMRegister, $src$$Register);
10921     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10922   %}
10923   ins_pipe(pipe_slow); // XXX
10924 %}
10925 
10926 instruct convL2F_reg_reg(regF dst, rRegL src)
10927 %{
10928   match(Set dst (ConvL2F src));
10929 
10930   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10931   ins_encode %{
10932     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10933   %}
10934   ins_pipe(pipe_slow); // XXX
10935 %}
10936 
10937 instruct convL2F_reg_mem(regF dst, memory src)
10938 %{
10939   match(Set dst (ConvL2F (LoadL src)));
10940 
10941   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10942   ins_encode %{
10943     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10944   %}
10945   ins_pipe(pipe_slow); // XXX
10946 %}
10947 
10948 instruct convL2D_reg_reg(regD dst, rRegL src)
10949 %{
10950   match(Set dst (ConvL2D src));
10951 
10952   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10953   ins_encode %{
10954     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10955   %}
10956   ins_pipe(pipe_slow); // XXX
10957 %}
10958 
10959 instruct convL2D_reg_mem(regD dst, memory src)
10960 %{
10961   match(Set dst (ConvL2D (LoadL src)));
10962 
10963   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10964   ins_encode %{
10965     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10966   %}
10967   ins_pipe(pipe_slow); // XXX
10968 %}
10969 
10970 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10971 %{
10972   match(Set dst (ConvI2L src));
10973 
10974   ins_cost(125);
10975   format %{ "movslq  $dst, $src\t# i2l" %}
10976   ins_encode %{
10977     __ movslq($dst$$Register, $src$$Register);
10978   %}
10979   ins_pipe(ialu_reg_reg);
10980 %}
10981 
10982 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10983 // %{
10984 //   match(Set dst (ConvI2L src));
10985 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10986 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10987 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10988 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10989 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10990 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10991 
10992 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10993 //   ins_encode(enc_copy(dst, src));
10994 // //   opcode(0x63); // needs REX.W
10995 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10996 //   ins_pipe(ialu_reg_reg);
10997 // %}
10998 
10999 // Zero-extend convert int to long
11000 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11001 %{
11002   match(Set dst (AndL (ConvI2L src) mask));
11003 
11004   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11005   ins_encode %{
11006     if ($dst$$reg != $src$$reg) {
11007       __ movl($dst$$Register, $src$$Register);
11008     }
11009   %}
11010   ins_pipe(ialu_reg_reg);
11011 %}
11012 
11013 // Zero-extend convert int to long
11014 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11015 %{
11016   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11017 
11018   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11019   ins_encode %{
11020     __ movl($dst$$Register, $src$$Address);
11021   %}
11022   ins_pipe(ialu_reg_mem);
11023 %}
11024 
11025 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11026 %{
11027   match(Set dst (AndL src mask));
11028 
11029   format %{ "movl    $dst, $src\t# zero-extend long" %}
11030   ins_encode %{
11031     __ movl($dst$$Register, $src$$Register);
11032   %}
11033   ins_pipe(ialu_reg_reg);
11034 %}
11035 
11036 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11037 %{
11038   match(Set dst (ConvL2I src));
11039 
11040   format %{ "movl    $dst, $src\t# l2i" %}
11041   ins_encode %{
11042     __ movl($dst$$Register, $src$$Register);
11043   %}
11044   ins_pipe(ialu_reg_reg);
11045 %}
11046 
11047 
11048 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11049   match(Set dst (MoveF2I src));
11050   effect(DEF dst, USE src);
11051 
11052   ins_cost(125);
11053   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11054   ins_encode %{
11055     __ movl($dst$$Register, Address(rsp, $src$$disp));
11056   %}
11057   ins_pipe(ialu_reg_mem);
11058 %}
11059 
11060 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11061   match(Set dst (MoveI2F src));
11062   effect(DEF dst, USE src);
11063 
11064   ins_cost(125);
11065   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11066   ins_encode %{
11067     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11068   %}
11069   ins_pipe(pipe_slow);
11070 %}
11071 
11072 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11073   match(Set dst (MoveD2L src));
11074   effect(DEF dst, USE src);
11075 
11076   ins_cost(125);
11077   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11078   ins_encode %{
11079     __ movq($dst$$Register, Address(rsp, $src$$disp));
11080   %}
11081   ins_pipe(ialu_reg_mem);
11082 %}
11083 
11084 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11085   predicate(!UseXmmLoadAndClearUpper);
11086   match(Set dst (MoveL2D src));
11087   effect(DEF dst, USE src);
11088 
11089   ins_cost(125);
11090   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11091   ins_encode %{
11092     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11093   %}
11094   ins_pipe(pipe_slow);
11095 %}
11096 
11097 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11098   predicate(UseXmmLoadAndClearUpper);
11099   match(Set dst (MoveL2D src));
11100   effect(DEF dst, USE src);
11101 
11102   ins_cost(125);
11103   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11104   ins_encode %{
11105     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11106   %}
11107   ins_pipe(pipe_slow);
11108 %}
11109 
11110 
11111 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11112   match(Set dst (MoveF2I src));
11113   effect(DEF dst, USE src);
11114 
11115   ins_cost(95); // XXX
11116   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11117   ins_encode %{
11118     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11119   %}
11120   ins_pipe(pipe_slow);
11121 %}
11122 
11123 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11124   match(Set dst (MoveI2F src));
11125   effect(DEF dst, USE src);
11126 
11127   ins_cost(100);
11128   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11129   ins_encode %{
11130     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11131   %}
11132   ins_pipe( ialu_mem_reg );
11133 %}
11134 
11135 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11136   match(Set dst (MoveD2L src));
11137   effect(DEF dst, USE src);
11138 
11139   ins_cost(95); // XXX
11140   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11141   ins_encode %{
11142     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11143   %}
11144   ins_pipe(pipe_slow);
11145 %}
11146 
11147 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11148   match(Set dst (MoveL2D src));
11149   effect(DEF dst, USE src);
11150 
11151   ins_cost(100);
11152   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11153   ins_encode %{
11154     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11155   %}
11156   ins_pipe(ialu_mem_reg);
11157 %}
11158 
11159 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11160   match(Set dst (MoveF2I src));
11161   effect(DEF dst, USE src);
11162   ins_cost(85);
11163   format %{ "movd    $dst,$src\t# MoveF2I" %}
11164   ins_encode %{
11165     __ movdl($dst$$Register, $src$$XMMRegister);
11166   %}
11167   ins_pipe( pipe_slow );
11168 %}
11169 
11170 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11171   match(Set dst (MoveD2L src));
11172   effect(DEF dst, USE src);
11173   ins_cost(85);
11174   format %{ "movd    $dst,$src\t# MoveD2L" %}
11175   ins_encode %{
11176     __ movdq($dst$$Register, $src$$XMMRegister);
11177   %}
11178   ins_pipe( pipe_slow );
11179 %}
11180 
11181 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11182   match(Set dst (MoveI2F src));
11183   effect(DEF dst, USE src);
11184   ins_cost(100);
11185   format %{ "movd    $dst,$src\t# MoveI2F" %}
11186   ins_encode %{
11187     __ movdl($dst$$XMMRegister, $src$$Register);
11188   %}
11189   ins_pipe( pipe_slow );
11190 %}
11191 
11192 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11193   match(Set dst (MoveL2D src));
11194   effect(DEF dst, USE src);
11195   ins_cost(100);
11196   format %{ "movd    $dst,$src\t# MoveL2D" %}
11197   ins_encode %{
11198      __ movdq($dst$$XMMRegister, $src$$Register);
11199   %}
11200   ins_pipe( pipe_slow );
11201 %}
11202 
11203 
11204 // Fast clearing of an array
11205 // Small ClearArray non-AVX512.
11206 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11207                   Universe dummy, rFlagsReg cr)
11208 %{
11209   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
11210   match(Set dummy (ClearArray (Binary cnt base) val));
11211   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
11212 
11213   format %{ $$template
11214     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11215     $$emit$$"jg      LARGE\n\t"
11216     $$emit$$"dec     rcx\n\t"
11217     $$emit$$"js      DONE\t# Zero length\n\t"
11218     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11219     $$emit$$"dec     rcx\n\t"
11220     $$emit$$"jge     LOOP\n\t"
11221     $$emit$$"jmp     DONE\n\t"
11222     $$emit$$"# LARGE:\n\t"
11223     if (UseFastStosb) {
11224        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11225        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11226     } else if (UseXMMForObjInit) {
11227        $$emit$$"movdq   $tmp, $val\n\t"
11228        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11229        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11230        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11231        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11232        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11233        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11234        $$emit$$"add     0x40,rax\n\t"
11235        $$emit$$"# L_zero_64_bytes:\n\t"
11236        $$emit$$"sub     0x8,rcx\n\t"
11237        $$emit$$"jge     L_loop\n\t"
11238        $$emit$$"add     0x4,rcx\n\t"
11239        $$emit$$"jl      L_tail\n\t"
11240        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11241        $$emit$$"add     0x20,rax\n\t"
11242        $$emit$$"sub     0x4,rcx\n\t"
11243        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11244        $$emit$$"add     0x4,rcx\n\t"
11245        $$emit$$"jle     L_end\n\t"
11246        $$emit$$"dec     rcx\n\t"
11247        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11248        $$emit$$"vmovq   xmm0,(rax)\n\t"
11249        $$emit$$"add     0x8,rax\n\t"
11250        $$emit$$"dec     rcx\n\t"
11251        $$emit$$"jge     L_sloop\n\t"
11252        $$emit$$"# L_end:\n\t"
11253     } else {
11254        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11255     }
11256     $$emit$$"# DONE"
11257   %}
11258   ins_encode %{
11259     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11260                  $tmp$$XMMRegister, false, false);
11261   %}
11262   ins_pipe(pipe_slow);
11263 %}
11264 
11265 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11266                             Universe dummy, rFlagsReg cr)
11267 %{
11268   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
11269   match(Set dummy (ClearArray (Binary cnt base) val));
11270   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
11271 
11272   format %{ $$template
11273     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11274     $$emit$$"jg      LARGE\n\t"
11275     $$emit$$"dec     rcx\n\t"
11276     $$emit$$"js      DONE\t# Zero length\n\t"
11277     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11278     $$emit$$"dec     rcx\n\t"
11279     $$emit$$"jge     LOOP\n\t"
11280     $$emit$$"jmp     DONE\n\t"
11281     $$emit$$"# LARGE:\n\t"
11282     if (UseXMMForObjInit) {
11283        $$emit$$"movdq   $tmp, $val\n\t"
11284        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11285        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11286        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11287        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11288        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11289        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11290        $$emit$$"add     0x40,rax\n\t"
11291        $$emit$$"# L_zero_64_bytes:\n\t"
11292        $$emit$$"sub     0x8,rcx\n\t"
11293        $$emit$$"jge     L_loop\n\t"
11294        $$emit$$"add     0x4,rcx\n\t"
11295        $$emit$$"jl      L_tail\n\t"
11296        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11297        $$emit$$"add     0x20,rax\n\t"
11298        $$emit$$"sub     0x4,rcx\n\t"
11299        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11300        $$emit$$"add     0x4,rcx\n\t"
11301        $$emit$$"jle     L_end\n\t"
11302        $$emit$$"dec     rcx\n\t"
11303        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11304        $$emit$$"vmovq   xmm0,(rax)\n\t"
11305        $$emit$$"add     0x8,rax\n\t"
11306        $$emit$$"dec     rcx\n\t"
11307        $$emit$$"jge     L_sloop\n\t"
11308        $$emit$$"# L_end:\n\t"
11309     } else {
11310        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11311     }
11312     $$emit$$"# DONE"
11313   %}
11314   ins_encode %{
11315     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11316                  $tmp$$XMMRegister, false, true);
11317   %}
11318   ins_pipe(pipe_slow);
11319 %}
11320 
11321 // Small ClearArray AVX512 non-constant length.
11322 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
11323                        Universe dummy, rFlagsReg cr)
11324 %{
11325   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
11326   match(Set dummy (ClearArray (Binary cnt base) val));
11327   ins_cost(125);
11328   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
11329 
11330   format %{ $$template
11331     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11332     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11333     $$emit$$"jg      LARGE\n\t"
11334     $$emit$$"dec     rcx\n\t"
11335     $$emit$$"js      DONE\t# Zero length\n\t"
11336     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11337     $$emit$$"dec     rcx\n\t"
11338     $$emit$$"jge     LOOP\n\t"
11339     $$emit$$"jmp     DONE\n\t"
11340     $$emit$$"# LARGE:\n\t"
11341     if (UseFastStosb) {
11342        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11343        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11344     } else if (UseXMMForObjInit) {
11345        $$emit$$"mov     rdi,rax\n\t"
11346        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11347        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11348        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11349        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11350        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11351        $$emit$$"add     0x40,rax\n\t"
11352        $$emit$$"# L_zero_64_bytes:\n\t"
11353        $$emit$$"sub     0x8,rcx\n\t"
11354        $$emit$$"jge     L_loop\n\t"
11355        $$emit$$"add     0x4,rcx\n\t"
11356        $$emit$$"jl      L_tail\n\t"
11357        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11358        $$emit$$"add     0x20,rax\n\t"
11359        $$emit$$"sub     0x4,rcx\n\t"
11360        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11361        $$emit$$"add     0x4,rcx\n\t"
11362        $$emit$$"jle     L_end\n\t"
11363        $$emit$$"dec     rcx\n\t"
11364        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11365        $$emit$$"vmovq   xmm0,(rax)\n\t"
11366        $$emit$$"add     0x8,rax\n\t"
11367        $$emit$$"dec     rcx\n\t"
11368        $$emit$$"jge     L_sloop\n\t"
11369        $$emit$$"# L_end:\n\t"
11370     } else {
11371        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11372     }
11373     $$emit$$"# DONE"
11374   %}
11375   ins_encode %{
11376     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11377                  $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
11378   %}
11379   ins_pipe(pipe_slow);
11380 %}
11381 
11382 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
11383                                  Universe dummy, rFlagsReg cr)
11384 %{
11385   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
11386   match(Set dummy (ClearArray (Binary cnt base) val));
11387   ins_cost(125);
11388   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
11389 
11390   format %{ $$template
11391     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11392     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11393     $$emit$$"jg      LARGE\n\t"
11394     $$emit$$"dec     rcx\n\t"
11395     $$emit$$"js      DONE\t# Zero length\n\t"
11396     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11397     $$emit$$"dec     rcx\n\t"
11398     $$emit$$"jge     LOOP\n\t"
11399     $$emit$$"jmp     DONE\n\t"
11400     $$emit$$"# LARGE:\n\t"
11401     if (UseFastStosb) {
11402        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11403        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11404     } else if (UseXMMForObjInit) {
11405        $$emit$$"mov     rdi,rax\n\t"
11406        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11407        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11408        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11409        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11410        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11411        $$emit$$"add     0x40,rax\n\t"
11412        $$emit$$"# L_zero_64_bytes:\n\t"
11413        $$emit$$"sub     0x8,rcx\n\t"
11414        $$emit$$"jge     L_loop\n\t"
11415        $$emit$$"add     0x4,rcx\n\t"
11416        $$emit$$"jl      L_tail\n\t"
11417        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11418        $$emit$$"add     0x20,rax\n\t"
11419        $$emit$$"sub     0x4,rcx\n\t"
11420        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11421        $$emit$$"add     0x4,rcx\n\t"
11422        $$emit$$"jle     L_end\n\t"
11423        $$emit$$"dec     rcx\n\t"
11424        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11425        $$emit$$"vmovq   xmm0,(rax)\n\t"
11426        $$emit$$"add     0x8,rax\n\t"
11427        $$emit$$"dec     rcx\n\t"
11428        $$emit$$"jge     L_sloop\n\t"
11429        $$emit$$"# L_end:\n\t"
11430     } else {
11431        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11432     }
11433     $$emit$$"# DONE"
11434   %}
11435   ins_encode %{
11436     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11437                  $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
11438   %}
11439   ins_pipe(pipe_slow);
11440 %}
11441 
11442 // Large ClearArray non-AVX512.
11443 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11444                         Universe dummy, rFlagsReg cr)
11445 %{
11446   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
11447   match(Set dummy (ClearArray (Binary cnt base) val));
11448   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
11449 
11450   format %{ $$template
11451     if (UseFastStosb) {
11452        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11453        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11454     } else if (UseXMMForObjInit) {
11455        $$emit$$"movdq   $tmp, $val\n\t"
11456        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11457        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11458        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11459        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11460        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11461        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11462        $$emit$$"add     0x40,rax\n\t"
11463        $$emit$$"# L_zero_64_bytes:\n\t"
11464        $$emit$$"sub     0x8,rcx\n\t"
11465        $$emit$$"jge     L_loop\n\t"
11466        $$emit$$"add     0x4,rcx\n\t"
11467        $$emit$$"jl      L_tail\n\t"
11468        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11469        $$emit$$"add     0x20,rax\n\t"
11470        $$emit$$"sub     0x4,rcx\n\t"
11471        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11472        $$emit$$"add     0x4,rcx\n\t"
11473        $$emit$$"jle     L_end\n\t"
11474        $$emit$$"dec     rcx\n\t"
11475        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11476        $$emit$$"vmovq   xmm0,(rax)\n\t"
11477        $$emit$$"add     0x8,rax\n\t"
11478        $$emit$$"dec     rcx\n\t"
11479        $$emit$$"jge     L_sloop\n\t"
11480        $$emit$$"# L_end:\n\t"
11481     } else {
11482        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11483     }
11484   %}
11485   ins_encode %{
11486     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11487                  $tmp$$XMMRegister, true, false);
11488   %}
11489   ins_pipe(pipe_slow);
11490 %}
11491 
11492 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11493                                   Universe dummy, rFlagsReg cr)
11494 %{
11495   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
11496   match(Set dummy (ClearArray (Binary cnt base) val));
11497   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
11498 
11499   format %{ $$template
11500     if (UseXMMForObjInit) {
11501        $$emit$$"movdq   $tmp, $val\n\t"
11502        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11503        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11504        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11505        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11506        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11507        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11508        $$emit$$"add     0x40,rax\n\t"
11509        $$emit$$"# L_zero_64_bytes:\n\t"
11510        $$emit$$"sub     0x8,rcx\n\t"
11511        $$emit$$"jge     L_loop\n\t"
11512        $$emit$$"add     0x4,rcx\n\t"
11513        $$emit$$"jl      L_tail\n\t"
11514        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11515        $$emit$$"add     0x20,rax\n\t"
11516        $$emit$$"sub     0x4,rcx\n\t"
11517        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11518        $$emit$$"add     0x4,rcx\n\t"
11519        $$emit$$"jle     L_end\n\t"
11520        $$emit$$"dec     rcx\n\t"
11521        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11522        $$emit$$"vmovq   xmm0,(rax)\n\t"
11523        $$emit$$"add     0x8,rax\n\t"
11524        $$emit$$"dec     rcx\n\t"
11525        $$emit$$"jge     L_sloop\n\t"
11526        $$emit$$"# L_end:\n\t"
11527     } else {
11528        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11529     }
11530   %}
11531   ins_encode %{
11532     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11533                  $tmp$$XMMRegister, true, true);
11534   %}
11535   ins_pipe(pipe_slow);
11536 %}
11537 
11538 // Large ClearArray AVX512.
11539 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
11540                              Universe dummy, rFlagsReg cr)
11541 %{
11542   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
11543   match(Set dummy (ClearArray (Binary cnt base) val));
11544   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
11545 
11546   format %{ $$template
11547     if (UseFastStosb) {
11548        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11549        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11550        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11551     } else if (UseXMMForObjInit) {
11552        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11553        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11554        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11555        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11556        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11557        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11558        $$emit$$"add     0x40,rax\n\t"
11559        $$emit$$"# L_zero_64_bytes:\n\t"
11560        $$emit$$"sub     0x8,rcx\n\t"
11561        $$emit$$"jge     L_loop\n\t"
11562        $$emit$$"add     0x4,rcx\n\t"
11563        $$emit$$"jl      L_tail\n\t"
11564        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11565        $$emit$$"add     0x20,rax\n\t"
11566        $$emit$$"sub     0x4,rcx\n\t"
11567        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11568        $$emit$$"add     0x4,rcx\n\t"
11569        $$emit$$"jle     L_end\n\t"
11570        $$emit$$"dec     rcx\n\t"
11571        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11572        $$emit$$"vmovq   xmm0,(rax)\n\t"
11573        $$emit$$"add     0x8,rax\n\t"
11574        $$emit$$"dec     rcx\n\t"
11575        $$emit$$"jge     L_sloop\n\t"
11576        $$emit$$"# L_end:\n\t"
11577     } else {
11578        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11579        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11580     }
11581   %}
11582   ins_encode %{
11583     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11584                  $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
11585   %}
11586   ins_pipe(pipe_slow);
11587 %}
11588 
11589 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
11590                                        Universe dummy, rFlagsReg cr)
11591 %{
11592   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
11593   match(Set dummy (ClearArray (Binary cnt base) val));
11594   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
11595 
11596   format %{ $$template
11597     if (UseFastStosb) {
11598        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11599        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11600        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11601     } else if (UseXMMForObjInit) {
11602        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11603        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11604        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11605        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11606        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11607        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11608        $$emit$$"add     0x40,rax\n\t"
11609        $$emit$$"# L_zero_64_bytes:\n\t"
11610        $$emit$$"sub     0x8,rcx\n\t"
11611        $$emit$$"jge     L_loop\n\t"
11612        $$emit$$"add     0x4,rcx\n\t"
11613        $$emit$$"jl      L_tail\n\t"
11614        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11615        $$emit$$"add     0x20,rax\n\t"
11616        $$emit$$"sub     0x4,rcx\n\t"
11617        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11618        $$emit$$"add     0x4,rcx\n\t"
11619        $$emit$$"jle     L_end\n\t"
11620        $$emit$$"dec     rcx\n\t"
11621        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11622        $$emit$$"vmovq   xmm0,(rax)\n\t"
11623        $$emit$$"add     0x8,rax\n\t"
11624        $$emit$$"dec     rcx\n\t"
11625        $$emit$$"jge     L_sloop\n\t"
11626        $$emit$$"# L_end:\n\t"
11627     } else {
11628        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11629        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11630     }
11631   %}
11632   ins_encode %{
11633     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11634                  $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
11635   %}
11636   ins_pipe(pipe_slow);
11637 %}
11638 
11639 // Small ClearArray AVX512 constant length.
11640 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
11641 %{
11642   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
11643             ((UseAVX > 2) && VM_Version::supports_avx512vlbw()));
11644   match(Set dummy (ClearArray (Binary cnt base) val));
11645   ins_cost(100);
11646   effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
11647   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
11648   ins_encode %{
11649     __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
11650   %}
11651   ins_pipe(pipe_slow);
11652 %}
11653 
11654 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11655                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11656 %{
11657   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11658   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11659   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11660 
11661   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11662   ins_encode %{
11663     __ string_compare($str1$$Register, $str2$$Register,
11664                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11665                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11666   %}
11667   ins_pipe( pipe_slow );
11668 %}
11669 
11670 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11671                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11672 %{
11673   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11674   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11675   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11676 
11677   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11678   ins_encode %{
11679     __ string_compare($str1$$Register, $str2$$Register,
11680                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11681                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11682   %}
11683   ins_pipe( pipe_slow );
11684 %}
11685 
11686 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11687                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11688 %{
11689   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11690   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11691   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11692 
11693   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11694   ins_encode %{
11695     __ string_compare($str1$$Register, $str2$$Register,
11696                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11697                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11698   %}
11699   ins_pipe( pipe_slow );
11700 %}
11701 
11702 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11703                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11704 %{
11705   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11706   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11707   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11708 
11709   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11710   ins_encode %{
11711     __ string_compare($str1$$Register, $str2$$Register,
11712                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11713                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11714   %}
11715   ins_pipe( pipe_slow );
11716 %}
11717 
11718 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11719                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11720 %{
11721   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11722   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11723   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11724 
11725   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11726   ins_encode %{
11727     __ string_compare($str1$$Register, $str2$$Register,
11728                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11729                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11730   %}
11731   ins_pipe( pipe_slow );
11732 %}
11733 
11734 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11735                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11736 %{
11737   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11738   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11739   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11740 
11741   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11742   ins_encode %{
11743     __ string_compare($str1$$Register, $str2$$Register,
11744                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11745                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11746   %}
11747   ins_pipe( pipe_slow );
11748 %}
11749 
11750 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11751                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11752 %{
11753   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11754   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11755   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11756 
11757   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11758   ins_encode %{
11759     __ string_compare($str2$$Register, $str1$$Register,
11760                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11761                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11762   %}
11763   ins_pipe( pipe_slow );
11764 %}
11765 
11766 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11767                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11768 %{
11769   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11770   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11771   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11772 
11773   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11774   ins_encode %{
11775     __ string_compare($str2$$Register, $str1$$Register,
11776                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11777                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11778   %}
11779   ins_pipe( pipe_slow );
11780 %}
11781 
11782 // fast search of substring with known size.
11783 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11784                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11785 %{
11786   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11787   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11788   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11789 
11790   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11791   ins_encode %{
11792     int icnt2 = (int)$int_cnt2$$constant;
11793     if (icnt2 >= 16) {
11794       // IndexOf for constant substrings with size >= 16 elements
11795       // which don't need to be loaded through stack.
11796       __ string_indexofC8($str1$$Register, $str2$$Register,
11797                           $cnt1$$Register, $cnt2$$Register,
11798                           icnt2, $result$$Register,
11799                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11800     } else {
11801       // Small strings are loaded through stack if they cross page boundary.
11802       __ string_indexof($str1$$Register, $str2$$Register,
11803                         $cnt1$$Register, $cnt2$$Register,
11804                         icnt2, $result$$Register,
11805                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11806     }
11807   %}
11808   ins_pipe( pipe_slow );
11809 %}
11810 
11811 // fast search of substring with known size.
11812 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11813                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11814 %{
11815   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11816   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11817   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11818 
11819   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11820   ins_encode %{
11821     int icnt2 = (int)$int_cnt2$$constant;
11822     if (icnt2 >= 8) {
11823       // IndexOf for constant substrings with size >= 8 elements
11824       // which don't need to be loaded through stack.
11825       __ string_indexofC8($str1$$Register, $str2$$Register,
11826                           $cnt1$$Register, $cnt2$$Register,
11827                           icnt2, $result$$Register,
11828                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11829     } else {
11830       // Small strings are loaded through stack if they cross page boundary.
11831       __ string_indexof($str1$$Register, $str2$$Register,
11832                         $cnt1$$Register, $cnt2$$Register,
11833                         icnt2, $result$$Register,
11834                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11835     }
11836   %}
11837   ins_pipe( pipe_slow );
11838 %}
11839 
11840 // fast search of substring with known size.
11841 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11842                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11843 %{
11844   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11845   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11846   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11847 
11848   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11849   ins_encode %{
11850     int icnt2 = (int)$int_cnt2$$constant;
11851     if (icnt2 >= 8) {
11852       // IndexOf for constant substrings with size >= 8 elements
11853       // which don't need to be loaded through stack.
11854       __ string_indexofC8($str1$$Register, $str2$$Register,
11855                           $cnt1$$Register, $cnt2$$Register,
11856                           icnt2, $result$$Register,
11857                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11858     } else {
11859       // Small strings are loaded through stack if they cross page boundary.
11860       __ string_indexof($str1$$Register, $str2$$Register,
11861                         $cnt1$$Register, $cnt2$$Register,
11862                         icnt2, $result$$Register,
11863                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11864     }
11865   %}
11866   ins_pipe( pipe_slow );
11867 %}
11868 
11869 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11870                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11871 %{
11872   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11873   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11874   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11875 
11876   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11877   ins_encode %{
11878     __ string_indexof($str1$$Register, $str2$$Register,
11879                       $cnt1$$Register, $cnt2$$Register,
11880                       (-1), $result$$Register,
11881                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11882   %}
11883   ins_pipe( pipe_slow );
11884 %}
11885 
11886 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11887                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11888 %{
11889   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11890   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11891   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11892 
11893   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11894   ins_encode %{
11895     __ string_indexof($str1$$Register, $str2$$Register,
11896                       $cnt1$$Register, $cnt2$$Register,
11897                       (-1), $result$$Register,
11898                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11899   %}
11900   ins_pipe( pipe_slow );
11901 %}
11902 
11903 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11904                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11905 %{
11906   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11907   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11908   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11909 
11910   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11911   ins_encode %{
11912     __ string_indexof($str1$$Register, $str2$$Register,
11913                       $cnt1$$Register, $cnt2$$Register,
11914                       (-1), $result$$Register,
11915                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11916   %}
11917   ins_pipe( pipe_slow );
11918 %}
11919 
11920 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11921                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11922 %{
11923   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11924   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11925   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11926   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11927   ins_encode %{
11928     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11929                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11930   %}
11931   ins_pipe( pipe_slow );
11932 %}
11933 
11934 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11935                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11936 %{
11937   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11938   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11939   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11940   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11941   ins_encode %{
11942     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11943                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11944   %}
11945   ins_pipe( pipe_slow );
11946 %}
11947 
11948 // fast string equals
11949 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11950                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11951 %{
11952   predicate(!VM_Version::supports_avx512vlbw());
11953   match(Set result (StrEquals (Binary str1 str2) cnt));
11954   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11955 
11956   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11957   ins_encode %{
11958     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11959                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11960                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11961   %}
11962   ins_pipe( pipe_slow );
11963 %}
11964 
11965 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11966                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11967 %{
11968   predicate(VM_Version::supports_avx512vlbw());
11969   match(Set result (StrEquals (Binary str1 str2) cnt));
11970   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11971 
11972   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11973   ins_encode %{
11974     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11975                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11976                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11977   %}
11978   ins_pipe( pipe_slow );
11979 %}
11980 
11981 // fast array equals
11982 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11983                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11984 %{
11985   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11986   match(Set result (AryEq ary1 ary2));
11987   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11988 
11989   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11990   ins_encode %{
11991     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11992                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11993                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11994   %}
11995   ins_pipe( pipe_slow );
11996 %}
11997 
11998 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11999                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12000 %{
12001   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12002   match(Set result (AryEq ary1 ary2));
12003   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12004 
12005   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12006   ins_encode %{
12007     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12008                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12009                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12010   %}
12011   ins_pipe( pipe_slow );
12012 %}
12013 
12014 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12015                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12016 %{
12017   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12018   match(Set result (AryEq ary1 ary2));
12019   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12020 
12021   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12022   ins_encode %{
12023     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12024                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12025                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12026   %}
12027   ins_pipe( pipe_slow );
12028 %}
12029 
12030 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12031                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12032 %{
12033   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12034   match(Set result (AryEq ary1 ary2));
12035   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12036 
12037   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12038   ins_encode %{
12039     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12040                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12041                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12042   %}
12043   ins_pipe( pipe_slow );
12044 %}
12045 
12046 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12047                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12048 %{
12049   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12050   match(Set result (CountPositives ary1 len));
12051   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12052 
12053   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12054   ins_encode %{
12055     __ count_positives($ary1$$Register, $len$$Register,
12056                        $result$$Register, $tmp3$$Register,
12057                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12058   %}
12059   ins_pipe( pipe_slow );
12060 %}
12061 
12062 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12063                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12064 %{
12065   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12066   match(Set result (CountPositives ary1 len));
12067   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12068 
12069   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12070   ins_encode %{
12071     __ count_positives($ary1$$Register, $len$$Register,
12072                        $result$$Register, $tmp3$$Register,
12073                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12074   %}
12075   ins_pipe( pipe_slow );
12076 %}
12077 
12078 // fast char[] to byte[] compression
12079 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12080                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12081   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12082   match(Set result (StrCompressedCopy src (Binary dst len)));
12083   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12084          USE_KILL len, KILL tmp5, KILL cr);
12085 
12086   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12087   ins_encode %{
12088     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12089                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12090                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12091                            knoreg, knoreg);
12092   %}
12093   ins_pipe( pipe_slow );
12094 %}
12095 
12096 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12097                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12098   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12099   match(Set result (StrCompressedCopy src (Binary dst len)));
12100   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12101          USE_KILL len, KILL tmp5, KILL cr);
12102 
12103   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12104   ins_encode %{
12105     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12106                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12107                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12108                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12109   %}
12110   ins_pipe( pipe_slow );
12111 %}
12112 // fast byte[] to char[] inflation
12113 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12114                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12115   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12116   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12117   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12118 
12119   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12120   ins_encode %{
12121     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12122                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12123   %}
12124   ins_pipe( pipe_slow );
12125 %}
12126 
12127 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12128                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12129   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12130   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12131   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12132 
12133   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12134   ins_encode %{
12135     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12136                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12137   %}
12138   ins_pipe( pipe_slow );
12139 %}
12140 
12141 // encode char[] to byte[] in ISO_8859_1
12142 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12143                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12144                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12145   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12146   match(Set result (EncodeISOArray src (Binary dst len)));
12147   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12148 
12149   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12150   ins_encode %{
12151     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12152                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12153                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12154   %}
12155   ins_pipe( pipe_slow );
12156 %}
12157 
12158 // encode char[] to byte[] in ASCII
12159 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12160                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12161                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12162   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12163   match(Set result (EncodeISOArray src (Binary dst len)));
12164   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12165 
12166   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12167   ins_encode %{
12168     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12169                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12170                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12171   %}
12172   ins_pipe( pipe_slow );
12173 %}
12174 
12175 //----------Overflow Math Instructions-----------------------------------------
12176 
12177 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12178 %{
12179   match(Set cr (OverflowAddI op1 op2));
12180   effect(DEF cr, USE_KILL op1, USE op2);
12181 
12182   format %{ "addl    $op1, $op2\t# overflow check int" %}
12183 
12184   ins_encode %{
12185     __ addl($op1$$Register, $op2$$Register);
12186   %}
12187   ins_pipe(ialu_reg_reg);
12188 %}
12189 
12190 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12191 %{
12192   match(Set cr (OverflowAddI op1 op2));
12193   effect(DEF cr, USE_KILL op1, USE op2);
12194 
12195   format %{ "addl    $op1, $op2\t# overflow check int" %}
12196 
12197   ins_encode %{
12198     __ addl($op1$$Register, $op2$$constant);
12199   %}
12200   ins_pipe(ialu_reg_reg);
12201 %}
12202 
12203 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12204 %{
12205   match(Set cr (OverflowAddL op1 op2));
12206   effect(DEF cr, USE_KILL op1, USE op2);
12207 
12208   format %{ "addq    $op1, $op2\t# overflow check long" %}
12209   ins_encode %{
12210     __ addq($op1$$Register, $op2$$Register);
12211   %}
12212   ins_pipe(ialu_reg_reg);
12213 %}
12214 
12215 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12216 %{
12217   match(Set cr (OverflowAddL op1 op2));
12218   effect(DEF cr, USE_KILL op1, USE op2);
12219 
12220   format %{ "addq    $op1, $op2\t# overflow check long" %}
12221   ins_encode %{
12222     __ addq($op1$$Register, $op2$$constant);
12223   %}
12224   ins_pipe(ialu_reg_reg);
12225 %}
12226 
12227 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12228 %{
12229   match(Set cr (OverflowSubI op1 op2));
12230 
12231   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12232   ins_encode %{
12233     __ cmpl($op1$$Register, $op2$$Register);
12234   %}
12235   ins_pipe(ialu_reg_reg);
12236 %}
12237 
12238 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12239 %{
12240   match(Set cr (OverflowSubI op1 op2));
12241 
12242   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12243   ins_encode %{
12244     __ cmpl($op1$$Register, $op2$$constant);
12245   %}
12246   ins_pipe(ialu_reg_reg);
12247 %}
12248 
12249 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12250 %{
12251   match(Set cr (OverflowSubL op1 op2));
12252 
12253   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12254   ins_encode %{
12255     __ cmpq($op1$$Register, $op2$$Register);
12256   %}
12257   ins_pipe(ialu_reg_reg);
12258 %}
12259 
12260 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12261 %{
12262   match(Set cr (OverflowSubL op1 op2));
12263 
12264   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12265   ins_encode %{
12266     __ cmpq($op1$$Register, $op2$$constant);
12267   %}
12268   ins_pipe(ialu_reg_reg);
12269 %}
12270 
12271 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
12272 %{
12273   match(Set cr (OverflowSubI zero op2));
12274   effect(DEF cr, USE_KILL op2);
12275 
12276   format %{ "negl    $op2\t# overflow check int" %}
12277   ins_encode %{
12278     __ negl($op2$$Register);
12279   %}
12280   ins_pipe(ialu_reg_reg);
12281 %}
12282 
12283 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
12284 %{
12285   match(Set cr (OverflowSubL zero op2));
12286   effect(DEF cr, USE_KILL op2);
12287 
12288   format %{ "negq    $op2\t# overflow check long" %}
12289   ins_encode %{
12290     __ negq($op2$$Register);
12291   %}
12292   ins_pipe(ialu_reg_reg);
12293 %}
12294 
12295 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12296 %{
12297   match(Set cr (OverflowMulI op1 op2));
12298   effect(DEF cr, USE_KILL op1, USE op2);
12299 
12300   format %{ "imull    $op1, $op2\t# overflow check int" %}
12301   ins_encode %{
12302     __ imull($op1$$Register, $op2$$Register);
12303   %}
12304   ins_pipe(ialu_reg_reg_alu0);
12305 %}
12306 
12307 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
12308 %{
12309   match(Set cr (OverflowMulI op1 op2));
12310   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12311 
12312   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
12313   ins_encode %{
12314     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
12315   %}
12316   ins_pipe(ialu_reg_reg_alu0);
12317 %}
12318 
12319 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12320 %{
12321   match(Set cr (OverflowMulL op1 op2));
12322   effect(DEF cr, USE_KILL op1, USE op2);
12323 
12324   format %{ "imulq    $op1, $op2\t# overflow check long" %}
12325   ins_encode %{
12326     __ imulq($op1$$Register, $op2$$Register);
12327   %}
12328   ins_pipe(ialu_reg_reg_alu0);
12329 %}
12330 
12331 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
12332 %{
12333   match(Set cr (OverflowMulL op1 op2));
12334   effect(DEF cr, TEMP tmp, USE op1, USE op2);
12335 
12336   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
12337   ins_encode %{
12338     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
12339   %}
12340   ins_pipe(ialu_reg_reg_alu0);
12341 %}
12342 
12343 
12344 //----------Control Flow Instructions------------------------------------------
12345 // Signed compare Instructions
12346 
12347 // XXX more variants!!
12348 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12349 %{
12350   match(Set cr (CmpI op1 op2));
12351   effect(DEF cr, USE op1, USE op2);
12352 
12353   format %{ "cmpl    $op1, $op2" %}
12354   ins_encode %{
12355     __ cmpl($op1$$Register, $op2$$Register);
12356   %}
12357   ins_pipe(ialu_cr_reg_reg);
12358 %}
12359 
12360 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12361 %{
12362   match(Set cr (CmpI op1 op2));
12363 
12364   format %{ "cmpl    $op1, $op2" %}
12365   ins_encode %{
12366     __ cmpl($op1$$Register, $op2$$constant);
12367   %}
12368   ins_pipe(ialu_cr_reg_imm);
12369 %}
12370 
12371 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12372 %{
12373   match(Set cr (CmpI op1 (LoadI op2)));
12374 
12375   ins_cost(500); // XXX
12376   format %{ "cmpl    $op1, $op2" %}
12377   ins_encode %{
12378     __ cmpl($op1$$Register, $op2$$Address);
12379   %}
12380   ins_pipe(ialu_cr_reg_mem);
12381 %}
12382 
12383 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
12384 %{
12385   match(Set cr (CmpI src zero));
12386 
12387   format %{ "testl   $src, $src" %}
12388   ins_encode %{
12389     __ testl($src$$Register, $src$$Register);
12390   %}
12391   ins_pipe(ialu_cr_reg_imm);
12392 %}
12393 
12394 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
12395 %{
12396   match(Set cr (CmpI (AndI src con) zero));
12397 
12398   format %{ "testl   $src, $con" %}
12399   ins_encode %{
12400     __ testl($src$$Register, $con$$constant);
12401   %}
12402   ins_pipe(ialu_cr_reg_imm);
12403 %}
12404 
12405 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
12406 %{
12407   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12408 
12409   format %{ "testl   $src, $mem" %}
12410   ins_encode %{
12411     __ testl($src$$Register, $mem$$Address);
12412   %}
12413   ins_pipe(ialu_cr_reg_mem);
12414 %}
12415 
12416 // Unsigned compare Instructions; really, same as signed except they
12417 // produce an rFlagsRegU instead of rFlagsReg.
12418 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12419 %{
12420   match(Set cr (CmpU op1 op2));
12421 
12422   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12423   ins_encode %{
12424     __ cmpl($op1$$Register, $op2$$Register);
12425   %}
12426   ins_pipe(ialu_cr_reg_reg);
12427 %}
12428 
12429 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12430 %{
12431   match(Set cr (CmpU op1 op2));
12432 
12433   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12434   ins_encode %{
12435     __ cmpl($op1$$Register, $op2$$constant);
12436   %}
12437   ins_pipe(ialu_cr_reg_imm);
12438 %}
12439 
12440 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12441 %{
12442   match(Set cr (CmpU op1 (LoadI op2)));
12443 
12444   ins_cost(500); // XXX
12445   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12446   ins_encode %{
12447     __ cmpl($op1$$Register, $op2$$Address);
12448   %}
12449   ins_pipe(ialu_cr_reg_mem);
12450 %}
12451 
12452 // // // Cisc-spilled version of cmpU_rReg
12453 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12454 // //%{
12455 // //  match(Set cr (CmpU (LoadI op1) op2));
12456 // //
12457 // //  format %{ "CMPu   $op1,$op2" %}
12458 // //  ins_cost(500);
12459 // //  opcode(0x39);  /* Opcode 39 /r */
12460 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12461 // //%}
12462 
12463 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
12464 %{
12465   match(Set cr (CmpU src zero));
12466 
12467   format %{ "testl   $src, $src\t# unsigned" %}
12468   ins_encode %{
12469     __ testl($src$$Register, $src$$Register);
12470   %}
12471   ins_pipe(ialu_cr_reg_imm);
12472 %}
12473 
12474 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12475 %{
12476   match(Set cr (CmpP op1 op2));
12477 
12478   format %{ "cmpq    $op1, $op2\t# ptr" %}
12479   ins_encode %{
12480     __ cmpq($op1$$Register, $op2$$Register);
12481   %}
12482   ins_pipe(ialu_cr_reg_reg);
12483 %}
12484 
12485 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12486 %{
12487   match(Set cr (CmpP op1 (LoadP op2)));
12488   predicate(n->in(2)->as_Load()->barrier_data() == 0);
12489 
12490   ins_cost(500); // XXX
12491   format %{ "cmpq    $op1, $op2\t# ptr" %}
12492   ins_encode %{
12493     __ cmpq($op1$$Register, $op2$$Address);
12494   %}
12495   ins_pipe(ialu_cr_reg_mem);
12496 %}
12497 
12498 // // // Cisc-spilled version of cmpP_rReg
12499 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12500 // //%{
12501 // //  match(Set cr (CmpP (LoadP op1) op2));
12502 // //
12503 // //  format %{ "CMPu   $op1,$op2" %}
12504 // //  ins_cost(500);
12505 // //  opcode(0x39);  /* Opcode 39 /r */
12506 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12507 // //%}
12508 
12509 // XXX this is generalized by compP_rReg_mem???
12510 // Compare raw pointer (used in out-of-heap check).
12511 // Only works because non-oop pointers must be raw pointers
12512 // and raw pointers have no anti-dependencies.
12513 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12514 %{
12515   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
12516             n->in(2)->as_Load()->barrier_data() == 0);
12517   match(Set cr (CmpP op1 (LoadP op2)));
12518 
12519   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12520   ins_encode %{
12521     __ cmpq($op1$$Register, $op2$$Address);
12522   %}
12523   ins_pipe(ialu_cr_reg_mem);
12524 %}
12525 
12526 // This will generate a signed flags result. This should be OK since
12527 // any compare to a zero should be eq/neq.
12528 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12529 %{
12530   match(Set cr (CmpP src zero));
12531 
12532   format %{ "testq   $src, $src\t# ptr" %}
12533   ins_encode %{
12534     __ testq($src$$Register, $src$$Register);
12535   %}
12536   ins_pipe(ialu_cr_reg_imm);
12537 %}
12538 
12539 // This will generate a signed flags result. This should be OK since
12540 // any compare to a zero should be eq/neq.
12541 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12542 %{
12543   predicate((!UseCompressedOops || (CompressedOops::base() != NULL)) &&
12544             n->in(1)->as_Load()->barrier_data() == 0);
12545   match(Set cr (CmpP (LoadP op) zero));
12546 
12547   ins_cost(500); // XXX
12548   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12549   ins_encode %{
12550     __ testq($op$$Address, 0xFFFFFFFF);
12551   %}
12552   ins_pipe(ialu_cr_reg_imm);
12553 %}
12554 
12555 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12556 %{
12557   predicate(UseCompressedOops && (CompressedOops::base() == NULL) &&
12558             n->in(1)->as_Load()->barrier_data() == 0);
12559   match(Set cr (CmpP (LoadP mem) zero));
12560 
12561   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12562   ins_encode %{
12563     __ cmpq(r12, $mem$$Address);
12564   %}
12565   ins_pipe(ialu_cr_reg_mem);
12566 %}
12567 
12568 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12569 %{
12570   match(Set cr (CmpN op1 op2));
12571 
12572   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12573   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12574   ins_pipe(ialu_cr_reg_reg);
12575 %}
12576 
12577 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12578 %{
12579   match(Set cr (CmpN src (LoadN mem)));
12580 
12581   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12582   ins_encode %{
12583     __ cmpl($src$$Register, $mem$$Address);
12584   %}
12585   ins_pipe(ialu_cr_reg_mem);
12586 %}
12587 
12588 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12589   match(Set cr (CmpN op1 op2));
12590 
12591   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12592   ins_encode %{
12593     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12594   %}
12595   ins_pipe(ialu_cr_reg_imm);
12596 %}
12597 
12598 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12599 %{
12600   match(Set cr (CmpN src (LoadN mem)));
12601 
12602   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12603   ins_encode %{
12604     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12605   %}
12606   ins_pipe(ialu_cr_reg_mem);
12607 %}
12608 
12609 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12610   match(Set cr (CmpN op1 op2));
12611 
12612   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12613   ins_encode %{
12614     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12615   %}
12616   ins_pipe(ialu_cr_reg_imm);
12617 %}
12618 
12619 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12620 %{
12621   match(Set cr (CmpN src (LoadNKlass mem)));
12622 
12623   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12624   ins_encode %{
12625     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12626   %}
12627   ins_pipe(ialu_cr_reg_mem);
12628 %}
12629 
12630 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12631   match(Set cr (CmpN src zero));
12632 
12633   format %{ "testl   $src, $src\t# compressed ptr" %}
12634   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12635   ins_pipe(ialu_cr_reg_imm);
12636 %}
12637 
12638 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12639 %{
12640   predicate(CompressedOops::base() != NULL);
12641   match(Set cr (CmpN (LoadN mem) zero));
12642 
12643   ins_cost(500); // XXX
12644   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12645   ins_encode %{
12646     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12647   %}
12648   ins_pipe(ialu_cr_reg_mem);
12649 %}
12650 
12651 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12652 %{
12653   predicate(CompressedOops::base() == NULL);
12654   match(Set cr (CmpN (LoadN mem) zero));
12655 
12656   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12657   ins_encode %{
12658     __ cmpl(r12, $mem$$Address);
12659   %}
12660   ins_pipe(ialu_cr_reg_mem);
12661 %}
12662 
12663 // Yanked all unsigned pointer compare operations.
12664 // Pointer compares are done with CmpP which is already unsigned.
12665 
12666 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12667 %{
12668   match(Set cr (CmpL op1 op2));
12669 
12670   format %{ "cmpq    $op1, $op2" %}
12671   ins_encode %{
12672     __ cmpq($op1$$Register, $op2$$Register);
12673   %}
12674   ins_pipe(ialu_cr_reg_reg);
12675 %}
12676 
12677 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12678 %{
12679   match(Set cr (CmpL op1 op2));
12680 
12681   format %{ "cmpq    $op1, $op2" %}
12682   ins_encode %{
12683     __ cmpq($op1$$Register, $op2$$constant);
12684   %}
12685   ins_pipe(ialu_cr_reg_imm);
12686 %}
12687 
12688 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12689 %{
12690   match(Set cr (CmpL op1 (LoadL op2)));
12691 
12692   format %{ "cmpq    $op1, $op2" %}
12693   ins_encode %{
12694     __ cmpq($op1$$Register, $op2$$Address);
12695   %}
12696   ins_pipe(ialu_cr_reg_mem);
12697 %}
12698 
12699 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12700 %{
12701   match(Set cr (CmpL src zero));
12702 
12703   format %{ "testq   $src, $src" %}
12704   ins_encode %{
12705     __ testq($src$$Register, $src$$Register);
12706   %}
12707   ins_pipe(ialu_cr_reg_imm);
12708 %}
12709 
12710 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12711 %{
12712   match(Set cr (CmpL (AndL src con) zero));
12713 
12714   format %{ "testq   $src, $con\t# long" %}
12715   ins_encode %{
12716     __ testq($src$$Register, $con$$constant);
12717   %}
12718   ins_pipe(ialu_cr_reg_imm);
12719 %}
12720 
12721 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12722 %{
12723   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12724 
12725   format %{ "testq   $src, $mem" %}
12726   ins_encode %{
12727     __ testq($src$$Register, $mem$$Address);
12728   %}
12729   ins_pipe(ialu_cr_reg_mem);
12730 %}
12731 
12732 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12733 %{
12734   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12735 
12736   format %{ "testq   $src, $mem" %}
12737   ins_encode %{
12738     __ testq($src$$Register, $mem$$Address);
12739   %}
12740   ins_pipe(ialu_cr_reg_mem);
12741 %}
12742 
12743 // Manifest a CmpL result in an integer register.  Very painful.
12744 // This is the test to avoid.
12745 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12746 %{
12747   match(Set dst (CmpL3 src1 src2));
12748   effect(KILL flags);
12749 
12750   ins_cost(275); // XXX
12751   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12752             "movl    $dst, -1\n\t"
12753             "jl,s    done\n\t"
12754             "setne   $dst\n\t"
12755             "movzbl  $dst, $dst\n\t"
12756     "done:" %}
12757   ins_encode %{
12758     Label done;
12759     __ cmpq($src1$$Register, $src2$$Register);
12760     __ movl($dst$$Register, -1);
12761     __ jccb(Assembler::less, done);
12762     __ setne($dst$$Register);
12763     __ movzbl($dst$$Register, $dst$$Register);
12764     __ bind(done);
12765   %}
12766   ins_pipe(pipe_slow);
12767 %}
12768 
12769 // Unsigned long compare Instructions; really, same as signed long except they
12770 // produce an rFlagsRegU instead of rFlagsReg.
12771 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12772 %{
12773   match(Set cr (CmpUL op1 op2));
12774 
12775   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12776   ins_encode %{
12777     __ cmpq($op1$$Register, $op2$$Register);
12778   %}
12779   ins_pipe(ialu_cr_reg_reg);
12780 %}
12781 
12782 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12783 %{
12784   match(Set cr (CmpUL op1 op2));
12785 
12786   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12787   ins_encode %{
12788     __ cmpq($op1$$Register, $op2$$constant);
12789   %}
12790   ins_pipe(ialu_cr_reg_imm);
12791 %}
12792 
12793 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12794 %{
12795   match(Set cr (CmpUL op1 (LoadL op2)));
12796 
12797   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12798   ins_encode %{
12799     __ cmpq($op1$$Register, $op2$$Address);
12800   %}
12801   ins_pipe(ialu_cr_reg_mem);
12802 %}
12803 
12804 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12805 %{
12806   match(Set cr (CmpUL src zero));
12807 
12808   format %{ "testq   $src, $src\t# unsigned" %}
12809   ins_encode %{
12810     __ testq($src$$Register, $src$$Register);
12811   %}
12812   ins_pipe(ialu_cr_reg_imm);
12813 %}
12814 
12815 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12816 %{
12817   match(Set cr (CmpI (LoadB mem) imm));
12818 
12819   ins_cost(125);
12820   format %{ "cmpb    $mem, $imm" %}
12821   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12822   ins_pipe(ialu_cr_reg_mem);
12823 %}
12824 
12825 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12826 %{
12827   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12828 
12829   ins_cost(125);
12830   format %{ "testb   $mem, $imm\t# ubyte" %}
12831   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12832   ins_pipe(ialu_cr_reg_mem);
12833 %}
12834 
12835 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12836 %{
12837   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12838 
12839   ins_cost(125);
12840   format %{ "testb   $mem, $imm\t# byte" %}
12841   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12842   ins_pipe(ialu_cr_reg_mem);
12843 %}
12844 
12845 //----------Max and Min--------------------------------------------------------
12846 // Min Instructions
12847 
12848 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12849 %{
12850   effect(USE_DEF dst, USE src, USE cr);
12851 
12852   format %{ "cmovlgt $dst, $src\t# min" %}
12853   ins_encode %{
12854     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12855   %}
12856   ins_pipe(pipe_cmov_reg);
12857 %}
12858 
12859 
12860 instruct minI_rReg(rRegI dst, rRegI src)
12861 %{
12862   match(Set dst (MinI dst src));
12863 
12864   ins_cost(200);
12865   expand %{
12866     rFlagsReg cr;
12867     compI_rReg(cr, dst, src);
12868     cmovI_reg_g(dst, src, cr);
12869   %}
12870 %}
12871 
12872 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12873 %{
12874   effect(USE_DEF dst, USE src, USE cr);
12875 
12876   format %{ "cmovllt $dst, $src\t# max" %}
12877   ins_encode %{
12878     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12879   %}
12880   ins_pipe(pipe_cmov_reg);
12881 %}
12882 
12883 
12884 instruct maxI_rReg(rRegI dst, rRegI src)
12885 %{
12886   match(Set dst (MaxI dst src));
12887 
12888   ins_cost(200);
12889   expand %{
12890     rFlagsReg cr;
12891     compI_rReg(cr, dst, src);
12892     cmovI_reg_l(dst, src, cr);
12893   %}
12894 %}
12895 
12896 // ============================================================================
12897 // Branch Instructions
12898 
12899 // Jump Direct - Label defines a relative address from JMP+1
12900 instruct jmpDir(label labl)
12901 %{
12902   match(Goto);
12903   effect(USE labl);
12904 
12905   ins_cost(300);
12906   format %{ "jmp     $labl" %}
12907   size(5);
12908   ins_encode %{
12909     Label* L = $labl$$label;
12910     __ jmp(*L, false); // Always long jump
12911   %}
12912   ins_pipe(pipe_jmp);
12913 %}
12914 
12915 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12916 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12917 %{
12918   match(If cop cr);
12919   effect(USE labl);
12920 
12921   ins_cost(300);
12922   format %{ "j$cop     $labl" %}
12923   size(6);
12924   ins_encode %{
12925     Label* L = $labl$$label;
12926     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12927   %}
12928   ins_pipe(pipe_jcc);
12929 %}
12930 
12931 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12932 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12933 %{
12934   predicate(!n->has_vector_mask_set());
12935   match(CountedLoopEnd cop cr);
12936   effect(USE labl);
12937 
12938   ins_cost(300);
12939   format %{ "j$cop     $labl\t# loop end" %}
12940   size(6);
12941   ins_encode %{
12942     Label* L = $labl$$label;
12943     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12944   %}
12945   ins_pipe(pipe_jcc);
12946 %}
12947 
12948 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12949 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12950   predicate(!n->has_vector_mask_set());
12951   match(CountedLoopEnd cop cmp);
12952   effect(USE labl);
12953 
12954   ins_cost(300);
12955   format %{ "j$cop,u   $labl\t# loop end" %}
12956   size(6);
12957   ins_encode %{
12958     Label* L = $labl$$label;
12959     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12960   %}
12961   ins_pipe(pipe_jcc);
12962 %}
12963 
12964 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12965   predicate(!n->has_vector_mask_set());
12966   match(CountedLoopEnd cop cmp);
12967   effect(USE labl);
12968 
12969   ins_cost(200);
12970   format %{ "j$cop,u   $labl\t# loop end" %}
12971   size(6);
12972   ins_encode %{
12973     Label* L = $labl$$label;
12974     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12975   %}
12976   ins_pipe(pipe_jcc);
12977 %}
12978 
12979 // mask version
12980 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12981 // Bounded mask operand used in following patten is needed for
12982 // post-loop multiversioning.
12983 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, kReg_K1 ktmp, rFlagsReg cr, label labl)
12984 %{
12985   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
12986   match(CountedLoopEnd cop cr);
12987   effect(USE labl, TEMP ktmp);
12988 
12989   ins_cost(400);
12990   format %{ "j$cop     $labl\t# loop end\n\t"
12991             "restorevectmask \t# vector mask restore for loops" %}
12992   size(10);
12993   ins_encode %{
12994     Label* L = $labl$$label;
12995     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12996     __ restorevectmask($ktmp$$KRegister);
12997   %}
12998   ins_pipe(pipe_jcc);
12999 %}
13000 
13001 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13002 // Bounded mask operand used in following patten is needed for
13003 // post-loop multiversioning.
13004 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, kReg_K1 ktmp, rFlagsRegU cmp, label labl) %{
13005   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
13006   match(CountedLoopEnd cop cmp);
13007   effect(USE labl, TEMP ktmp);
13008 
13009   ins_cost(400);
13010   format %{ "j$cop,u   $labl\t# loop end\n\t"
13011             "restorevectmask \t# vector mask restore for loops" %}
13012   size(10);
13013   ins_encode %{
13014     Label* L = $labl$$label;
13015     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13016     __ restorevectmask($ktmp$$KRegister);
13017   %}
13018   ins_pipe(pipe_jcc);
13019 %}
13020 
13021 // Bounded mask operand used in following patten is needed for
13022 // post-loop multiversioning.
13023 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, kReg_K1 ktmp, rFlagsRegUCF cmp, label labl) %{
13024   predicate(PostLoopMultiversioning && n->has_vector_mask_set());
13025   match(CountedLoopEnd cop cmp);
13026   effect(USE labl, TEMP ktmp);
13027 
13028   ins_cost(300);
13029   format %{ "j$cop,u   $labl\t# loop end\n\t"
13030             "restorevectmask \t# vector mask restore for loops" %}
13031   size(10);
13032   ins_encode %{
13033     Label* L = $labl$$label;
13034     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13035     __ restorevectmask($ktmp$$KRegister);
13036   %}
13037   ins_pipe(pipe_jcc);
13038 %}
13039 
13040 // Jump Direct Conditional - using unsigned comparison
13041 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13042   match(If cop cmp);
13043   effect(USE labl);
13044 
13045   ins_cost(300);
13046   format %{ "j$cop,u   $labl" %}
13047   size(6);
13048   ins_encode %{
13049     Label* L = $labl$$label;
13050     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13051   %}
13052   ins_pipe(pipe_jcc);
13053 %}
13054 
13055 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13056   match(If cop cmp);
13057   effect(USE labl);
13058 
13059   ins_cost(200);
13060   format %{ "j$cop,u   $labl" %}
13061   size(6);
13062   ins_encode %{
13063     Label* L = $labl$$label;
13064     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13065   %}
13066   ins_pipe(pipe_jcc);
13067 %}
13068 
13069 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13070   match(If cop cmp);
13071   effect(USE labl);
13072 
13073   ins_cost(200);
13074   format %{ $$template
13075     if ($cop$$cmpcode == Assembler::notEqual) {
13076       $$emit$$"jp,u    $labl\n\t"
13077       $$emit$$"j$cop,u   $labl"
13078     } else {
13079       $$emit$$"jp,u    done\n\t"
13080       $$emit$$"j$cop,u   $labl\n\t"
13081       $$emit$$"done:"
13082     }
13083   %}
13084   ins_encode %{
13085     Label* l = $labl$$label;
13086     if ($cop$$cmpcode == Assembler::notEqual) {
13087       __ jcc(Assembler::parity, *l, false);
13088       __ jcc(Assembler::notEqual, *l, false);
13089     } else if ($cop$$cmpcode == Assembler::equal) {
13090       Label done;
13091       __ jccb(Assembler::parity, done);
13092       __ jcc(Assembler::equal, *l, false);
13093       __ bind(done);
13094     } else {
13095        ShouldNotReachHere();
13096     }
13097   %}
13098   ins_pipe(pipe_jcc);
13099 %}
13100 
13101 // ============================================================================
13102 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13103 // superklass array for an instance of the superklass.  Set a hidden
13104 // internal cache on a hit (cache is checked with exposed code in
13105 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13106 // encoding ALSO sets flags.
13107 
13108 instruct partialSubtypeCheck(rdi_RegP result,
13109                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13110                              rFlagsReg cr)
13111 %{
13112   match(Set result (PartialSubtypeCheck sub super));
13113   effect(KILL rcx, KILL cr);
13114 
13115   ins_cost(1100);  // slightly larger than the next version
13116   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13117             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13118             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13119             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13120             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13121             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13122             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13123     "miss:\t" %}
13124 
13125   opcode(0x1); // Force a XOR of RDI
13126   ins_encode(enc_PartialSubtypeCheck());
13127   ins_pipe(pipe_slow);
13128 %}
13129 
13130 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
13131                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13132                                      immP0 zero,
13133                                      rdi_RegP result)
13134 %{
13135   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13136   effect(KILL rcx, KILL result);
13137 
13138   ins_cost(1000);
13139   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13140             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13141             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13142             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
13143             "jne,s   miss\t\t# Missed: flags nz\n\t"
13144             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13145     "miss:\t" %}
13146 
13147   opcode(0x0); // No need to XOR RDI
13148   ins_encode(enc_PartialSubtypeCheck());
13149   ins_pipe(pipe_slow);
13150 %}
13151 
13152 // ============================================================================
13153 // Branch Instructions -- short offset versions
13154 //
13155 // These instructions are used to replace jumps of a long offset (the default
13156 // match) with jumps of a shorter offset.  These instructions are all tagged
13157 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13158 // match rules in general matching.  Instead, the ADLC generates a conversion
13159 // method in the MachNode which can be used to do in-place replacement of the
13160 // long variant with the shorter variant.  The compiler will determine if a
13161 // branch can be taken by the is_short_branch_offset() predicate in the machine
13162 // specific code section of the file.
13163 
13164 // Jump Direct - Label defines a relative address from JMP+1
13165 instruct jmpDir_short(label labl) %{
13166   match(Goto);
13167   effect(USE labl);
13168 
13169   ins_cost(300);
13170   format %{ "jmp,s   $labl" %}
13171   size(2);
13172   ins_encode %{
13173     Label* L = $labl$$label;
13174     __ jmpb(*L);
13175   %}
13176   ins_pipe(pipe_jmp);
13177   ins_short_branch(1);
13178 %}
13179 
13180 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13181 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13182   match(If cop cr);
13183   effect(USE labl);
13184 
13185   ins_cost(300);
13186   format %{ "j$cop,s   $labl" %}
13187   size(2);
13188   ins_encode %{
13189     Label* L = $labl$$label;
13190     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13191   %}
13192   ins_pipe(pipe_jcc);
13193   ins_short_branch(1);
13194 %}
13195 
13196 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13197 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13198   match(CountedLoopEnd cop cr);
13199   effect(USE labl);
13200 
13201   ins_cost(300);
13202   format %{ "j$cop,s   $labl\t# loop end" %}
13203   size(2);
13204   ins_encode %{
13205     Label* L = $labl$$label;
13206     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13207   %}
13208   ins_pipe(pipe_jcc);
13209   ins_short_branch(1);
13210 %}
13211 
13212 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13213 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13214   match(CountedLoopEnd cop cmp);
13215   effect(USE labl);
13216 
13217   ins_cost(300);
13218   format %{ "j$cop,us  $labl\t# loop end" %}
13219   size(2);
13220   ins_encode %{
13221     Label* L = $labl$$label;
13222     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13223   %}
13224   ins_pipe(pipe_jcc);
13225   ins_short_branch(1);
13226 %}
13227 
13228 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13229   match(CountedLoopEnd cop cmp);
13230   effect(USE labl);
13231 
13232   ins_cost(300);
13233   format %{ "j$cop,us  $labl\t# loop end" %}
13234   size(2);
13235   ins_encode %{
13236     Label* L = $labl$$label;
13237     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13238   %}
13239   ins_pipe(pipe_jcc);
13240   ins_short_branch(1);
13241 %}
13242 
13243 // Jump Direct Conditional - using unsigned comparison
13244 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13245   match(If cop cmp);
13246   effect(USE labl);
13247 
13248   ins_cost(300);
13249   format %{ "j$cop,us  $labl" %}
13250   size(2);
13251   ins_encode %{
13252     Label* L = $labl$$label;
13253     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13254   %}
13255   ins_pipe(pipe_jcc);
13256   ins_short_branch(1);
13257 %}
13258 
13259 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13260   match(If cop cmp);
13261   effect(USE labl);
13262 
13263   ins_cost(300);
13264   format %{ "j$cop,us  $labl" %}
13265   size(2);
13266   ins_encode %{
13267     Label* L = $labl$$label;
13268     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13269   %}
13270   ins_pipe(pipe_jcc);
13271   ins_short_branch(1);
13272 %}
13273 
13274 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13275   match(If cop cmp);
13276   effect(USE labl);
13277 
13278   ins_cost(300);
13279   format %{ $$template
13280     if ($cop$$cmpcode == Assembler::notEqual) {
13281       $$emit$$"jp,u,s  $labl\n\t"
13282       $$emit$$"j$cop,u,s  $labl"
13283     } else {
13284       $$emit$$"jp,u,s  done\n\t"
13285       $$emit$$"j$cop,u,s  $labl\n\t"
13286       $$emit$$"done:"
13287     }
13288   %}
13289   size(4);
13290   ins_encode %{
13291     Label* l = $labl$$label;
13292     if ($cop$$cmpcode == Assembler::notEqual) {
13293       __ jccb(Assembler::parity, *l);
13294       __ jccb(Assembler::notEqual, *l);
13295     } else if ($cop$$cmpcode == Assembler::equal) {
13296       Label done;
13297       __ jccb(Assembler::parity, done);
13298       __ jccb(Assembler::equal, *l);
13299       __ bind(done);
13300     } else {
13301        ShouldNotReachHere();
13302     }
13303   %}
13304   ins_pipe(pipe_jcc);
13305   ins_short_branch(1);
13306 %}
13307 
13308 // ============================================================================
13309 // inlined locking and unlocking
13310 
13311 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
13312   predicate(Compile::current()->use_rtm());
13313   match(Set cr (FastLock object box));
13314   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
13315   ins_cost(300);
13316   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
13317   ins_encode %{
13318     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13319                  $scr$$Register, $cx1$$Register, $cx2$$Register,
13320                  _rtm_counters, _stack_rtm_counters,
13321                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
13322                  true, ra_->C->profile_rtm());
13323   %}
13324   ins_pipe(pipe_slow);
13325 %}
13326 
13327 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr, rRegP cx1) %{
13328   predicate(!Compile::current()->use_rtm());
13329   match(Set cr (FastLock object box));
13330   effect(TEMP tmp, TEMP scr, TEMP cx1, USE_KILL box);
13331   ins_cost(300);
13332   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
13333   ins_encode %{
13334     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
13335                  $scr$$Register, $cx1$$Register, noreg, NULL, NULL, NULL, false, false);
13336   %}
13337   ins_pipe(pipe_slow);
13338 %}
13339 
13340 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
13341   match(Set cr (FastUnlock object box));
13342   effect(TEMP tmp, USE_KILL box);
13343   ins_cost(300);
13344   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
13345   ins_encode %{
13346     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
13347   %}
13348   ins_pipe(pipe_slow);
13349 %}
13350 
13351 
13352 // ============================================================================
13353 // Safepoint Instructions
13354 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13355 %{
13356   match(SafePoint poll);
13357   effect(KILL cr, USE poll);
13358 
13359   format %{ "testl   rax, [$poll]\t"
13360             "# Safepoint: poll for GC" %}
13361   ins_cost(125);
13362   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13363   ins_encode %{
13364     __ relocate(relocInfo::poll_type);
13365     address pre_pc = __ pc();
13366     __ testl(rax, Address($poll$$Register, 0));
13367     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13368   %}
13369   ins_pipe(ialu_reg_mem);
13370 %}
13371 
13372 instruct mask_all_evexL(kReg dst, rRegL src) %{
13373   match(Set dst (MaskAll src));
13374   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
13375   ins_encode %{
13376     int mask_len = Matcher::vector_length(this);
13377     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
13378   %}
13379   ins_pipe( pipe_slow );
13380 %}
13381 
13382 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
13383   predicate(Matcher::vector_length(n) > 32);
13384   match(Set dst (MaskAll src));
13385   effect(TEMP tmp);
13386   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
13387   ins_encode %{
13388     int mask_len = Matcher::vector_length(this);
13389     __ movslq($tmp$$Register, $src$$Register);
13390     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
13391   %}
13392   ins_pipe( pipe_slow );
13393 %}
13394 
13395 // ============================================================================
13396 // Procedure Call/Return Instructions
13397 // Call Java Static Instruction
13398 // Note: If this code changes, the corresponding ret_addr_offset() and
13399 //       compute_padding() functions will have to be adjusted.
13400 instruct CallStaticJavaDirect(method meth) %{
13401   match(CallStaticJava);
13402   effect(USE meth);
13403 
13404   ins_cost(300);
13405   format %{ "call,static " %}
13406   opcode(0xE8); /* E8 cd */
13407   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13408   ins_pipe(pipe_slow);
13409   ins_alignment(4);
13410 %}
13411 
13412 // Call Java Dynamic Instruction
13413 // Note: If this code changes, the corresponding ret_addr_offset() and
13414 //       compute_padding() functions will have to be adjusted.
13415 instruct CallDynamicJavaDirect(method meth)
13416 %{
13417   match(CallDynamicJava);
13418   effect(USE meth);
13419 
13420   ins_cost(300);
13421   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13422             "call,dynamic " %}
13423   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13424   ins_pipe(pipe_slow);
13425   ins_alignment(4);
13426 %}
13427 
13428 // Call Runtime Instruction
13429 instruct CallRuntimeDirect(method meth)
13430 %{
13431   match(CallRuntime);
13432   effect(USE meth);
13433 
13434   ins_cost(300);
13435   format %{ "call,runtime " %}
13436   ins_encode(clear_avx, Java_To_Runtime(meth));
13437   ins_pipe(pipe_slow);
13438 %}
13439 
13440 // Call runtime without safepoint
13441 instruct CallLeafDirect(method meth)
13442 %{
13443   match(CallLeaf);
13444   effect(USE meth);
13445 
13446   ins_cost(300);
13447   format %{ "call_leaf,runtime " %}
13448   ins_encode(clear_avx, Java_To_Runtime(meth));
13449   ins_pipe(pipe_slow);
13450 %}
13451 
13452 // Call runtime without safepoint and with vector arguments
13453 instruct CallLeafDirectVector(method meth)
13454 %{
13455   match(CallLeafVector);
13456   effect(USE meth);
13457 
13458   ins_cost(300);
13459   format %{ "call_leaf,vector " %}
13460   ins_encode(Java_To_Runtime(meth));
13461   ins_pipe(pipe_slow);
13462 %}
13463 
13464 //
13465 instruct CallNativeDirect(method meth)
13466 %{
13467   match(CallNative);
13468   effect(USE meth);
13469 
13470   ins_cost(300);
13471   format %{ "call_native " %}
13472   ins_encode(clear_avx, Java_To_Runtime(meth));
13473   ins_pipe(pipe_slow);
13474 %}
13475 
13476 // Call runtime without safepoint
13477 // entry point is null, target holds the address to call
13478 instruct CallLeafNoFPInDirect(rRegP target)
13479 %{
13480   predicate(n->as_Call()->entry_point() == NULL);
13481   match(CallLeafNoFP target);
13482 
13483   ins_cost(300);
13484   format %{ "call_leaf_nofp,runtime indirect " %}
13485   ins_encode %{
13486      __ call($target$$Register);
13487   %}
13488 
13489   ins_pipe(pipe_slow);
13490 %}
13491 
13492 instruct CallLeafNoFPDirect(method meth)
13493 %{
13494   predicate(n->as_Call()->entry_point() != NULL);
13495   match(CallLeafNoFP);
13496   effect(USE meth);
13497 
13498   ins_cost(300);
13499   format %{ "call_leaf_nofp,runtime " %}
13500   ins_encode(clear_avx, Java_To_Runtime(meth));
13501   ins_pipe(pipe_slow);
13502 %}
13503 
13504 // Return Instruction
13505 // Remove the return address & jump to it.
13506 // Notice: We always emit a nop after a ret to make sure there is room
13507 // for safepoint patching
13508 instruct Ret()
13509 %{
13510   match(Return);
13511 
13512   format %{ "ret" %}
13513   ins_encode %{
13514     __ ret(0);
13515   %}
13516   ins_pipe(pipe_jmp);
13517 %}
13518 
13519 // Tail Call; Jump from runtime stub to Java code.
13520 // Also known as an 'interprocedural jump'.
13521 // Target of jump will eventually return to caller.
13522 // TailJump below removes the return address.
13523 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
13524 %{
13525   match(TailCall jump_target method_ptr);
13526 
13527   ins_cost(300);
13528   format %{ "jmp     $jump_target\t# rbx holds method" %}
13529   ins_encode %{
13530     __ jmp($jump_target$$Register);
13531   %}
13532   ins_pipe(pipe_jmp);
13533 %}
13534 
13535 // Tail Jump; remove the return address; jump to target.
13536 // TailCall above leaves the return address around.
13537 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13538 %{
13539   match(TailJump jump_target ex_oop);
13540 
13541   ins_cost(300);
13542   format %{ "popq    rdx\t# pop return address\n\t"
13543             "jmp     $jump_target" %}
13544   ins_encode %{
13545     __ popq(as_Register(RDX_enc));
13546     __ jmp($jump_target$$Register);
13547   %}
13548   ins_pipe(pipe_jmp);
13549 %}
13550 
13551 // Create exception oop: created by stack-crawling runtime code.
13552 // Created exception is now available to this handler, and is setup
13553 // just prior to jumping to this handler.  No code emitted.
13554 instruct CreateException(rax_RegP ex_oop)
13555 %{
13556   match(Set ex_oop (CreateEx));
13557 
13558   size(0);
13559   // use the following format syntax
13560   format %{ "# exception oop is in rax; no code emitted" %}
13561   ins_encode();
13562   ins_pipe(empty);
13563 %}
13564 
13565 // Rethrow exception:
13566 // The exception oop will come in the first argument position.
13567 // Then JUMP (not call) to the rethrow stub code.
13568 instruct RethrowException()
13569 %{
13570   match(Rethrow);
13571 
13572   // use the following format syntax
13573   format %{ "jmp     rethrow_stub" %}
13574   ins_encode(enc_rethrow);
13575   ins_pipe(pipe_jmp);
13576 %}
13577 
13578 // ============================================================================
13579 // This name is KNOWN by the ADLC and cannot be changed.
13580 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13581 // for this guy.
13582 instruct tlsLoadP(r15_RegP dst) %{
13583   match(Set dst (ThreadLocal));
13584   effect(DEF dst);
13585 
13586   size(0);
13587   format %{ "# TLS is in R15" %}
13588   ins_encode( /*empty encoding*/ );
13589   ins_pipe(ialu_reg_reg);
13590 %}
13591 
13592 
13593 //----------PEEPHOLE RULES-----------------------------------------------------
13594 // These must follow all instruction definitions as they use the names
13595 // defined in the instructions definitions.
13596 //
13597 // peepmatch ( root_instr_name [preceding_instruction]* );
13598 //
13599 // peepconstraint %{
13600 // (instruction_number.operand_name relational_op instruction_number.operand_name
13601 //  [, ...] );
13602 // // instruction numbers are zero-based using left to right order in peepmatch
13603 //
13604 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13605 // // provide an instruction_number.operand_name for each operand that appears
13606 // // in the replacement instruction's match rule
13607 //
13608 // ---------VM FLAGS---------------------------------------------------------
13609 //
13610 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13611 //
13612 // Each peephole rule is given an identifying number starting with zero and
13613 // increasing by one in the order seen by the parser.  An individual peephole
13614 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13615 // on the command-line.
13616 //
13617 // ---------CURRENT LIMITATIONS----------------------------------------------
13618 //
13619 // Only match adjacent instructions in same basic block
13620 // Only equality constraints
13621 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13622 // Only one replacement instruction
13623 //
13624 // ---------EXAMPLE----------------------------------------------------------
13625 //
13626 // // pertinent parts of existing instructions in architecture description
13627 // instruct movI(rRegI dst, rRegI src)
13628 // %{
13629 //   match(Set dst (CopyI src));
13630 // %}
13631 //
13632 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
13633 // %{
13634 //   match(Set dst (AddI dst src));
13635 //   effect(KILL cr);
13636 // %}
13637 //
13638 // // Change (inc mov) to lea
13639 // peephole %{
13640 //   // increment preceeded by register-register move
13641 //   peepmatch ( incI_rReg movI );
13642 //   // require that the destination register of the increment
13643 //   // match the destination register of the move
13644 //   peepconstraint ( 0.dst == 1.dst );
13645 //   // construct a replacement instruction that sets
13646 //   // the destination to ( move's source register + one )
13647 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13648 // %}
13649 //
13650 
13651 // Implementation no longer uses movX instructions since
13652 // machine-independent system no longer uses CopyX nodes.
13653 //
13654 // peephole
13655 // %{
13656 //   peepmatch (incI_rReg movI);
13657 //   peepconstraint (0.dst == 1.dst);
13658 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13659 // %}
13660 
13661 // peephole
13662 // %{
13663 //   peepmatch (decI_rReg movI);
13664 //   peepconstraint (0.dst == 1.dst);
13665 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13666 // %}
13667 
13668 // peephole
13669 // %{
13670 //   peepmatch (addI_rReg_imm movI);
13671 //   peepconstraint (0.dst == 1.dst);
13672 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13673 // %}
13674 
13675 // peephole
13676 // %{
13677 //   peepmatch (incL_rReg movL);
13678 //   peepconstraint (0.dst == 1.dst);
13679 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13680 // %}
13681 
13682 // peephole
13683 // %{
13684 //   peepmatch (decL_rReg movL);
13685 //   peepconstraint (0.dst == 1.dst);
13686 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13687 // %}
13688 
13689 // peephole
13690 // %{
13691 //   peepmatch (addL_rReg_imm movL);
13692 //   peepconstraint (0.dst == 1.dst);
13693 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13694 // %}
13695 
13696 // peephole
13697 // %{
13698 //   peepmatch (addP_rReg_imm movP);
13699 //   peepconstraint (0.dst == 1.dst);
13700 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13701 // %}
13702 
13703 // // Change load of spilled value to only a spill
13704 // instruct storeI(memory mem, rRegI src)
13705 // %{
13706 //   match(Set mem (StoreI mem src));
13707 // %}
13708 //
13709 // instruct loadI(rRegI dst, memory mem)
13710 // %{
13711 //   match(Set dst (LoadI mem));
13712 // %}
13713 //
13714 
13715 peephole
13716 %{
13717   peepmatch (loadI storeI);
13718   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13719   peepreplace (storeI(1.mem 1.mem 1.src));
13720 %}
13721 
13722 peephole
13723 %{
13724   peepmatch (loadL storeL);
13725   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13726   peepreplace (storeL(1.mem 1.mem 1.src));
13727 %}
13728 
13729 //----------SMARTSPILL RULES---------------------------------------------------
13730 // These must follow all instruction definitions as they use the names
13731 // defined in the instructions definitions.