1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 %}
  426 
  427 // Register masks
  428 source_hpp %{
  429 
  430 extern RegMask _ANY_REG_mask;
  431 extern RegMask _PTR_REG_mask;
  432 extern RegMask _PTR_REG_NO_RBP_mask;
  433 extern RegMask _PTR_NO_RAX_REG_mask;
  434 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  435 extern RegMask _LONG_REG_mask;
  436 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  437 extern RegMask _LONG_NO_RCX_REG_mask;
  438 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  439 extern RegMask _INT_REG_mask;
  440 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  441 extern RegMask _INT_NO_RCX_REG_mask;
  442 extern RegMask _INT_NO_RBP_R13_REG_mask;
  443 extern RegMask _FLOAT_REG_mask;
  444 
  445 extern RegMask _STACK_OR_PTR_REG_mask;
  446 extern RegMask _STACK_OR_LONG_REG_mask;
  447 extern RegMask _STACK_OR_INT_REG_mask;
  448 
  449 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  450 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  451 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  452 
  453 %}
  454 
  455 source %{
  456 #define   RELOC_IMM64    Assembler::imm_operand
  457 #define   RELOC_DISP32   Assembler::disp32_operand
  458 
  459 #define __ masm->
  460 
  461 RegMask _ANY_REG_mask;
  462 RegMask _PTR_REG_mask;
  463 RegMask _PTR_REG_NO_RBP_mask;
  464 RegMask _PTR_NO_RAX_REG_mask;
  465 RegMask _PTR_NO_RAX_RBX_REG_mask;
  466 RegMask _LONG_REG_mask;
  467 RegMask _LONG_NO_RAX_RDX_REG_mask;
  468 RegMask _LONG_NO_RCX_REG_mask;
  469 RegMask _LONG_NO_RBP_R13_REG_mask;
  470 RegMask _INT_REG_mask;
  471 RegMask _INT_NO_RAX_RDX_REG_mask;
  472 RegMask _INT_NO_RCX_REG_mask;
  473 RegMask _INT_NO_RBP_R13_REG_mask;
  474 RegMask _FLOAT_REG_mask;
  475 RegMask _STACK_OR_PTR_REG_mask;
  476 RegMask _STACK_OR_LONG_REG_mask;
  477 RegMask _STACK_OR_INT_REG_mask;
  478 
  479 static bool need_r12_heapbase() {
  480   return UseCompressedOops;
  481 }
  482 
  483 void reg_mask_init() {
  484   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  485 
  486   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  487   // We derive a number of subsets from it.
  488   _ANY_REG_mask = _ALL_REG_mask;
  489 
  490   if (PreserveFramePointer) {
  491     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  492     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  493   }
  494   if (need_r12_heapbase()) {
  495     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  496     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  497   }
  498 
  499   _PTR_REG_mask = _ANY_REG_mask;
  500   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  501   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  502   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  503   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  504   if (!UseAPX) {
  505     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  506       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  507       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  508     }
  509   }
  510 
  511   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  512   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  513 
  514   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  515   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  516   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  517 
  518   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  519   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  520   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  521 
  522   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  523   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  524   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  525 
  526 
  527   _LONG_REG_mask = _PTR_REG_mask;
  528   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  529   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  530 
  531   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  532   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  533   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  534   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  535   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  536 
  537   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  538   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  539   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  540 
  541   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  542   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  543   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  544   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  545   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  546 
  547   _INT_REG_mask = _ALL_INT_REG_mask;
  548   if (!UseAPX) {
  549     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  550       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  551     }
  552   }
  553 
  554   if (PreserveFramePointer) {
  555     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  556   }
  557   if (need_r12_heapbase()) {
  558     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  559   }
  560 
  561   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  562   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  563 
  564   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  565   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  566   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  567 
  568   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  569   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  570 
  571   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  572   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  573   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  574 
  575   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  576   // from the float_reg_legacy/float_reg_evex register class.
  577   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  578 }
  579 
  580 static bool generate_vzeroupper(Compile* C) {
  581   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  582 }
  583 
  584 static int clear_avx_size() {
  585   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  586 }
  587 
  588 // !!!!! Special hack to get all types of calls to specify the byte offset
  589 //       from the start of the call to the point where the return address
  590 //       will point.
  591 int MachCallStaticJavaNode::ret_addr_offset()
  592 {
  593   int offset = 5; // 5 bytes from start of call to where return address points
  594   offset += clear_avx_size();
  595   return offset;
  596 }
  597 
  598 int MachCallDynamicJavaNode::ret_addr_offset()
  599 {
  600   int offset = 15; // 15 bytes from start of call to where return address points
  601   offset += clear_avx_size();
  602   return offset;
  603 }
  604 
  605 int MachCallRuntimeNode::ret_addr_offset() {
  606   if (_entry_point == nullptr) {
  607     // CallLeafNoFPInDirect
  608     return 3; // callq (register)
  609   }
  610   int offset = 13; // movq r10,#addr; callq (r10)
  611   if (this->ideal_Opcode() != Op_CallLeafVector) {
  612     offset += clear_avx_size();
  613   }
  614   return offset;
  615 }
  616 
  617 //
  618 // Compute padding required for nodes which need alignment
  619 //
  620 
  621 // The address of the call instruction needs to be 4-byte aligned to
  622 // ensure that it does not span a cache line so that it can be patched.
  623 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  624 {
  625   current_offset += clear_avx_size(); // skip vzeroupper
  626   current_offset += 1; // skip call opcode byte
  627   return align_up(current_offset, alignment_required()) - current_offset;
  628 }
  629 
  630 // The address of the call instruction needs to be 4-byte aligned to
  631 // ensure that it does not span a cache line so that it can be patched.
  632 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  633 {
  634   current_offset += clear_avx_size(); // skip vzeroupper
  635   current_offset += 11; // skip movq instruction + call opcode byte
  636   return align_up(current_offset, alignment_required()) - current_offset;
  637 }
  638 
  639 // This could be in MacroAssembler but it's fairly C2 specific
  640 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  641   Label exit;
  642   __ jccb(Assembler::noParity, exit);
  643   __ pushf();
  644   //
  645   // comiss/ucomiss instructions set ZF,PF,CF flags and
  646   // zero OF,AF,SF for NaN values.
  647   // Fixup flags by zeroing ZF,PF so that compare of NaN
  648   // values returns 'less than' result (CF is set).
  649   // Leave the rest of flags unchanged.
  650   //
  651   //    7 6 5 4 3 2 1 0
  652   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  653   //    0 0 1 0 1 0 1 1   (0x2B)
  654   //
  655   __ andq(Address(rsp, 0), 0xffffff2b);
  656   __ popf();
  657   __ bind(exit);
  658 }
  659 
  660 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  661   Label done;
  662   __ movl(dst, -1);
  663   __ jcc(Assembler::parity, done);
  664   __ jcc(Assembler::below, done);
  665   __ setcc(Assembler::notEqual, dst);
  666   __ bind(done);
  667 }
  668 
  669 // Math.min()    # Math.max()
  670 // --------------------------
  671 // ucomis[s/d]   #
  672 // ja   -> b     # a
  673 // jp   -> NaN   # NaN
  674 // jb   -> a     # b
  675 // je            #
  676 // |-jz -> a | b # a & b
  677 // |    -> a     #
  678 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  679                             XMMRegister a, XMMRegister b,
  680                             XMMRegister xmmt, Register rt,
  681                             bool min, bool single) {
  682 
  683   Label nan, zero, below, above, done;
  684 
  685   if (single)
  686     __ ucomiss(a, b);
  687   else
  688     __ ucomisd(a, b);
  689 
  690   if (dst->encoding() != (min ? b : a)->encoding())
  691     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  692   else
  693     __ jccb(Assembler::above, done);
  694 
  695   __ jccb(Assembler::parity, nan);  // PF=1
  696   __ jccb(Assembler::below, below); // CF=1
  697 
  698   // equal
  699   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  700   if (single) {
  701     __ ucomiss(a, xmmt);
  702     __ jccb(Assembler::equal, zero);
  703 
  704     __ movflt(dst, a);
  705     __ jmp(done);
  706   }
  707   else {
  708     __ ucomisd(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movdbl(dst, a);
  712     __ jmp(done);
  713   }
  714 
  715   __ bind(zero);
  716   if (min)
  717     __ vpor(dst, a, b, Assembler::AVX_128bit);
  718   else
  719     __ vpand(dst, a, b, Assembler::AVX_128bit);
  720 
  721   __ jmp(done);
  722 
  723   __ bind(above);
  724   if (single)
  725     __ movflt(dst, min ? b : a);
  726   else
  727     __ movdbl(dst, min ? b : a);
  728 
  729   __ jmp(done);
  730 
  731   __ bind(nan);
  732   if (single) {
  733     __ movl(rt, 0x7fc00000); // Float.NaN
  734     __ movdl(dst, rt);
  735   }
  736   else {
  737     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  738     __ movdq(dst, rt);
  739   }
  740   __ jmp(done);
  741 
  742   __ bind(below);
  743   if (single)
  744     __ movflt(dst, min ? a : b);
  745   else
  746     __ movdbl(dst, min ? a : b);
  747 
  748   __ bind(done);
  749 }
  750 
  751 //=============================================================================
  752 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  753 
  754 int ConstantTable::calculate_table_base_offset() const {
  755   return 0;  // absolute addressing, no offset
  756 }
  757 
  758 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  759 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  760   ShouldNotReachHere();
  761 }
  762 
  763 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  764   // Empty encoding
  765 }
  766 
  767 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  768   return 0;
  769 }
  770 
  771 #ifndef PRODUCT
  772 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  773   st->print("# MachConstantBaseNode (empty encoding)");
  774 }
  775 #endif
  776 
  777 
  778 //=============================================================================
  779 #ifndef PRODUCT
  780 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  781   Compile* C = ra_->C;
  782 
  783   int framesize = C->output()->frame_size_in_bytes();
  784   int bangsize = C->output()->bang_size_in_bytes();
  785   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  786   // Remove wordSize for return addr which is already pushed.
  787   framesize -= wordSize;
  788 
  789   if (C->output()->need_stack_bang(bangsize)) {
  790     framesize -= wordSize;
  791     st->print("# stack bang (%d bytes)", bangsize);
  792     st->print("\n\t");
  793     st->print("pushq   rbp\t# Save rbp");
  794     if (PreserveFramePointer) {
  795         st->print("\n\t");
  796         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  797     }
  798     if (framesize) {
  799       st->print("\n\t");
  800       st->print("subq    rsp, #%d\t# Create frame",framesize);
  801     }
  802   } else {
  803     st->print("subq    rsp, #%d\t# Create frame",framesize);
  804     st->print("\n\t");
  805     framesize -= wordSize;
  806     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  807     if (PreserveFramePointer) {
  808       st->print("\n\t");
  809       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  810       if (framesize > 0) {
  811         st->print("\n\t");
  812         st->print("addq    rbp, #%d", framesize);
  813       }
  814     }
  815   }
  816 
  817   if (VerifyStackAtCalls) {
  818     st->print("\n\t");
  819     framesize -= wordSize;
  820     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  821 #ifdef ASSERT
  822     st->print("\n\t");
  823     st->print("# stack alignment check");
  824 #endif
  825   }
  826   if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
  827     st->print("\n\t");
  828     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  829     st->print("\n\t");
  830     st->print("je      fast_entry\t");
  831     st->print("\n\t");
  832     st->print("call    #nmethod_entry_barrier_stub\t");
  833     st->print("\n\tfast_entry:");
  834   }
  835   st->cr();
  836 }
  837 #endif
  838 
  839 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  840   Compile* C = ra_->C;
  841 
  842   __ verified_entry(C);
  843 
  844   if (ra_->C->stub_function() == nullptr) {
  845     __ entry_barrier();
  846   }
  847 
  848   if (!Compile::current()->output()->in_scratch_emit_size()) {
  849     __ bind(*_verified_entry);
  850   }
  851 
  852   C->output()->set_frame_complete(__ offset());
  853 
  854   if (C->has_mach_constant_base_node()) {
  855     // NOTE: We set the table base offset here because users might be
  856     // emitted before MachConstantBaseNode.
  857     ConstantTable& constant_table = C->output()->constant_table();
  858     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  859   }
  860 }
  861 
  862 int MachPrologNode::reloc() const
  863 {
  864   return 0; // a large enough number
  865 }
  866 
  867 //=============================================================================
  868 #ifndef PRODUCT
  869 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  870 {
  871   Compile* C = ra_->C;
  872   if (generate_vzeroupper(C)) {
  873     st->print("vzeroupper");
  874     st->cr(); st->print("\t");
  875   }
  876 
  877   int framesize = C->output()->frame_size_in_bytes();
  878   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  879   // Remove word for return adr already pushed
  880   // and RBP
  881   framesize -= 2*wordSize;
  882 
  883   if (framesize) {
  884     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  885     st->print("\t");
  886   }
  887 
  888   st->print_cr("popq    rbp");
  889   if (do_polling() && C->is_method_compilation()) {
  890     st->print("\t");
  891     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  892                  "ja      #safepoint_stub\t"
  893                  "# Safepoint: poll for GC");
  894   }
  895 }
  896 #endif
  897 
  898 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  899 {
  900   Compile* C = ra_->C;
  901 
  902   if (generate_vzeroupper(C)) {
  903     // Clear upper bits of YMM registers when current compiled code uses
  904     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  905     __ vzeroupper();
  906   }
  907 
  908   // Subtract two words to account for return address and rbp
  909   int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
  910   __ remove_frame(initial_framesize, C->needs_stack_repair());
  911 
  912   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  913     __ reserved_stack_check();
  914   }
  915 
  916   if (do_polling() && C->is_method_compilation()) {
  917     Label dummy_label;
  918     Label* code_stub = &dummy_label;
  919     if (!C->output()->in_scratch_emit_size()) {
  920       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  921       C->output()->add_stub(stub);
  922       code_stub = &stub->entry();
  923     }
  924     __ relocate(relocInfo::poll_return_type);
  925     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  926   }
  927 }
  928 
  929 int MachEpilogNode::reloc() const
  930 {
  931   return 2; // a large enough number
  932 }
  933 
  934 const Pipeline* MachEpilogNode::pipeline() const
  935 {
  936   return MachNode::pipeline_class();
  937 }
  938 
  939 //=============================================================================
  940 
  941 enum RC {
  942   rc_bad,
  943   rc_int,
  944   rc_kreg,
  945   rc_float,
  946   rc_stack
  947 };
  948 
  949 static enum RC rc_class(OptoReg::Name reg)
  950 {
  951   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  952 
  953   if (OptoReg::is_stack(reg)) return rc_stack;
  954 
  955   VMReg r = OptoReg::as_VMReg(reg);
  956 
  957   if (r->is_Register()) return rc_int;
  958 
  959   if (r->is_KRegister()) return rc_kreg;
  960 
  961   assert(r->is_XMMRegister(), "must be");
  962   return rc_float;
  963 }
  964 
  965 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  966 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
  967                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  968 
  969 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
  970                      int stack_offset, int reg, uint ireg, outputStream* st);
  971 
  972 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
  973                                       int dst_offset, uint ireg, outputStream* st) {
  974   if (masm) {
  975     switch (ireg) {
  976     case Op_VecS:
  977       __ movq(Address(rsp, -8), rax);
  978       __ movl(rax, Address(rsp, src_offset));
  979       __ movl(Address(rsp, dst_offset), rax);
  980       __ movq(rax, Address(rsp, -8));
  981       break;
  982     case Op_VecD:
  983       __ pushq(Address(rsp, src_offset));
  984       __ popq (Address(rsp, dst_offset));
  985       break;
  986     case Op_VecX:
  987       __ pushq(Address(rsp, src_offset));
  988       __ popq (Address(rsp, dst_offset));
  989       __ pushq(Address(rsp, src_offset+8));
  990       __ popq (Address(rsp, dst_offset+8));
  991       break;
  992     case Op_VecY:
  993       __ vmovdqu(Address(rsp, -32), xmm0);
  994       __ vmovdqu(xmm0, Address(rsp, src_offset));
  995       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  996       __ vmovdqu(xmm0, Address(rsp, -32));
  997       break;
  998     case Op_VecZ:
  999       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1000       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1001       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1002       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1003       break;
 1004     default:
 1005       ShouldNotReachHere();
 1006     }
 1007 #ifndef PRODUCT
 1008   } else {
 1009     switch (ireg) {
 1010     case Op_VecS:
 1011       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1012                 "movl    rax, [rsp + #%d]\n\t"
 1013                 "movl    [rsp + #%d], rax\n\t"
 1014                 "movq    rax, [rsp - #8]",
 1015                 src_offset, dst_offset);
 1016       break;
 1017     case Op_VecD:
 1018       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1019                 "popq    [rsp + #%d]",
 1020                 src_offset, dst_offset);
 1021       break;
 1022      case Op_VecX:
 1023       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1024                 "popq    [rsp + #%d]\n\t"
 1025                 "pushq   [rsp + #%d]\n\t"
 1026                 "popq    [rsp + #%d]",
 1027                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1028       break;
 1029     case Op_VecY:
 1030       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1031                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1032                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1033                 "vmovdqu xmm0, [rsp - #32]",
 1034                 src_offset, dst_offset);
 1035       break;
 1036     case Op_VecZ:
 1037       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1038                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1039                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1040                 "vmovdqu xmm0, [rsp - #64]",
 1041                 src_offset, dst_offset);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #endif
 1047   }
 1048 }
 1049 
 1050 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1051                                        PhaseRegAlloc* ra_,
 1052                                        bool do_size,
 1053                                        outputStream* st) const {
 1054   assert(masm != nullptr || st  != nullptr, "sanity");
 1055   // Get registers to move
 1056   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1057   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1058   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1059   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1060 
 1061   enum RC src_second_rc = rc_class(src_second);
 1062   enum RC src_first_rc = rc_class(src_first);
 1063   enum RC dst_second_rc = rc_class(dst_second);
 1064   enum RC dst_first_rc = rc_class(dst_first);
 1065 
 1066   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1067          "must move at least 1 register" );
 1068 
 1069   if (src_first == dst_first && src_second == dst_second) {
 1070     // Self copy, no move
 1071     return 0;
 1072   }
 1073   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1074     uint ireg = ideal_reg();
 1075     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1076     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1077     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1078       // mem -> mem
 1079       int src_offset = ra_->reg2offset(src_first);
 1080       int dst_offset = ra_->reg2offset(dst_first);
 1081       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1082     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1083       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1084     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1085       int stack_offset = ra_->reg2offset(dst_first);
 1086       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1087     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1088       int stack_offset = ra_->reg2offset(src_first);
 1089       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1090     } else {
 1091       ShouldNotReachHere();
 1092     }
 1093     return 0;
 1094   }
 1095   if (src_first_rc == rc_stack) {
 1096     // mem ->
 1097     if (dst_first_rc == rc_stack) {
 1098       // mem -> mem
 1099       assert(src_second != dst_first, "overlap");
 1100       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1101           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1102         // 64-bit
 1103         int src_offset = ra_->reg2offset(src_first);
 1104         int dst_offset = ra_->reg2offset(dst_first);
 1105         if (masm) {
 1106           __ pushq(Address(rsp, src_offset));
 1107           __ popq (Address(rsp, dst_offset));
 1108 #ifndef PRODUCT
 1109         } else {
 1110           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1111                     "popq    [rsp + #%d]",
 1112                      src_offset, dst_offset);
 1113 #endif
 1114         }
 1115       } else {
 1116         // 32-bit
 1117         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1118         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1119         // No pushl/popl, so:
 1120         int src_offset = ra_->reg2offset(src_first);
 1121         int dst_offset = ra_->reg2offset(dst_first);
 1122         if (masm) {
 1123           __ movq(Address(rsp, -8), rax);
 1124           __ movl(rax, Address(rsp, src_offset));
 1125           __ movl(Address(rsp, dst_offset), rax);
 1126           __ movq(rax, Address(rsp, -8));
 1127 #ifndef PRODUCT
 1128         } else {
 1129           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1130                     "movl    rax, [rsp + #%d]\n\t"
 1131                     "movl    [rsp + #%d], rax\n\t"
 1132                     "movq    rax, [rsp - #8]",
 1133                      src_offset, dst_offset);
 1134 #endif
 1135         }
 1136       }
 1137       return 0;
 1138     } else if (dst_first_rc == rc_int) {
 1139       // mem -> gpr
 1140       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1141           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1142         // 64-bit
 1143         int offset = ra_->reg2offset(src_first);
 1144         if (masm) {
 1145           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1146 #ifndef PRODUCT
 1147         } else {
 1148           st->print("movq    %s, [rsp + #%d]\t# spill",
 1149                      Matcher::regName[dst_first],
 1150                      offset);
 1151 #endif
 1152         }
 1153       } else {
 1154         // 32-bit
 1155         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1156         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1157         int offset = ra_->reg2offset(src_first);
 1158         if (masm) {
 1159           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1160 #ifndef PRODUCT
 1161         } else {
 1162           st->print("movl    %s, [rsp + #%d]\t# spill",
 1163                      Matcher::regName[dst_first],
 1164                      offset);
 1165 #endif
 1166         }
 1167       }
 1168       return 0;
 1169     } else if (dst_first_rc == rc_float) {
 1170       // mem-> xmm
 1171       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1172           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1173         // 64-bit
 1174         int offset = ra_->reg2offset(src_first);
 1175         if (masm) {
 1176           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1177 #ifndef PRODUCT
 1178         } else {
 1179           st->print("%s  %s, [rsp + #%d]\t# spill",
 1180                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1181                      Matcher::regName[dst_first],
 1182                      offset);
 1183 #endif
 1184         }
 1185       } else {
 1186         // 32-bit
 1187         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1188         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1189         int offset = ra_->reg2offset(src_first);
 1190         if (masm) {
 1191           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1192 #ifndef PRODUCT
 1193         } else {
 1194           st->print("movss   %s, [rsp + #%d]\t# spill",
 1195                      Matcher::regName[dst_first],
 1196                      offset);
 1197 #endif
 1198         }
 1199       }
 1200       return 0;
 1201     } else if (dst_first_rc == rc_kreg) {
 1202       // mem -> kreg
 1203       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1204           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1205         // 64-bit
 1206         int offset = ra_->reg2offset(src_first);
 1207         if (masm) {
 1208           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1212                      Matcher::regName[dst_first],
 1213                      offset);
 1214 #endif
 1215         }
 1216       }
 1217       return 0;
 1218     }
 1219   } else if (src_first_rc == rc_int) {
 1220     // gpr ->
 1221     if (dst_first_rc == rc_stack) {
 1222       // gpr -> mem
 1223       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1224           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1225         // 64-bit
 1226         int offset = ra_->reg2offset(dst_first);
 1227         if (masm) {
 1228           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp + #%d], %s\t# spill",
 1232                      offset,
 1233                      Matcher::regName[src_first]);
 1234 #endif
 1235         }
 1236       } else {
 1237         // 32-bit
 1238         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1239         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1240         int offset = ra_->reg2offset(dst_first);
 1241         if (masm) {
 1242           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1243 #ifndef PRODUCT
 1244         } else {
 1245           st->print("movl    [rsp + #%d], %s\t# spill",
 1246                      offset,
 1247                      Matcher::regName[src_first]);
 1248 #endif
 1249         }
 1250       }
 1251       return 0;
 1252     } else if (dst_first_rc == rc_int) {
 1253       // gpr -> gpr
 1254       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1255           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1256         // 64-bit
 1257         if (masm) {
 1258           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1259                   as_Register(Matcher::_regEncode[src_first]));
 1260 #ifndef PRODUCT
 1261         } else {
 1262           st->print("movq    %s, %s\t# spill",
 1263                      Matcher::regName[dst_first],
 1264                      Matcher::regName[src_first]);
 1265 #endif
 1266         }
 1267         return 0;
 1268       } else {
 1269         // 32-bit
 1270         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1271         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1272         if (masm) {
 1273           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1274                   as_Register(Matcher::_regEncode[src_first]));
 1275 #ifndef PRODUCT
 1276         } else {
 1277           st->print("movl    %s, %s\t# spill",
 1278                      Matcher::regName[dst_first],
 1279                      Matcher::regName[src_first]);
 1280 #endif
 1281         }
 1282         return 0;
 1283       }
 1284     } else if (dst_first_rc == rc_float) {
 1285       // gpr -> xmm
 1286       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1287           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1288         // 64-bit
 1289         if (masm) {
 1290           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1291 #ifndef PRODUCT
 1292         } else {
 1293           st->print("movdq   %s, %s\t# spill",
 1294                      Matcher::regName[dst_first],
 1295                      Matcher::regName[src_first]);
 1296 #endif
 1297         }
 1298       } else {
 1299         // 32-bit
 1300         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1301         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1302         if (masm) {
 1303           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1304 #ifndef PRODUCT
 1305         } else {
 1306           st->print("movdl   %s, %s\t# spill",
 1307                      Matcher::regName[dst_first],
 1308                      Matcher::regName[src_first]);
 1309 #endif
 1310         }
 1311       }
 1312       return 0;
 1313     } else if (dst_first_rc == rc_kreg) {
 1314       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1315           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1316         // 64-bit
 1317         if (masm) {
 1318           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1319   #ifndef PRODUCT
 1320         } else {
 1321            st->print("kmovq   %s, %s\t# spill",
 1322                        Matcher::regName[dst_first],
 1323                        Matcher::regName[src_first]);
 1324   #endif
 1325         }
 1326       }
 1327       Unimplemented();
 1328       return 0;
 1329     }
 1330   } else if (src_first_rc == rc_float) {
 1331     // xmm ->
 1332     if (dst_first_rc == rc_stack) {
 1333       // xmm -> mem
 1334       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1335           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1336         // 64-bit
 1337         int offset = ra_->reg2offset(dst_first);
 1338         if (masm) {
 1339           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1340 #ifndef PRODUCT
 1341         } else {
 1342           st->print("movsd   [rsp + #%d], %s\t# spill",
 1343                      offset,
 1344                      Matcher::regName[src_first]);
 1345 #endif
 1346         }
 1347       } else {
 1348         // 32-bit
 1349         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1350         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1351         int offset = ra_->reg2offset(dst_first);
 1352         if (masm) {
 1353           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1354 #ifndef PRODUCT
 1355         } else {
 1356           st->print("movss   [rsp + #%d], %s\t# spill",
 1357                      offset,
 1358                      Matcher::regName[src_first]);
 1359 #endif
 1360         }
 1361       }
 1362       return 0;
 1363     } else if (dst_first_rc == rc_int) {
 1364       // xmm -> gpr
 1365       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1366           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1367         // 64-bit
 1368         if (masm) {
 1369           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movdq   %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377       } else {
 1378         // 32-bit
 1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1381         if (masm) {
 1382           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1383 #ifndef PRODUCT
 1384         } else {
 1385           st->print("movdl   %s, %s\t# spill",
 1386                      Matcher::regName[dst_first],
 1387                      Matcher::regName[src_first]);
 1388 #endif
 1389         }
 1390       }
 1391       return 0;
 1392     } else if (dst_first_rc == rc_float) {
 1393       // xmm -> xmm
 1394       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1395           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1396         // 64-bit
 1397         if (masm) {
 1398           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1399 #ifndef PRODUCT
 1400         } else {
 1401           st->print("%s  %s, %s\t# spill",
 1402                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1403                      Matcher::regName[dst_first],
 1404                      Matcher::regName[src_first]);
 1405 #endif
 1406         }
 1407       } else {
 1408         // 32-bit
 1409         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1410         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1411         if (masm) {
 1412           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1413 #ifndef PRODUCT
 1414         } else {
 1415           st->print("%s  %s, %s\t# spill",
 1416                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1417                      Matcher::regName[dst_first],
 1418                      Matcher::regName[src_first]);
 1419 #endif
 1420         }
 1421       }
 1422       return 0;
 1423     } else if (dst_first_rc == rc_kreg) {
 1424       assert(false, "Illegal spilling");
 1425       return 0;
 1426     }
 1427   } else if (src_first_rc == rc_kreg) {
 1428     if (dst_first_rc == rc_stack) {
 1429       // mem -> kreg
 1430       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1431           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1432         // 64-bit
 1433         int offset = ra_->reg2offset(dst_first);
 1434         if (masm) {
 1435           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1436 #ifndef PRODUCT
 1437         } else {
 1438           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1439                      offset,
 1440                      Matcher::regName[src_first]);
 1441 #endif
 1442         }
 1443       }
 1444       return 0;
 1445     } else if (dst_first_rc == rc_int) {
 1446       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1447           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1448         // 64-bit
 1449         if (masm) {
 1450           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1451 #ifndef PRODUCT
 1452         } else {
 1453          st->print("kmovq   %s, %s\t# spill",
 1454                      Matcher::regName[dst_first],
 1455                      Matcher::regName[src_first]);
 1456 #endif
 1457         }
 1458       }
 1459       Unimplemented();
 1460       return 0;
 1461     } else if (dst_first_rc == rc_kreg) {
 1462       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1463           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1464         // 64-bit
 1465         if (masm) {
 1466           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1467 #ifndef PRODUCT
 1468         } else {
 1469          st->print("kmovq   %s, %s\t# spill",
 1470                      Matcher::regName[dst_first],
 1471                      Matcher::regName[src_first]);
 1472 #endif
 1473         }
 1474       }
 1475       return 0;
 1476     } else if (dst_first_rc == rc_float) {
 1477       assert(false, "Illegal spill");
 1478       return 0;
 1479     }
 1480   }
 1481 
 1482   assert(0," foo ");
 1483   Unimplemented();
 1484   return 0;
 1485 }
 1486 
 1487 #ifndef PRODUCT
 1488 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1489   implementation(nullptr, ra_, false, st);
 1490 }
 1491 #endif
 1492 
 1493 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1494   implementation(masm, ra_, false, nullptr);
 1495 }
 1496 
 1497 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1498   return MachNode::size(ra_);
 1499 }
 1500 
 1501 //=============================================================================
 1502 #ifndef PRODUCT
 1503 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1504 {
 1505   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1506   int reg = ra_->get_reg_first(this);
 1507   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1508             Matcher::regName[reg], offset);
 1509 }
 1510 #endif
 1511 
 1512 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1513 {
 1514   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1515   int reg = ra_->get_encode(this);
 1516 
 1517   __ lea(as_Register(reg), Address(rsp, offset));
 1518 }
 1519 
 1520 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1521 {
 1522   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1523   return (offset < 0x80) ? 5 : 8; // REX
 1524 }
 1525 
 1526 //=============================================================================
 1527 #ifndef PRODUCT
 1528 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1529 {
 1530   st->print_cr("MachVEPNode");
 1531 }
 1532 #endif
 1533 
 1534 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1535 {
 1536   CodeBuffer* cbuf = masm->code();
 1537   uint insts_size = cbuf->insts_size();
 1538   if (!_verified) {
 1539     __ ic_check(1);
 1540   } else {
 1541     // TODO 8284443 Avoid creation of temporary frame
 1542     if (ra_->C->stub_function() == nullptr) {
 1543       __ verified_entry(ra_->C, 0);
 1544       __ entry_barrier();
 1545       int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
 1546       __ remove_frame(initial_framesize, false);
 1547     }
 1548     // Unpack inline type args passed as oop and then jump to
 1549     // the verified entry point (skipping the unverified entry).
 1550     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 1551     // Emit code for verified entry and save increment for stack repair on return
 1552     __ verified_entry(ra_->C, sp_inc);
 1553     if (Compile::current()->output()->in_scratch_emit_size()) {
 1554       Label dummy_verified_entry;
 1555       __ jmp(dummy_verified_entry);
 1556     } else {
 1557       __ jmp(*_verified_entry);
 1558     }
 1559   }
 1560   /* WARNING these NOPs are critical so that verified entry point is properly
 1561      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1562   int nops_cnt = 4 - ((cbuf->insts_size() - insts_size) & 0x3);
 1563   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1564   if (nops_cnt > 0) {
 1565     __ nop(nops_cnt);
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 //=============================================================================
 1590 
 1591 bool Matcher::supports_vector_calling_convention(void) {
 1592   if (EnableVectorSupport && UseVectorStubs) {
 1593     return true;
 1594   }
 1595   return false;
 1596 }
 1597 
 1598 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1599   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1600   int lo = XMM0_num;
 1601   int hi = XMM0b_num;
 1602   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1603   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1604   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1605   return OptoRegPair(hi, lo);
 1606 }
 1607 
 1608 // Is this branch offset short enough that a short branch can be used?
 1609 //
 1610 // NOTE: If the platform does not provide any short branch variants, then
 1611 //       this method should return false for offset 0.
 1612 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1613   // The passed offset is relative to address of the branch.
 1614   // On 86 a branch displacement is calculated relative to address
 1615   // of a next instruction.
 1616   offset -= br_size;
 1617 
 1618   // the short version of jmpConUCF2 contains multiple branches,
 1619   // making the reach slightly less
 1620   if (rule == jmpConUCF2_rule)
 1621     return (-126 <= offset && offset <= 125);
 1622   return (-128 <= offset && offset <= 127);
 1623 }
 1624 
 1625 // Return whether or not this register is ever used as an argument.
 1626 // This function is used on startup to build the trampoline stubs in
 1627 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1628 // call in the trampoline, and arguments in those registers not be
 1629 // available to the callee.
 1630 bool Matcher::can_be_java_arg(int reg)
 1631 {
 1632   return
 1633     reg ==  RDI_num || reg == RDI_H_num ||
 1634     reg ==  RSI_num || reg == RSI_H_num ||
 1635     reg ==  RDX_num || reg == RDX_H_num ||
 1636     reg ==  RCX_num || reg == RCX_H_num ||
 1637     reg ==   R8_num || reg ==  R8_H_num ||
 1638     reg ==   R9_num || reg ==  R9_H_num ||
 1639     reg ==  R12_num || reg == R12_H_num ||
 1640     reg == XMM0_num || reg == XMM0b_num ||
 1641     reg == XMM1_num || reg == XMM1b_num ||
 1642     reg == XMM2_num || reg == XMM2b_num ||
 1643     reg == XMM3_num || reg == XMM3b_num ||
 1644     reg == XMM4_num || reg == XMM4b_num ||
 1645     reg == XMM5_num || reg == XMM5b_num ||
 1646     reg == XMM6_num || reg == XMM6b_num ||
 1647     reg == XMM7_num || reg == XMM7b_num;
 1648 }
 1649 
 1650 bool Matcher::is_spillable_arg(int reg)
 1651 {
 1652   return can_be_java_arg(reg);
 1653 }
 1654 
 1655 uint Matcher::int_pressure_limit()
 1656 {
 1657   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1658 }
 1659 
 1660 uint Matcher::float_pressure_limit()
 1661 {
 1662   // After experiment around with different values, the following default threshold
 1663   // works best for LCM's register pressure scheduling on x64.
 1664   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1665   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1666   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1667 }
 1668 
 1669 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1670   // In 64 bit mode a code which use multiply when
 1671   // devisor is constant is faster than hardware
 1672   // DIV instruction (it uses MulHiL).
 1673   return false;
 1674 }
 1675 
 1676 // Register for DIVI projection of divmodI
 1677 RegMask Matcher::divI_proj_mask() {
 1678   return INT_RAX_REG_mask();
 1679 }
 1680 
 1681 // Register for MODI projection of divmodI
 1682 RegMask Matcher::modI_proj_mask() {
 1683   return INT_RDX_REG_mask();
 1684 }
 1685 
 1686 // Register for DIVL projection of divmodL
 1687 RegMask Matcher::divL_proj_mask() {
 1688   return LONG_RAX_REG_mask();
 1689 }
 1690 
 1691 // Register for MODL projection of divmodL
 1692 RegMask Matcher::modL_proj_mask() {
 1693   return LONG_RDX_REG_mask();
 1694 }
 1695 
 1696 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1697 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1698     return NO_REG_mask();
 1699 }
 1700 
 1701 %}
 1702 
 1703 //----------ENCODING BLOCK-----------------------------------------------------
 1704 // This block specifies the encoding classes used by the compiler to
 1705 // output byte streams.  Encoding classes are parameterized macros
 1706 // used by Machine Instruction Nodes in order to generate the bit
 1707 // encoding of the instruction.  Operands specify their base encoding
 1708 // interface with the interface keyword.  There are currently
 1709 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1710 // COND_INTER.  REG_INTER causes an operand to generate a function
 1711 // which returns its register number when queried.  CONST_INTER causes
 1712 // an operand to generate a function which returns the value of the
 1713 // constant when queried.  MEMORY_INTER causes an operand to generate
 1714 // four functions which return the Base Register, the Index Register,
 1715 // the Scale Value, and the Offset Value of the operand when queried.
 1716 // COND_INTER causes an operand to generate six functions which return
 1717 // the encoding code (ie - encoding bits for the instruction)
 1718 // associated with each basic boolean condition for a conditional
 1719 // instruction.
 1720 //
 1721 // Instructions specify two basic values for encoding.  Again, a
 1722 // function is available to check if the constant displacement is an
 1723 // oop. They use the ins_encode keyword to specify their encoding
 1724 // classes (which must be a sequence of enc_class names, and their
 1725 // parameters, specified in the encoding block), and they use the
 1726 // opcode keyword to specify, in order, their primary, secondary, and
 1727 // tertiary opcode.  Only the opcode sections which a particular
 1728 // instruction needs for encoding need to be specified.
 1729 encode %{
 1730   enc_class cdql_enc(no_rax_rdx_RegI div)
 1731   %{
 1732     // Full implementation of Java idiv and irem; checks for
 1733     // special case as described in JVM spec., p.243 & p.271.
 1734     //
 1735     //         normal case                           special case
 1736     //
 1737     // input : rax: dividend                         min_int
 1738     //         reg: divisor                          -1
 1739     //
 1740     // output: rax: quotient  (= rax idiv reg)       min_int
 1741     //         rdx: remainder (= rax irem reg)       0
 1742     //
 1743     //  Code sequnce:
 1744     //
 1745     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1746     //    5:   75 07/08                jne    e <normal>
 1747     //    7:   33 d2                   xor    %edx,%edx
 1748     //  [div >= 8 -> offset + 1]
 1749     //  [REX_B]
 1750     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1751     //    c:   74 03/04                je     11 <done>
 1752     // 000000000000000e <normal>:
 1753     //    e:   99                      cltd
 1754     //  [div >= 8 -> offset + 1]
 1755     //  [REX_B]
 1756     //    f:   f7 f9                   idiv   $div
 1757     // 0000000000000011 <done>:
 1758     Label normal;
 1759     Label done;
 1760 
 1761     // cmp    $0x80000000,%eax
 1762     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1763 
 1764     // jne    e <normal>
 1765     __ jccb(Assembler::notEqual, normal);
 1766 
 1767     // xor    %edx,%edx
 1768     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1769 
 1770     // cmp    $0xffffffffffffffff,%ecx
 1771     __ cmpl($div$$Register, -1);
 1772 
 1773     // je     11 <done>
 1774     __ jccb(Assembler::equal, done);
 1775 
 1776     // <normal>
 1777     // cltd
 1778     __ bind(normal);
 1779     __ cdql();
 1780 
 1781     // idivl
 1782     // <done>
 1783     __ idivl($div$$Register);
 1784     __ bind(done);
 1785   %}
 1786 
 1787   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1788   %{
 1789     // Full implementation of Java ldiv and lrem; checks for
 1790     // special case as described in JVM spec., p.243 & p.271.
 1791     //
 1792     //         normal case                           special case
 1793     //
 1794     // input : rax: dividend                         min_long
 1795     //         reg: divisor                          -1
 1796     //
 1797     // output: rax: quotient  (= rax idiv reg)       min_long
 1798     //         rdx: remainder (= rax irem reg)       0
 1799     //
 1800     //  Code sequnce:
 1801     //
 1802     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1803     //    7:   00 00 80
 1804     //    a:   48 39 d0                cmp    %rdx,%rax
 1805     //    d:   75 08                   jne    17 <normal>
 1806     //    f:   33 d2                   xor    %edx,%edx
 1807     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1808     //   15:   74 05                   je     1c <done>
 1809     // 0000000000000017 <normal>:
 1810     //   17:   48 99                   cqto
 1811     //   19:   48 f7 f9                idiv   $div
 1812     // 000000000000001c <done>:
 1813     Label normal;
 1814     Label done;
 1815 
 1816     // mov    $0x8000000000000000,%rdx
 1817     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1818 
 1819     // cmp    %rdx,%rax
 1820     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1821 
 1822     // jne    17 <normal>
 1823     __ jccb(Assembler::notEqual, normal);
 1824 
 1825     // xor    %edx,%edx
 1826     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1827 
 1828     // cmp    $0xffffffffffffffff,$div
 1829     __ cmpq($div$$Register, -1);
 1830 
 1831     // je     1e <done>
 1832     __ jccb(Assembler::equal, done);
 1833 
 1834     // <normal>
 1835     // cqto
 1836     __ bind(normal);
 1837     __ cdqq();
 1838 
 1839     // idivq (note: must be emitted by the user of this rule)
 1840     // <done>
 1841     __ idivq($div$$Register);
 1842     __ bind(done);
 1843   %}
 1844 
 1845   enc_class clear_avx %{
 1846     debug_only(int off0 = __ offset());
 1847     if (generate_vzeroupper(Compile::current())) {
 1848       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1849       // Clear upper bits of YMM registers when current compiled code uses
 1850       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1851       __ vzeroupper();
 1852     }
 1853     debug_only(int off1 = __ offset());
 1854     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1855   %}
 1856 
 1857   enc_class Java_To_Runtime(method meth) %{
 1858     // No relocation needed
 1859     __ mov64(r10, (int64_t) $meth$$method);
 1860     __ call(r10);
 1861     __ post_call_nop();
 1862   %}
 1863 
 1864   enc_class Java_Static_Call(method meth)
 1865   %{
 1866     // JAVA STATIC CALL
 1867     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1868     // determine who we intended to call.
 1869     if (!_method) {
 1870       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1871     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1872       // The NOP here is purely to ensure that eliding a call to
 1873       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1874       __ addr_nop_5();
 1875       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1876     } else {
 1877       int method_index = resolved_method_index(masm);
 1878       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1879                                                   : static_call_Relocation::spec(method_index);
 1880       address mark = __ pc();
 1881       int call_offset = __ offset();
 1882       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1883       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1884         // Calls of the same statically bound method can share
 1885         // a stub to the interpreter.
 1886         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1887       } else {
 1888         // Emit stubs for static call.
 1889         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1890         __ clear_inst_mark();
 1891         if (stub == nullptr) {
 1892           ciEnv::current()->record_failure("CodeCache is full");
 1893           return;
 1894         }
 1895       }
 1896     }
 1897     __ post_call_nop();
 1898   %}
 1899 
 1900   enc_class Java_Dynamic_Call(method meth) %{
 1901     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1902     __ post_call_nop();
 1903   %}
 1904 
 1905 %}
 1906 
 1907 
 1908 
 1909 //----------FRAME--------------------------------------------------------------
 1910 // Definition of frame structure and management information.
 1911 //
 1912 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1913 //                             |   (to get allocators register number
 1914 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1915 //  r   CALLER     |        |
 1916 //  o     |        +--------+      pad to even-align allocators stack-slot
 1917 //  w     V        |  pad0  |        numbers; owned by CALLER
 1918 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1919 //  h     ^        |   in   |  5
 1920 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1921 //  |     |        |        |  3
 1922 //  |     |        +--------+
 1923 //  V     |        | old out|      Empty on Intel, window on Sparc
 1924 //        |    old |preserve|      Must be even aligned.
 1925 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1926 //        |        |   in   |  3   area for Intel ret address
 1927 //     Owned by    |preserve|      Empty on Sparc.
 1928 //       SELF      +--------+
 1929 //        |        |  pad2  |  2   pad to align old SP
 1930 //        |        +--------+  1
 1931 //        |        | locks  |  0
 1932 //        |        +--------+----> OptoReg::stack0(), even aligned
 1933 //        |        |  pad1  | 11   pad to align new SP
 1934 //        |        +--------+
 1935 //        |        |        | 10
 1936 //        |        | spills |  9   spills
 1937 //        V        |        |  8   (pad0 slot for callee)
 1938 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1939 //        ^        |  out   |  7
 1940 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1941 //     Owned by    +--------+
 1942 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1943 //        |    new |preserve|      Must be even-aligned.
 1944 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1945 //        |        |        |
 1946 //
 1947 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1948 //         known from SELF's arguments and the Java calling convention.
 1949 //         Region 6-7 is determined per call site.
 1950 // Note 2: If the calling convention leaves holes in the incoming argument
 1951 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1952 //         are owned by the CALLEE.  Holes should not be necessary in the
 1953 //         incoming area, as the Java calling convention is completely under
 1954 //         the control of the AD file.  Doubles can be sorted and packed to
 1955 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1956 //         varargs C calling conventions.
 1957 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1958 //         even aligned with pad0 as needed.
 1959 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1960 //         region 6-11 is even aligned; it may be padded out more so that
 1961 //         the region from SP to FP meets the minimum stack alignment.
 1962 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1963 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1964 //         SP meets the minimum alignment.
 1965 
 1966 frame
 1967 %{
 1968   // These three registers define part of the calling convention
 1969   // between compiled code and the interpreter.
 1970   inline_cache_reg(RAX);                // Inline Cache Register
 1971 
 1972   // Optional: name the operand used by cisc-spilling to access
 1973   // [stack_pointer + offset]
 1974   cisc_spilling_operand_name(indOffset32);
 1975 
 1976   // Number of stack slots consumed by locking an object
 1977   sync_stack_slots(2);
 1978 
 1979   // Compiled code's Frame Pointer
 1980   frame_pointer(RSP);
 1981 
 1982   // Interpreter stores its frame pointer in a register which is
 1983   // stored to the stack by I2CAdaptors.
 1984   // I2CAdaptors convert from interpreted java to compiled java.
 1985   interpreter_frame_pointer(RBP);
 1986 
 1987   // Stack alignment requirement
 1988   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1989 
 1990   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1991   // for calls to C.  Supports the var-args backing area for register parms.
 1992   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1993 
 1994   // The after-PROLOG location of the return address.  Location of
 1995   // return address specifies a type (REG or STACK) and a number
 1996   // representing the register number (i.e. - use a register name) or
 1997   // stack slot.
 1998   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1999   // Otherwise, it is above the locks and verification slot and alignment word
 2000   return_addr(STACK - 2 +
 2001               align_up((Compile::current()->in_preserve_stack_slots() +
 2002                         Compile::current()->fixed_slots()),
 2003                        stack_alignment_in_slots()));
 2004 
 2005   // Location of compiled Java return values.  Same as C for now.
 2006   return_value
 2007   %{
 2008     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2009            "only return normal values");
 2010 
 2011     static const int lo[Op_RegL + 1] = {
 2012       0,
 2013       0,
 2014       RAX_num,  // Op_RegN
 2015       RAX_num,  // Op_RegI
 2016       RAX_num,  // Op_RegP
 2017       XMM0_num, // Op_RegF
 2018       XMM0_num, // Op_RegD
 2019       RAX_num   // Op_RegL
 2020     };
 2021     static const int hi[Op_RegL + 1] = {
 2022       0,
 2023       0,
 2024       OptoReg::Bad, // Op_RegN
 2025       OptoReg::Bad, // Op_RegI
 2026       RAX_H_num,    // Op_RegP
 2027       OptoReg::Bad, // Op_RegF
 2028       XMM0b_num,    // Op_RegD
 2029       RAX_H_num     // Op_RegL
 2030     };
 2031     // Excluded flags and vector registers.
 2032     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2033     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2034   %}
 2035 %}
 2036 
 2037 //----------ATTRIBUTES---------------------------------------------------------
 2038 //----------Operand Attributes-------------------------------------------------
 2039 op_attrib op_cost(0);        // Required cost attribute
 2040 
 2041 //----------Instruction Attributes---------------------------------------------
 2042 ins_attrib ins_cost(100);       // Required cost attribute
 2043 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2044 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2045                                 // a non-matching short branch variant
 2046                                 // of some long branch?
 2047 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2048                                 // be a power of 2) specifies the
 2049                                 // alignment that some part of the
 2050                                 // instruction (not necessarily the
 2051                                 // start) requires.  If > 1, a
 2052                                 // compute_padding() function must be
 2053                                 // provided for the instruction
 2054 
 2055 //----------OPERANDS-----------------------------------------------------------
 2056 // Operand definitions must precede instruction definitions for correct parsing
 2057 // in the ADLC because operands constitute user defined types which are used in
 2058 // instruction definitions.
 2059 
 2060 //----------Simple Operands----------------------------------------------------
 2061 // Immediate Operands
 2062 // Integer Immediate
 2063 operand immI()
 2064 %{
 2065   match(ConI);
 2066 
 2067   op_cost(10);
 2068   format %{ %}
 2069   interface(CONST_INTER);
 2070 %}
 2071 
 2072 // Constant for test vs zero
 2073 operand immI_0()
 2074 %{
 2075   predicate(n->get_int() == 0);
 2076   match(ConI);
 2077 
 2078   op_cost(0);
 2079   format %{ %}
 2080   interface(CONST_INTER);
 2081 %}
 2082 
 2083 // Constant for increment
 2084 operand immI_1()
 2085 %{
 2086   predicate(n->get_int() == 1);
 2087   match(ConI);
 2088 
 2089   op_cost(0);
 2090   format %{ %}
 2091   interface(CONST_INTER);
 2092 %}
 2093 
 2094 // Constant for decrement
 2095 operand immI_M1()
 2096 %{
 2097   predicate(n->get_int() == -1);
 2098   match(ConI);
 2099 
 2100   op_cost(0);
 2101   format %{ %}
 2102   interface(CONST_INTER);
 2103 %}
 2104 
 2105 operand immI_2()
 2106 %{
 2107   predicate(n->get_int() == 2);
 2108   match(ConI);
 2109 
 2110   op_cost(0);
 2111   format %{ %}
 2112   interface(CONST_INTER);
 2113 %}
 2114 
 2115 operand immI_4()
 2116 %{
 2117   predicate(n->get_int() == 4);
 2118   match(ConI);
 2119 
 2120   op_cost(0);
 2121   format %{ %}
 2122   interface(CONST_INTER);
 2123 %}
 2124 
 2125 operand immI_8()
 2126 %{
 2127   predicate(n->get_int() == 8);
 2128   match(ConI);
 2129 
 2130   op_cost(0);
 2131   format %{ %}
 2132   interface(CONST_INTER);
 2133 %}
 2134 
 2135 // Valid scale values for addressing modes
 2136 operand immI2()
 2137 %{
 2138   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2139   match(ConI);
 2140 
 2141   format %{ %}
 2142   interface(CONST_INTER);
 2143 %}
 2144 
 2145 operand immU7()
 2146 %{
 2147   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2148   match(ConI);
 2149 
 2150   op_cost(5);
 2151   format %{ %}
 2152   interface(CONST_INTER);
 2153 %}
 2154 
 2155 operand immI8()
 2156 %{
 2157   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2158   match(ConI);
 2159 
 2160   op_cost(5);
 2161   format %{ %}
 2162   interface(CONST_INTER);
 2163 %}
 2164 
 2165 operand immU8()
 2166 %{
 2167   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2168   match(ConI);
 2169 
 2170   op_cost(5);
 2171   format %{ %}
 2172   interface(CONST_INTER);
 2173 %}
 2174 
 2175 operand immI16()
 2176 %{
 2177   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2178   match(ConI);
 2179 
 2180   op_cost(10);
 2181   format %{ %}
 2182   interface(CONST_INTER);
 2183 %}
 2184 
 2185 // Int Immediate non-negative
 2186 operand immU31()
 2187 %{
 2188   predicate(n->get_int() >= 0);
 2189   match(ConI);
 2190 
 2191   op_cost(0);
 2192   format %{ %}
 2193   interface(CONST_INTER);
 2194 %}
 2195 
 2196 // Pointer Immediate
 2197 operand immP()
 2198 %{
 2199   match(ConP);
 2200 
 2201   op_cost(10);
 2202   format %{ %}
 2203   interface(CONST_INTER);
 2204 %}
 2205 
 2206 // Null Pointer Immediate
 2207 operand immP0()
 2208 %{
 2209   predicate(n->get_ptr() == 0);
 2210   match(ConP);
 2211 
 2212   op_cost(5);
 2213   format %{ %}
 2214   interface(CONST_INTER);
 2215 %}
 2216 
 2217 // Pointer Immediate
 2218 operand immN() %{
 2219   match(ConN);
 2220 
 2221   op_cost(10);
 2222   format %{ %}
 2223   interface(CONST_INTER);
 2224 %}
 2225 
 2226 operand immNKlass() %{
 2227   match(ConNKlass);
 2228 
 2229   op_cost(10);
 2230   format %{ %}
 2231   interface(CONST_INTER);
 2232 %}
 2233 
 2234 // Null Pointer Immediate
 2235 operand immN0() %{
 2236   predicate(n->get_narrowcon() == 0);
 2237   match(ConN);
 2238 
 2239   op_cost(5);
 2240   format %{ %}
 2241   interface(CONST_INTER);
 2242 %}
 2243 
 2244 operand immP31()
 2245 %{
 2246   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2247             && (n->get_ptr() >> 31) == 0);
 2248   match(ConP);
 2249 
 2250   op_cost(5);
 2251   format %{ %}
 2252   interface(CONST_INTER);
 2253 %}
 2254 
 2255 
 2256 // Long Immediate
 2257 operand immL()
 2258 %{
 2259   match(ConL);
 2260 
 2261   op_cost(20);
 2262   format %{ %}
 2263   interface(CONST_INTER);
 2264 %}
 2265 
 2266 // Long Immediate 8-bit
 2267 operand immL8()
 2268 %{
 2269   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2270   match(ConL);
 2271 
 2272   op_cost(5);
 2273   format %{ %}
 2274   interface(CONST_INTER);
 2275 %}
 2276 
 2277 // Long Immediate 32-bit unsigned
 2278 operand immUL32()
 2279 %{
 2280   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2281   match(ConL);
 2282 
 2283   op_cost(10);
 2284   format %{ %}
 2285   interface(CONST_INTER);
 2286 %}
 2287 
 2288 // Long Immediate 32-bit signed
 2289 operand immL32()
 2290 %{
 2291   predicate(n->get_long() == (int) (n->get_long()));
 2292   match(ConL);
 2293 
 2294   op_cost(15);
 2295   format %{ %}
 2296   interface(CONST_INTER);
 2297 %}
 2298 
 2299 operand immL_Pow2()
 2300 %{
 2301   predicate(is_power_of_2((julong)n->get_long()));
 2302   match(ConL);
 2303 
 2304   op_cost(15);
 2305   format %{ %}
 2306   interface(CONST_INTER);
 2307 %}
 2308 
 2309 operand immL_NotPow2()
 2310 %{
 2311   predicate(is_power_of_2((julong)~n->get_long()));
 2312   match(ConL);
 2313 
 2314   op_cost(15);
 2315   format %{ %}
 2316   interface(CONST_INTER);
 2317 %}
 2318 
 2319 // Long Immediate zero
 2320 operand immL0()
 2321 %{
 2322   predicate(n->get_long() == 0L);
 2323   match(ConL);
 2324 
 2325   op_cost(10);
 2326   format %{ %}
 2327   interface(CONST_INTER);
 2328 %}
 2329 
 2330 // Constant for increment
 2331 operand immL1()
 2332 %{
 2333   predicate(n->get_long() == 1);
 2334   match(ConL);
 2335 
 2336   format %{ %}
 2337   interface(CONST_INTER);
 2338 %}
 2339 
 2340 // Constant for decrement
 2341 operand immL_M1()
 2342 %{
 2343   predicate(n->get_long() == -1);
 2344   match(ConL);
 2345 
 2346   format %{ %}
 2347   interface(CONST_INTER);
 2348 %}
 2349 
 2350 // Long Immediate: low 32-bit mask
 2351 operand immL_32bits()
 2352 %{
 2353   predicate(n->get_long() == 0xFFFFFFFFL);
 2354   match(ConL);
 2355   op_cost(20);
 2356 
 2357   format %{ %}
 2358   interface(CONST_INTER);
 2359 %}
 2360 
 2361 // Int Immediate: 2^n-1, positive
 2362 operand immI_Pow2M1()
 2363 %{
 2364   predicate((n->get_int() > 0)
 2365             && is_power_of_2((juint)n->get_int() + 1));
 2366   match(ConI);
 2367 
 2368   op_cost(20);
 2369   format %{ %}
 2370   interface(CONST_INTER);
 2371 %}
 2372 
 2373 // Float Immediate zero
 2374 operand immF0()
 2375 %{
 2376   predicate(jint_cast(n->getf()) == 0);
 2377   match(ConF);
 2378 
 2379   op_cost(5);
 2380   format %{ %}
 2381   interface(CONST_INTER);
 2382 %}
 2383 
 2384 // Float Immediate
 2385 operand immF()
 2386 %{
 2387   match(ConF);
 2388 
 2389   op_cost(15);
 2390   format %{ %}
 2391   interface(CONST_INTER);
 2392 %}
 2393 
 2394 // Half Float Immediate
 2395 operand immH()
 2396 %{
 2397   match(ConH);
 2398 
 2399   op_cost(15);
 2400   format %{ %}
 2401   interface(CONST_INTER);
 2402 %}
 2403 
 2404 // Double Immediate zero
 2405 operand immD0()
 2406 %{
 2407   predicate(jlong_cast(n->getd()) == 0);
 2408   match(ConD);
 2409 
 2410   op_cost(5);
 2411   format %{ %}
 2412   interface(CONST_INTER);
 2413 %}
 2414 
 2415 // Double Immediate
 2416 operand immD()
 2417 %{
 2418   match(ConD);
 2419 
 2420   op_cost(15);
 2421   format %{ %}
 2422   interface(CONST_INTER);
 2423 %}
 2424 
 2425 // Immediates for special shifts (sign extend)
 2426 
 2427 // Constants for increment
 2428 operand immI_16()
 2429 %{
 2430   predicate(n->get_int() == 16);
 2431   match(ConI);
 2432 
 2433   format %{ %}
 2434   interface(CONST_INTER);
 2435 %}
 2436 
 2437 operand immI_24()
 2438 %{
 2439   predicate(n->get_int() == 24);
 2440   match(ConI);
 2441 
 2442   format %{ %}
 2443   interface(CONST_INTER);
 2444 %}
 2445 
 2446 // Constant for byte-wide masking
 2447 operand immI_255()
 2448 %{
 2449   predicate(n->get_int() == 255);
 2450   match(ConI);
 2451 
 2452   format %{ %}
 2453   interface(CONST_INTER);
 2454 %}
 2455 
 2456 // Constant for short-wide masking
 2457 operand immI_65535()
 2458 %{
 2459   predicate(n->get_int() == 65535);
 2460   match(ConI);
 2461 
 2462   format %{ %}
 2463   interface(CONST_INTER);
 2464 %}
 2465 
 2466 // Constant for byte-wide masking
 2467 operand immL_255()
 2468 %{
 2469   predicate(n->get_long() == 255);
 2470   match(ConL);
 2471 
 2472   format %{ %}
 2473   interface(CONST_INTER);
 2474 %}
 2475 
 2476 // Constant for short-wide masking
 2477 operand immL_65535()
 2478 %{
 2479   predicate(n->get_long() == 65535);
 2480   match(ConL);
 2481 
 2482   format %{ %}
 2483   interface(CONST_INTER);
 2484 %}
 2485 
 2486 operand kReg()
 2487 %{
 2488   constraint(ALLOC_IN_RC(vectmask_reg));
 2489   match(RegVectMask);
 2490   format %{%}
 2491   interface(REG_INTER);
 2492 %}
 2493 
 2494 // Register Operands
 2495 // Integer Register
 2496 operand rRegI()
 2497 %{
 2498   constraint(ALLOC_IN_RC(int_reg));
 2499   match(RegI);
 2500 
 2501   match(rax_RegI);
 2502   match(rbx_RegI);
 2503   match(rcx_RegI);
 2504   match(rdx_RegI);
 2505   match(rdi_RegI);
 2506 
 2507   format %{ %}
 2508   interface(REG_INTER);
 2509 %}
 2510 
 2511 // Special Registers
 2512 operand rax_RegI()
 2513 %{
 2514   constraint(ALLOC_IN_RC(int_rax_reg));
 2515   match(RegI);
 2516   match(rRegI);
 2517 
 2518   format %{ "RAX" %}
 2519   interface(REG_INTER);
 2520 %}
 2521 
 2522 // Special Registers
 2523 operand rbx_RegI()
 2524 %{
 2525   constraint(ALLOC_IN_RC(int_rbx_reg));
 2526   match(RegI);
 2527   match(rRegI);
 2528 
 2529   format %{ "RBX" %}
 2530   interface(REG_INTER);
 2531 %}
 2532 
 2533 operand rcx_RegI()
 2534 %{
 2535   constraint(ALLOC_IN_RC(int_rcx_reg));
 2536   match(RegI);
 2537   match(rRegI);
 2538 
 2539   format %{ "RCX" %}
 2540   interface(REG_INTER);
 2541 %}
 2542 
 2543 operand rdx_RegI()
 2544 %{
 2545   constraint(ALLOC_IN_RC(int_rdx_reg));
 2546   match(RegI);
 2547   match(rRegI);
 2548 
 2549   format %{ "RDX" %}
 2550   interface(REG_INTER);
 2551 %}
 2552 
 2553 operand rdi_RegI()
 2554 %{
 2555   constraint(ALLOC_IN_RC(int_rdi_reg));
 2556   match(RegI);
 2557   match(rRegI);
 2558 
 2559   format %{ "RDI" %}
 2560   interface(REG_INTER);
 2561 %}
 2562 
 2563 operand no_rax_rdx_RegI()
 2564 %{
 2565   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2566   match(RegI);
 2567   match(rbx_RegI);
 2568   match(rcx_RegI);
 2569   match(rdi_RegI);
 2570 
 2571   format %{ %}
 2572   interface(REG_INTER);
 2573 %}
 2574 
 2575 operand no_rbp_r13_RegI()
 2576 %{
 2577   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2578   match(RegI);
 2579   match(rRegI);
 2580   match(rax_RegI);
 2581   match(rbx_RegI);
 2582   match(rcx_RegI);
 2583   match(rdx_RegI);
 2584   match(rdi_RegI);
 2585 
 2586   format %{ %}
 2587   interface(REG_INTER);
 2588 %}
 2589 
 2590 // Pointer Register
 2591 operand any_RegP()
 2592 %{
 2593   constraint(ALLOC_IN_RC(any_reg));
 2594   match(RegP);
 2595   match(rax_RegP);
 2596   match(rbx_RegP);
 2597   match(rdi_RegP);
 2598   match(rsi_RegP);
 2599   match(rbp_RegP);
 2600   match(r15_RegP);
 2601   match(rRegP);
 2602 
 2603   format %{ %}
 2604   interface(REG_INTER);
 2605 %}
 2606 
 2607 operand rRegP()
 2608 %{
 2609   constraint(ALLOC_IN_RC(ptr_reg));
 2610   match(RegP);
 2611   match(rax_RegP);
 2612   match(rbx_RegP);
 2613   match(rdi_RegP);
 2614   match(rsi_RegP);
 2615   match(rbp_RegP);  // See Q&A below about
 2616   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2617 
 2618   format %{ %}
 2619   interface(REG_INTER);
 2620 %}
 2621 
 2622 operand rRegN() %{
 2623   constraint(ALLOC_IN_RC(int_reg));
 2624   match(RegN);
 2625 
 2626   format %{ %}
 2627   interface(REG_INTER);
 2628 %}
 2629 
 2630 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2631 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2632 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2633 // The output of an instruction is controlled by the allocator, which respects
 2634 // register class masks, not match rules.  Unless an instruction mentions
 2635 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2636 // by the allocator as an input.
 2637 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2638 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2639 // result, RBP is not included in the output of the instruction either.
 2640 
 2641 // This operand is not allowed to use RBP even if
 2642 // RBP is not used to hold the frame pointer.
 2643 operand no_rbp_RegP()
 2644 %{
 2645   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2646   match(RegP);
 2647   match(rbx_RegP);
 2648   match(rsi_RegP);
 2649   match(rdi_RegP);
 2650 
 2651   format %{ %}
 2652   interface(REG_INTER);
 2653 %}
 2654 
 2655 // Special Registers
 2656 // Return a pointer value
 2657 operand rax_RegP()
 2658 %{
 2659   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2660   match(RegP);
 2661   match(rRegP);
 2662 
 2663   format %{ %}
 2664   interface(REG_INTER);
 2665 %}
 2666 
 2667 // Special Registers
 2668 // Return a compressed pointer value
 2669 operand rax_RegN()
 2670 %{
 2671   constraint(ALLOC_IN_RC(int_rax_reg));
 2672   match(RegN);
 2673   match(rRegN);
 2674 
 2675   format %{ %}
 2676   interface(REG_INTER);
 2677 %}
 2678 
 2679 // Used in AtomicAdd
 2680 operand rbx_RegP()
 2681 %{
 2682   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2683   match(RegP);
 2684   match(rRegP);
 2685 
 2686   format %{ %}
 2687   interface(REG_INTER);
 2688 %}
 2689 
 2690 operand rsi_RegP()
 2691 %{
 2692   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2693   match(RegP);
 2694   match(rRegP);
 2695 
 2696   format %{ %}
 2697   interface(REG_INTER);
 2698 %}
 2699 
 2700 operand rbp_RegP()
 2701 %{
 2702   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2703   match(RegP);
 2704   match(rRegP);
 2705 
 2706   format %{ %}
 2707   interface(REG_INTER);
 2708 %}
 2709 
 2710 // Used in rep stosq
 2711 operand rdi_RegP()
 2712 %{
 2713   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2714   match(RegP);
 2715   match(rRegP);
 2716 
 2717   format %{ %}
 2718   interface(REG_INTER);
 2719 %}
 2720 
 2721 operand r15_RegP()
 2722 %{
 2723   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2724   match(RegP);
 2725   match(rRegP);
 2726 
 2727   format %{ %}
 2728   interface(REG_INTER);
 2729 %}
 2730 
 2731 operand rRegL()
 2732 %{
 2733   constraint(ALLOC_IN_RC(long_reg));
 2734   match(RegL);
 2735   match(rax_RegL);
 2736   match(rdx_RegL);
 2737 
 2738   format %{ %}
 2739   interface(REG_INTER);
 2740 %}
 2741 
 2742 // Special Registers
 2743 operand no_rax_rdx_RegL()
 2744 %{
 2745   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2746   match(RegL);
 2747   match(rRegL);
 2748 
 2749   format %{ %}
 2750   interface(REG_INTER);
 2751 %}
 2752 
 2753 operand rax_RegL()
 2754 %{
 2755   constraint(ALLOC_IN_RC(long_rax_reg));
 2756   match(RegL);
 2757   match(rRegL);
 2758 
 2759   format %{ "RAX" %}
 2760   interface(REG_INTER);
 2761 %}
 2762 
 2763 operand rcx_RegL()
 2764 %{
 2765   constraint(ALLOC_IN_RC(long_rcx_reg));
 2766   match(RegL);
 2767   match(rRegL);
 2768 
 2769   format %{ %}
 2770   interface(REG_INTER);
 2771 %}
 2772 
 2773 operand rdx_RegL()
 2774 %{
 2775   constraint(ALLOC_IN_RC(long_rdx_reg));
 2776   match(RegL);
 2777   match(rRegL);
 2778 
 2779   format %{ %}
 2780   interface(REG_INTER);
 2781 %}
 2782 
 2783 operand r11_RegL()
 2784 %{
 2785   constraint(ALLOC_IN_RC(long_r11_reg));
 2786   match(RegL);
 2787   match(rRegL);
 2788 
 2789   format %{ %}
 2790   interface(REG_INTER);
 2791 %}
 2792 
 2793 operand no_rbp_r13_RegL()
 2794 %{
 2795   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2796   match(RegL);
 2797   match(rRegL);
 2798   match(rax_RegL);
 2799   match(rcx_RegL);
 2800   match(rdx_RegL);
 2801 
 2802   format %{ %}
 2803   interface(REG_INTER);
 2804 %}
 2805 
 2806 // Flags register, used as output of compare instructions
 2807 operand rFlagsReg()
 2808 %{
 2809   constraint(ALLOC_IN_RC(int_flags));
 2810   match(RegFlags);
 2811 
 2812   format %{ "RFLAGS" %}
 2813   interface(REG_INTER);
 2814 %}
 2815 
 2816 // Flags register, used as output of FLOATING POINT compare instructions
 2817 operand rFlagsRegU()
 2818 %{
 2819   constraint(ALLOC_IN_RC(int_flags));
 2820   match(RegFlags);
 2821 
 2822   format %{ "RFLAGS_U" %}
 2823   interface(REG_INTER);
 2824 %}
 2825 
 2826 operand rFlagsRegUCF() %{
 2827   constraint(ALLOC_IN_RC(int_flags));
 2828   match(RegFlags);
 2829   predicate(false);
 2830 
 2831   format %{ "RFLAGS_U_CF" %}
 2832   interface(REG_INTER);
 2833 %}
 2834 
 2835 // Float register operands
 2836 operand regF() %{
 2837    constraint(ALLOC_IN_RC(float_reg));
 2838    match(RegF);
 2839 
 2840    format %{ %}
 2841    interface(REG_INTER);
 2842 %}
 2843 
 2844 // Float register operands
 2845 operand legRegF() %{
 2846    constraint(ALLOC_IN_RC(float_reg_legacy));
 2847    match(RegF);
 2848 
 2849    format %{ %}
 2850    interface(REG_INTER);
 2851 %}
 2852 
 2853 // Float register operands
 2854 operand vlRegF() %{
 2855    constraint(ALLOC_IN_RC(float_reg_vl));
 2856    match(RegF);
 2857 
 2858    format %{ %}
 2859    interface(REG_INTER);
 2860 %}
 2861 
 2862 // Double register operands
 2863 operand regD() %{
 2864    constraint(ALLOC_IN_RC(double_reg));
 2865    match(RegD);
 2866 
 2867    format %{ %}
 2868    interface(REG_INTER);
 2869 %}
 2870 
 2871 // Double register operands
 2872 operand legRegD() %{
 2873    constraint(ALLOC_IN_RC(double_reg_legacy));
 2874    match(RegD);
 2875 
 2876    format %{ %}
 2877    interface(REG_INTER);
 2878 %}
 2879 
 2880 // Double register operands
 2881 operand vlRegD() %{
 2882    constraint(ALLOC_IN_RC(double_reg_vl));
 2883    match(RegD);
 2884 
 2885    format %{ %}
 2886    interface(REG_INTER);
 2887 %}
 2888 
 2889 //----------Memory Operands----------------------------------------------------
 2890 // Direct Memory Operand
 2891 // operand direct(immP addr)
 2892 // %{
 2893 //   match(addr);
 2894 
 2895 //   format %{ "[$addr]" %}
 2896 //   interface(MEMORY_INTER) %{
 2897 //     base(0xFFFFFFFF);
 2898 //     index(0x4);
 2899 //     scale(0x0);
 2900 //     disp($addr);
 2901 //   %}
 2902 // %}
 2903 
 2904 // Indirect Memory Operand
 2905 operand indirect(any_RegP reg)
 2906 %{
 2907   constraint(ALLOC_IN_RC(ptr_reg));
 2908   match(reg);
 2909 
 2910   format %{ "[$reg]" %}
 2911   interface(MEMORY_INTER) %{
 2912     base($reg);
 2913     index(0x4);
 2914     scale(0x0);
 2915     disp(0x0);
 2916   %}
 2917 %}
 2918 
 2919 // Indirect Memory Plus Short Offset Operand
 2920 operand indOffset8(any_RegP reg, immL8 off)
 2921 %{
 2922   constraint(ALLOC_IN_RC(ptr_reg));
 2923   match(AddP reg off);
 2924 
 2925   format %{ "[$reg + $off (8-bit)]" %}
 2926   interface(MEMORY_INTER) %{
 2927     base($reg);
 2928     index(0x4);
 2929     scale(0x0);
 2930     disp($off);
 2931   %}
 2932 %}
 2933 
 2934 // Indirect Memory Plus Long Offset Operand
 2935 operand indOffset32(any_RegP reg, immL32 off)
 2936 %{
 2937   constraint(ALLOC_IN_RC(ptr_reg));
 2938   match(AddP reg off);
 2939 
 2940   format %{ "[$reg + $off (32-bit)]" %}
 2941   interface(MEMORY_INTER) %{
 2942     base($reg);
 2943     index(0x4);
 2944     scale(0x0);
 2945     disp($off);
 2946   %}
 2947 %}
 2948 
 2949 // Indirect Memory Plus Index Register Plus Offset Operand
 2950 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2951 %{
 2952   constraint(ALLOC_IN_RC(ptr_reg));
 2953   match(AddP (AddP reg lreg) off);
 2954 
 2955   op_cost(10);
 2956   format %{"[$reg + $off + $lreg]" %}
 2957   interface(MEMORY_INTER) %{
 2958     base($reg);
 2959     index($lreg);
 2960     scale(0x0);
 2961     disp($off);
 2962   %}
 2963 %}
 2964 
 2965 // Indirect Memory Plus Index Register Plus Offset Operand
 2966 operand indIndex(any_RegP reg, rRegL lreg)
 2967 %{
 2968   constraint(ALLOC_IN_RC(ptr_reg));
 2969   match(AddP reg lreg);
 2970 
 2971   op_cost(10);
 2972   format %{"[$reg + $lreg]" %}
 2973   interface(MEMORY_INTER) %{
 2974     base($reg);
 2975     index($lreg);
 2976     scale(0x0);
 2977     disp(0x0);
 2978   %}
 2979 %}
 2980 
 2981 // Indirect Memory Times Scale Plus Index Register
 2982 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2983 %{
 2984   constraint(ALLOC_IN_RC(ptr_reg));
 2985   match(AddP reg (LShiftL lreg scale));
 2986 
 2987   op_cost(10);
 2988   format %{"[$reg + $lreg << $scale]" %}
 2989   interface(MEMORY_INTER) %{
 2990     base($reg);
 2991     index($lreg);
 2992     scale($scale);
 2993     disp(0x0);
 2994   %}
 2995 %}
 2996 
 2997 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 2998 %{
 2999   constraint(ALLOC_IN_RC(ptr_reg));
 3000   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3001   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3002 
 3003   op_cost(10);
 3004   format %{"[$reg + pos $idx << $scale]" %}
 3005   interface(MEMORY_INTER) %{
 3006     base($reg);
 3007     index($idx);
 3008     scale($scale);
 3009     disp(0x0);
 3010   %}
 3011 %}
 3012 
 3013 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3014 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3015 %{
 3016   constraint(ALLOC_IN_RC(ptr_reg));
 3017   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3018 
 3019   op_cost(10);
 3020   format %{"[$reg + $off + $lreg << $scale]" %}
 3021   interface(MEMORY_INTER) %{
 3022     base($reg);
 3023     index($lreg);
 3024     scale($scale);
 3025     disp($off);
 3026   %}
 3027 %}
 3028 
 3029 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3030 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3031 %{
 3032   constraint(ALLOC_IN_RC(ptr_reg));
 3033   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3034   match(AddP (AddP reg (ConvI2L idx)) off);
 3035 
 3036   op_cost(10);
 3037   format %{"[$reg + $off + $idx]" %}
 3038   interface(MEMORY_INTER) %{
 3039     base($reg);
 3040     index($idx);
 3041     scale(0x0);
 3042     disp($off);
 3043   %}
 3044 %}
 3045 
 3046 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3047 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3048 %{
 3049   constraint(ALLOC_IN_RC(ptr_reg));
 3050   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3051   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3052 
 3053   op_cost(10);
 3054   format %{"[$reg + $off + $idx << $scale]" %}
 3055   interface(MEMORY_INTER) %{
 3056     base($reg);
 3057     index($idx);
 3058     scale($scale);
 3059     disp($off);
 3060   %}
 3061 %}
 3062 
 3063 // Indirect Narrow Oop Operand
 3064 operand indCompressedOop(rRegN reg) %{
 3065   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3066   constraint(ALLOC_IN_RC(ptr_reg));
 3067   match(DecodeN reg);
 3068 
 3069   op_cost(10);
 3070   format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
 3071   interface(MEMORY_INTER) %{
 3072     base(0xc); // R12
 3073     index($reg);
 3074     scale(0x3);
 3075     disp(0x0);
 3076   %}
 3077 %}
 3078 
 3079 // Indirect Narrow Oop Plus Offset Operand
 3080 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3081 // we can't free r12 even with CompressedOops::base() == nullptr.
 3082 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3083   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3084   constraint(ALLOC_IN_RC(ptr_reg));
 3085   match(AddP (DecodeN reg) off);
 3086 
 3087   op_cost(10);
 3088   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3089   interface(MEMORY_INTER) %{
 3090     base(0xc); // R12
 3091     index($reg);
 3092     scale(0x3);
 3093     disp($off);
 3094   %}
 3095 %}
 3096 
 3097 // Indirect Memory Operand
 3098 operand indirectNarrow(rRegN reg)
 3099 %{
 3100   predicate(CompressedOops::shift() == 0);
 3101   constraint(ALLOC_IN_RC(ptr_reg));
 3102   match(DecodeN reg);
 3103 
 3104   format %{ "[$reg]" %}
 3105   interface(MEMORY_INTER) %{
 3106     base($reg);
 3107     index(0x4);
 3108     scale(0x0);
 3109     disp(0x0);
 3110   %}
 3111 %}
 3112 
 3113 // Indirect Memory Plus Short Offset Operand
 3114 operand indOffset8Narrow(rRegN reg, immL8 off)
 3115 %{
 3116   predicate(CompressedOops::shift() == 0);
 3117   constraint(ALLOC_IN_RC(ptr_reg));
 3118   match(AddP (DecodeN reg) off);
 3119 
 3120   format %{ "[$reg + $off (8-bit)]" %}
 3121   interface(MEMORY_INTER) %{
 3122     base($reg);
 3123     index(0x4);
 3124     scale(0x0);
 3125     disp($off);
 3126   %}
 3127 %}
 3128 
 3129 // Indirect Memory Plus Long Offset Operand
 3130 operand indOffset32Narrow(rRegN reg, immL32 off)
 3131 %{
 3132   predicate(CompressedOops::shift() == 0);
 3133   constraint(ALLOC_IN_RC(ptr_reg));
 3134   match(AddP (DecodeN reg) off);
 3135 
 3136   format %{ "[$reg + $off (32-bit)]" %}
 3137   interface(MEMORY_INTER) %{
 3138     base($reg);
 3139     index(0x4);
 3140     scale(0x0);
 3141     disp($off);
 3142   %}
 3143 %}
 3144 
 3145 // Indirect Memory Plus Index Register Plus Offset Operand
 3146 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3147 %{
 3148   predicate(CompressedOops::shift() == 0);
 3149   constraint(ALLOC_IN_RC(ptr_reg));
 3150   match(AddP (AddP (DecodeN reg) lreg) off);
 3151 
 3152   op_cost(10);
 3153   format %{"[$reg + $off + $lreg]" %}
 3154   interface(MEMORY_INTER) %{
 3155     base($reg);
 3156     index($lreg);
 3157     scale(0x0);
 3158     disp($off);
 3159   %}
 3160 %}
 3161 
 3162 // Indirect Memory Plus Index Register Plus Offset Operand
 3163 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3164 %{
 3165   predicate(CompressedOops::shift() == 0);
 3166   constraint(ALLOC_IN_RC(ptr_reg));
 3167   match(AddP (DecodeN reg) lreg);
 3168 
 3169   op_cost(10);
 3170   format %{"[$reg + $lreg]" %}
 3171   interface(MEMORY_INTER) %{
 3172     base($reg);
 3173     index($lreg);
 3174     scale(0x0);
 3175     disp(0x0);
 3176   %}
 3177 %}
 3178 
 3179 // Indirect Memory Times Scale Plus Index Register
 3180 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3181 %{
 3182   predicate(CompressedOops::shift() == 0);
 3183   constraint(ALLOC_IN_RC(ptr_reg));
 3184   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3185 
 3186   op_cost(10);
 3187   format %{"[$reg + $lreg << $scale]" %}
 3188   interface(MEMORY_INTER) %{
 3189     base($reg);
 3190     index($lreg);
 3191     scale($scale);
 3192     disp(0x0);
 3193   %}
 3194 %}
 3195 
 3196 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3197 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3198 %{
 3199   predicate(CompressedOops::shift() == 0);
 3200   constraint(ALLOC_IN_RC(ptr_reg));
 3201   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3202 
 3203   op_cost(10);
 3204   format %{"[$reg + $off + $lreg << $scale]" %}
 3205   interface(MEMORY_INTER) %{
 3206     base($reg);
 3207     index($lreg);
 3208     scale($scale);
 3209     disp($off);
 3210   %}
 3211 %}
 3212 
 3213 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3214 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3215 %{
 3216   constraint(ALLOC_IN_RC(ptr_reg));
 3217   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3218   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3219 
 3220   op_cost(10);
 3221   format %{"[$reg + $off + $idx]" %}
 3222   interface(MEMORY_INTER) %{
 3223     base($reg);
 3224     index($idx);
 3225     scale(0x0);
 3226     disp($off);
 3227   %}
 3228 %}
 3229 
 3230 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3231 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3232 %{
 3233   constraint(ALLOC_IN_RC(ptr_reg));
 3234   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3235   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3236 
 3237   op_cost(10);
 3238   format %{"[$reg + $off + $idx << $scale]" %}
 3239   interface(MEMORY_INTER) %{
 3240     base($reg);
 3241     index($idx);
 3242     scale($scale);
 3243     disp($off);
 3244   %}
 3245 %}
 3246 
 3247 //----------Special Memory Operands--------------------------------------------
 3248 // Stack Slot Operand - This operand is used for loading and storing temporary
 3249 //                      values on the stack where a match requires a value to
 3250 //                      flow through memory.
 3251 operand stackSlotP(sRegP reg)
 3252 %{
 3253   constraint(ALLOC_IN_RC(stack_slots));
 3254   // No match rule because this operand is only generated in matching
 3255 
 3256   format %{ "[$reg]" %}
 3257   interface(MEMORY_INTER) %{
 3258     base(0x4);   // RSP
 3259     index(0x4);  // No Index
 3260     scale(0x0);  // No Scale
 3261     disp($reg);  // Stack Offset
 3262   %}
 3263 %}
 3264 
 3265 operand stackSlotI(sRegI reg)
 3266 %{
 3267   constraint(ALLOC_IN_RC(stack_slots));
 3268   // No match rule because this operand is only generated in matching
 3269 
 3270   format %{ "[$reg]" %}
 3271   interface(MEMORY_INTER) %{
 3272     base(0x4);   // RSP
 3273     index(0x4);  // No Index
 3274     scale(0x0);  // No Scale
 3275     disp($reg);  // Stack Offset
 3276   %}
 3277 %}
 3278 
 3279 operand stackSlotF(sRegF reg)
 3280 %{
 3281   constraint(ALLOC_IN_RC(stack_slots));
 3282   // No match rule because this operand is only generated in matching
 3283 
 3284   format %{ "[$reg]" %}
 3285   interface(MEMORY_INTER) %{
 3286     base(0x4);   // RSP
 3287     index(0x4);  // No Index
 3288     scale(0x0);  // No Scale
 3289     disp($reg);  // Stack Offset
 3290   %}
 3291 %}
 3292 
 3293 operand stackSlotD(sRegD reg)
 3294 %{
 3295   constraint(ALLOC_IN_RC(stack_slots));
 3296   // No match rule because this operand is only generated in matching
 3297 
 3298   format %{ "[$reg]" %}
 3299   interface(MEMORY_INTER) %{
 3300     base(0x4);   // RSP
 3301     index(0x4);  // No Index
 3302     scale(0x0);  // No Scale
 3303     disp($reg);  // Stack Offset
 3304   %}
 3305 %}
 3306 operand stackSlotL(sRegL reg)
 3307 %{
 3308   constraint(ALLOC_IN_RC(stack_slots));
 3309   // No match rule because this operand is only generated in matching
 3310 
 3311   format %{ "[$reg]" %}
 3312   interface(MEMORY_INTER) %{
 3313     base(0x4);   // RSP
 3314     index(0x4);  // No Index
 3315     scale(0x0);  // No Scale
 3316     disp($reg);  // Stack Offset
 3317   %}
 3318 %}
 3319 
 3320 //----------Conditional Branch Operands----------------------------------------
 3321 // Comparison Op  - This is the operation of the comparison, and is limited to
 3322 //                  the following set of codes:
 3323 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3324 //
 3325 // Other attributes of the comparison, such as unsignedness, are specified
 3326 // by the comparison instruction that sets a condition code flags register.
 3327 // That result is represented by a flags operand whose subtype is appropriate
 3328 // to the unsignedness (etc.) of the comparison.
 3329 //
 3330 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3331 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3332 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3333 
 3334 // Comparison Code
 3335 operand cmpOp()
 3336 %{
 3337   match(Bool);
 3338 
 3339   format %{ "" %}
 3340   interface(COND_INTER) %{
 3341     equal(0x4, "e");
 3342     not_equal(0x5, "ne");
 3343     less(0xC, "l");
 3344     greater_equal(0xD, "ge");
 3345     less_equal(0xE, "le");
 3346     greater(0xF, "g");
 3347     overflow(0x0, "o");
 3348     no_overflow(0x1, "no");
 3349   %}
 3350 %}
 3351 
 3352 // Comparison Code, unsigned compare.  Used by FP also, with
 3353 // C2 (unordered) turned into GT or LT already.  The other bits
 3354 // C0 and C3 are turned into Carry & Zero flags.
 3355 operand cmpOpU()
 3356 %{
 3357   match(Bool);
 3358 
 3359   format %{ "" %}
 3360   interface(COND_INTER) %{
 3361     equal(0x4, "e");
 3362     not_equal(0x5, "ne");
 3363     less(0x2, "b");
 3364     greater_equal(0x3, "ae");
 3365     less_equal(0x6, "be");
 3366     greater(0x7, "a");
 3367     overflow(0x0, "o");
 3368     no_overflow(0x1, "no");
 3369   %}
 3370 %}
 3371 
 3372 
 3373 // Floating comparisons that don't require any fixup for the unordered case,
 3374 // If both inputs of the comparison are the same, ZF is always set so we
 3375 // don't need to use cmpOpUCF2 for eq/ne
 3376 operand cmpOpUCF() %{
 3377   match(Bool);
 3378   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3379             n->as_Bool()->_test._test == BoolTest::ge ||
 3380             n->as_Bool()->_test._test == BoolTest::le ||
 3381             n->as_Bool()->_test._test == BoolTest::gt ||
 3382             n->in(1)->in(1) == n->in(1)->in(2));
 3383   format %{ "" %}
 3384   interface(COND_INTER) %{
 3385     equal(0xb, "np");
 3386     not_equal(0xa, "p");
 3387     less(0x2, "b");
 3388     greater_equal(0x3, "ae");
 3389     less_equal(0x6, "be");
 3390     greater(0x7, "a");
 3391     overflow(0x0, "o");
 3392     no_overflow(0x1, "no");
 3393   %}
 3394 %}
 3395 
 3396 
 3397 // Floating comparisons that can be fixed up with extra conditional jumps
 3398 operand cmpOpUCF2() %{
 3399   match(Bool);
 3400   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3401              n->as_Bool()->_test._test == BoolTest::eq) &&
 3402             n->in(1)->in(1) != n->in(1)->in(2));
 3403   format %{ "" %}
 3404   interface(COND_INTER) %{
 3405     equal(0x4, "e");
 3406     not_equal(0x5, "ne");
 3407     less(0x2, "b");
 3408     greater_equal(0x3, "ae");
 3409     less_equal(0x6, "be");
 3410     greater(0x7, "a");
 3411     overflow(0x0, "o");
 3412     no_overflow(0x1, "no");
 3413   %}
 3414 %}
 3415 
 3416 //----------OPERAND CLASSES----------------------------------------------------
 3417 // Operand Classes are groups of operands that are used as to simplify
 3418 // instruction definitions by not requiring the AD writer to specify separate
 3419 // instructions for every form of operand when the instruction accepts
 3420 // multiple operand types with the same basic encoding and format.  The classic
 3421 // case of this is memory operands.
 3422 
 3423 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3424                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3425                indCompressedOop, indCompressedOopOffset,
 3426                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3427                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3428                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3429 
 3430 //----------PIPELINE-----------------------------------------------------------
 3431 // Rules which define the behavior of the target architectures pipeline.
 3432 pipeline %{
 3433 
 3434 //----------ATTRIBUTES---------------------------------------------------------
 3435 attributes %{
 3436   variable_size_instructions;        // Fixed size instructions
 3437   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3438   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3439   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3440   instruction_fetch_units = 1;       // of 16 bytes
 3441 
 3442   // List of nop instructions
 3443   nops( MachNop );
 3444 %}
 3445 
 3446 //----------RESOURCES----------------------------------------------------------
 3447 // Resources are the functional units available to the machine
 3448 
 3449 // Generic P2/P3 pipeline
 3450 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3451 // 3 instructions decoded per cycle.
 3452 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3453 // 3 ALU op, only ALU0 handles mul instructions.
 3454 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3455            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3456            BR, FPU,
 3457            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3458 
 3459 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3460 // Pipeline Description specifies the stages in the machine's pipeline
 3461 
 3462 // Generic P2/P3 pipeline
 3463 pipe_desc(S0, S1, S2, S3, S4, S5);
 3464 
 3465 //----------PIPELINE CLASSES---------------------------------------------------
 3466 // Pipeline Classes describe the stages in which input and output are
 3467 // referenced by the hardware pipeline.
 3468 
 3469 // Naming convention: ialu or fpu
 3470 // Then: _reg
 3471 // Then: _reg if there is a 2nd register
 3472 // Then: _long if it's a pair of instructions implementing a long
 3473 // Then: _fat if it requires the big decoder
 3474 //   Or: _mem if it requires the big decoder and a memory unit.
 3475 
 3476 // Integer ALU reg operation
 3477 pipe_class ialu_reg(rRegI dst)
 3478 %{
 3479     single_instruction;
 3480     dst    : S4(write);
 3481     dst    : S3(read);
 3482     DECODE : S0;        // any decoder
 3483     ALU    : S3;        // any alu
 3484 %}
 3485 
 3486 // Long ALU reg operation
 3487 pipe_class ialu_reg_long(rRegL dst)
 3488 %{
 3489     instruction_count(2);
 3490     dst    : S4(write);
 3491     dst    : S3(read);
 3492     DECODE : S0(2);     // any 2 decoders
 3493     ALU    : S3(2);     // both alus
 3494 %}
 3495 
 3496 // Integer ALU reg operation using big decoder
 3497 pipe_class ialu_reg_fat(rRegI dst)
 3498 %{
 3499     single_instruction;
 3500     dst    : S4(write);
 3501     dst    : S3(read);
 3502     D0     : S0;        // big decoder only
 3503     ALU    : S3;        // any alu
 3504 %}
 3505 
 3506 // Integer ALU reg-reg operation
 3507 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3508 %{
 3509     single_instruction;
 3510     dst    : S4(write);
 3511     src    : S3(read);
 3512     DECODE : S0;        // any decoder
 3513     ALU    : S3;        // any alu
 3514 %}
 3515 
 3516 // Integer ALU reg-reg operation
 3517 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3518 %{
 3519     single_instruction;
 3520     dst    : S4(write);
 3521     src    : S3(read);
 3522     D0     : S0;        // big decoder only
 3523     ALU    : S3;        // any alu
 3524 %}
 3525 
 3526 // Integer ALU reg-mem operation
 3527 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3528 %{
 3529     single_instruction;
 3530     dst    : S5(write);
 3531     mem    : S3(read);
 3532     D0     : S0;        // big decoder only
 3533     ALU    : S4;        // any alu
 3534     MEM    : S3;        // any mem
 3535 %}
 3536 
 3537 // Integer mem operation (prefetch)
 3538 pipe_class ialu_mem(memory mem)
 3539 %{
 3540     single_instruction;
 3541     mem    : S3(read);
 3542     D0     : S0;        // big decoder only
 3543     MEM    : S3;        // any mem
 3544 %}
 3545 
 3546 // Integer Store to Memory
 3547 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3548 %{
 3549     single_instruction;
 3550     mem    : S3(read);
 3551     src    : S5(read);
 3552     D0     : S0;        // big decoder only
 3553     ALU    : S4;        // any alu
 3554     MEM    : S3;
 3555 %}
 3556 
 3557 // // Long Store to Memory
 3558 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3559 // %{
 3560 //     instruction_count(2);
 3561 //     mem    : S3(read);
 3562 //     src    : S5(read);
 3563 //     D0     : S0(2);          // big decoder only; twice
 3564 //     ALU    : S4(2);     // any 2 alus
 3565 //     MEM    : S3(2);  // Both mems
 3566 // %}
 3567 
 3568 // Integer Store to Memory
 3569 pipe_class ialu_mem_imm(memory mem)
 3570 %{
 3571     single_instruction;
 3572     mem    : S3(read);
 3573     D0     : S0;        // big decoder only
 3574     ALU    : S4;        // any alu
 3575     MEM    : S3;
 3576 %}
 3577 
 3578 // Integer ALU0 reg-reg operation
 3579 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3580 %{
 3581     single_instruction;
 3582     dst    : S4(write);
 3583     src    : S3(read);
 3584     D0     : S0;        // Big decoder only
 3585     ALU0   : S3;        // only alu0
 3586 %}
 3587 
 3588 // Integer ALU0 reg-mem operation
 3589 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3590 %{
 3591     single_instruction;
 3592     dst    : S5(write);
 3593     mem    : S3(read);
 3594     D0     : S0;        // big decoder only
 3595     ALU0   : S4;        // ALU0 only
 3596     MEM    : S3;        // any mem
 3597 %}
 3598 
 3599 // Integer ALU reg-reg operation
 3600 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3601 %{
 3602     single_instruction;
 3603     cr     : S4(write);
 3604     src1   : S3(read);
 3605     src2   : S3(read);
 3606     DECODE : S0;        // any decoder
 3607     ALU    : S3;        // any alu
 3608 %}
 3609 
 3610 // Integer ALU reg-imm operation
 3611 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3612 %{
 3613     single_instruction;
 3614     cr     : S4(write);
 3615     src1   : S3(read);
 3616     DECODE : S0;        // any decoder
 3617     ALU    : S3;        // any alu
 3618 %}
 3619 
 3620 // Integer ALU reg-mem operation
 3621 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3622 %{
 3623     single_instruction;
 3624     cr     : S4(write);
 3625     src1   : S3(read);
 3626     src2   : S3(read);
 3627     D0     : S0;        // big decoder only
 3628     ALU    : S4;        // any alu
 3629     MEM    : S3;
 3630 %}
 3631 
 3632 // Conditional move reg-reg
 3633 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3634 %{
 3635     instruction_count(4);
 3636     y      : S4(read);
 3637     q      : S3(read);
 3638     p      : S3(read);
 3639     DECODE : S0(4);     // any decoder
 3640 %}
 3641 
 3642 // Conditional move reg-reg
 3643 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3644 %{
 3645     single_instruction;
 3646     dst    : S4(write);
 3647     src    : S3(read);
 3648     cr     : S3(read);
 3649     DECODE : S0;        // any decoder
 3650 %}
 3651 
 3652 // Conditional move reg-mem
 3653 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3654 %{
 3655     single_instruction;
 3656     dst    : S4(write);
 3657     src    : S3(read);
 3658     cr     : S3(read);
 3659     DECODE : S0;        // any decoder
 3660     MEM    : S3;
 3661 %}
 3662 
 3663 // Conditional move reg-reg long
 3664 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3665 %{
 3666     single_instruction;
 3667     dst    : S4(write);
 3668     src    : S3(read);
 3669     cr     : S3(read);
 3670     DECODE : S0(2);     // any 2 decoders
 3671 %}
 3672 
 3673 // Float reg-reg operation
 3674 pipe_class fpu_reg(regD dst)
 3675 %{
 3676     instruction_count(2);
 3677     dst    : S3(read);
 3678     DECODE : S0(2);     // any 2 decoders
 3679     FPU    : S3;
 3680 %}
 3681 
 3682 // Float reg-reg operation
 3683 pipe_class fpu_reg_reg(regD dst, regD src)
 3684 %{
 3685     instruction_count(2);
 3686     dst    : S4(write);
 3687     src    : S3(read);
 3688     DECODE : S0(2);     // any 2 decoders
 3689     FPU    : S3;
 3690 %}
 3691 
 3692 // Float reg-reg operation
 3693 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3694 %{
 3695     instruction_count(3);
 3696     dst    : S4(write);
 3697     src1   : S3(read);
 3698     src2   : S3(read);
 3699     DECODE : S0(3);     // any 3 decoders
 3700     FPU    : S3(2);
 3701 %}
 3702 
 3703 // Float reg-reg operation
 3704 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3705 %{
 3706     instruction_count(4);
 3707     dst    : S4(write);
 3708     src1   : S3(read);
 3709     src2   : S3(read);
 3710     src3   : S3(read);
 3711     DECODE : S0(4);     // any 3 decoders
 3712     FPU    : S3(2);
 3713 %}
 3714 
 3715 // Float reg-reg operation
 3716 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3717 %{
 3718     instruction_count(4);
 3719     dst    : S4(write);
 3720     src1   : S3(read);
 3721     src2   : S3(read);
 3722     src3   : S3(read);
 3723     DECODE : S1(3);     // any 3 decoders
 3724     D0     : S0;        // Big decoder only
 3725     FPU    : S3(2);
 3726     MEM    : S3;
 3727 %}
 3728 
 3729 // Float reg-mem operation
 3730 pipe_class fpu_reg_mem(regD dst, memory mem)
 3731 %{
 3732     instruction_count(2);
 3733     dst    : S5(write);
 3734     mem    : S3(read);
 3735     D0     : S0;        // big decoder only
 3736     DECODE : S1;        // any decoder for FPU POP
 3737     FPU    : S4;
 3738     MEM    : S3;        // any mem
 3739 %}
 3740 
 3741 // Float reg-mem operation
 3742 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3743 %{
 3744     instruction_count(3);
 3745     dst    : S5(write);
 3746     src1   : S3(read);
 3747     mem    : S3(read);
 3748     D0     : S0;        // big decoder only
 3749     DECODE : S1(2);     // any decoder for FPU POP
 3750     FPU    : S4;
 3751     MEM    : S3;        // any mem
 3752 %}
 3753 
 3754 // Float mem-reg operation
 3755 pipe_class fpu_mem_reg(memory mem, regD src)
 3756 %{
 3757     instruction_count(2);
 3758     src    : S5(read);
 3759     mem    : S3(read);
 3760     DECODE : S0;        // any decoder for FPU PUSH
 3761     D0     : S1;        // big decoder only
 3762     FPU    : S4;
 3763     MEM    : S3;        // any mem
 3764 %}
 3765 
 3766 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3767 %{
 3768     instruction_count(3);
 3769     src1   : S3(read);
 3770     src2   : S3(read);
 3771     mem    : S3(read);
 3772     DECODE : S0(2);     // any decoder for FPU PUSH
 3773     D0     : S1;        // big decoder only
 3774     FPU    : S4;
 3775     MEM    : S3;        // any mem
 3776 %}
 3777 
 3778 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3779 %{
 3780     instruction_count(3);
 3781     src1   : S3(read);
 3782     src2   : S3(read);
 3783     mem    : S4(read);
 3784     DECODE : S0;        // any decoder for FPU PUSH
 3785     D0     : S0(2);     // big decoder only
 3786     FPU    : S4;
 3787     MEM    : S3(2);     // any mem
 3788 %}
 3789 
 3790 pipe_class fpu_mem_mem(memory dst, memory src1)
 3791 %{
 3792     instruction_count(2);
 3793     src1   : S3(read);
 3794     dst    : S4(read);
 3795     D0     : S0(2);     // big decoder only
 3796     MEM    : S3(2);     // any mem
 3797 %}
 3798 
 3799 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3800 %{
 3801     instruction_count(3);
 3802     src1   : S3(read);
 3803     src2   : S3(read);
 3804     dst    : S4(read);
 3805     D0     : S0(3);     // big decoder only
 3806     FPU    : S4;
 3807     MEM    : S3(3);     // any mem
 3808 %}
 3809 
 3810 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3811 %{
 3812     instruction_count(3);
 3813     src1   : S4(read);
 3814     mem    : S4(read);
 3815     DECODE : S0;        // any decoder for FPU PUSH
 3816     D0     : S0(2);     // big decoder only
 3817     FPU    : S4;
 3818     MEM    : S3(2);     // any mem
 3819 %}
 3820 
 3821 // Float load constant
 3822 pipe_class fpu_reg_con(regD dst)
 3823 %{
 3824     instruction_count(2);
 3825     dst    : S5(write);
 3826     D0     : S0;        // big decoder only for the load
 3827     DECODE : S1;        // any decoder for FPU POP
 3828     FPU    : S4;
 3829     MEM    : S3;        // any mem
 3830 %}
 3831 
 3832 // Float load constant
 3833 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3834 %{
 3835     instruction_count(3);
 3836     dst    : S5(write);
 3837     src    : S3(read);
 3838     D0     : S0;        // big decoder only for the load
 3839     DECODE : S1(2);     // any decoder for FPU POP
 3840     FPU    : S4;
 3841     MEM    : S3;        // any mem
 3842 %}
 3843 
 3844 // UnConditional branch
 3845 pipe_class pipe_jmp(label labl)
 3846 %{
 3847     single_instruction;
 3848     BR   : S3;
 3849 %}
 3850 
 3851 // Conditional branch
 3852 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3853 %{
 3854     single_instruction;
 3855     cr    : S1(read);
 3856     BR    : S3;
 3857 %}
 3858 
 3859 // Allocation idiom
 3860 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3861 %{
 3862     instruction_count(1); force_serialization;
 3863     fixed_latency(6);
 3864     heap_ptr : S3(read);
 3865     DECODE   : S0(3);
 3866     D0       : S2;
 3867     MEM      : S3;
 3868     ALU      : S3(2);
 3869     dst      : S5(write);
 3870     BR       : S5;
 3871 %}
 3872 
 3873 // Generic big/slow expanded idiom
 3874 pipe_class pipe_slow()
 3875 %{
 3876     instruction_count(10); multiple_bundles; force_serialization;
 3877     fixed_latency(100);
 3878     D0  : S0(2);
 3879     MEM : S3(2);
 3880 %}
 3881 
 3882 // The real do-nothing guy
 3883 pipe_class empty()
 3884 %{
 3885     instruction_count(0);
 3886 %}
 3887 
 3888 // Define the class for the Nop node
 3889 define
 3890 %{
 3891    MachNop = empty;
 3892 %}
 3893 
 3894 %}
 3895 
 3896 //----------INSTRUCTIONS-------------------------------------------------------
 3897 //
 3898 // match      -- States which machine-independent subtree may be replaced
 3899 //               by this instruction.
 3900 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3901 //               selection to identify a minimum cost tree of machine
 3902 //               instructions that matches a tree of machine-independent
 3903 //               instructions.
 3904 // format     -- A string providing the disassembly for this instruction.
 3905 //               The value of an instruction's operand may be inserted
 3906 //               by referring to it with a '$' prefix.
 3907 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3908 //               to within an encode class as $primary, $secondary, and $tertiary
 3909 //               rrspectively.  The primary opcode is commonly used to
 3910 //               indicate the type of machine instruction, while secondary
 3911 //               and tertiary are often used for prefix options or addressing
 3912 //               modes.
 3913 // ins_encode -- A list of encode classes with parameters. The encode class
 3914 //               name must have been defined in an 'enc_class' specification
 3915 //               in the encode section of the architecture description.
 3916 
 3917 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3918 // Load Float
 3919 instruct MoveF2VL(vlRegF dst, regF src) %{
 3920   match(Set dst src);
 3921   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3922   ins_encode %{
 3923     ShouldNotReachHere();
 3924   %}
 3925   ins_pipe( fpu_reg_reg );
 3926 %}
 3927 
 3928 // Load Float
 3929 instruct MoveF2LEG(legRegF dst, regF src) %{
 3930   match(Set dst src);
 3931   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3932   ins_encode %{
 3933     ShouldNotReachHere();
 3934   %}
 3935   ins_pipe( fpu_reg_reg );
 3936 %}
 3937 
 3938 // Load Float
 3939 instruct MoveVL2F(regF dst, vlRegF src) %{
 3940   match(Set dst src);
 3941   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3942   ins_encode %{
 3943     ShouldNotReachHere();
 3944   %}
 3945   ins_pipe( fpu_reg_reg );
 3946 %}
 3947 
 3948 // Load Float
 3949 instruct MoveLEG2F(regF dst, legRegF src) %{
 3950   match(Set dst src);
 3951   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3952   ins_encode %{
 3953     ShouldNotReachHere();
 3954   %}
 3955   ins_pipe( fpu_reg_reg );
 3956 %}
 3957 
 3958 // Load Double
 3959 instruct MoveD2VL(vlRegD dst, regD src) %{
 3960   match(Set dst src);
 3961   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3962   ins_encode %{
 3963     ShouldNotReachHere();
 3964   %}
 3965   ins_pipe( fpu_reg_reg );
 3966 %}
 3967 
 3968 // Load Double
 3969 instruct MoveD2LEG(legRegD dst, regD src) %{
 3970   match(Set dst src);
 3971   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3972   ins_encode %{
 3973     ShouldNotReachHere();
 3974   %}
 3975   ins_pipe( fpu_reg_reg );
 3976 %}
 3977 
 3978 // Load Double
 3979 instruct MoveVL2D(regD dst, vlRegD src) %{
 3980   match(Set dst src);
 3981   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3982   ins_encode %{
 3983     ShouldNotReachHere();
 3984   %}
 3985   ins_pipe( fpu_reg_reg );
 3986 %}
 3987 
 3988 // Load Double
 3989 instruct MoveLEG2D(regD dst, legRegD src) %{
 3990   match(Set dst src);
 3991   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3992   ins_encode %{
 3993     ShouldNotReachHere();
 3994   %}
 3995   ins_pipe( fpu_reg_reg );
 3996 %}
 3997 
 3998 //----------Load/Store/Move Instructions---------------------------------------
 3999 //----------Load Instructions--------------------------------------------------
 4000 
 4001 // Load Byte (8 bit signed)
 4002 instruct loadB(rRegI dst, memory mem)
 4003 %{
 4004   match(Set dst (LoadB mem));
 4005 
 4006   ins_cost(125);
 4007   format %{ "movsbl  $dst, $mem\t# byte" %}
 4008 
 4009   ins_encode %{
 4010     __ movsbl($dst$$Register, $mem$$Address);
 4011   %}
 4012 
 4013   ins_pipe(ialu_reg_mem);
 4014 %}
 4015 
 4016 // Load Byte (8 bit signed) into Long Register
 4017 instruct loadB2L(rRegL dst, memory mem)
 4018 %{
 4019   match(Set dst (ConvI2L (LoadB mem)));
 4020 
 4021   ins_cost(125);
 4022   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4023 
 4024   ins_encode %{
 4025     __ movsbq($dst$$Register, $mem$$Address);
 4026   %}
 4027 
 4028   ins_pipe(ialu_reg_mem);
 4029 %}
 4030 
 4031 // Load Unsigned Byte (8 bit UNsigned)
 4032 instruct loadUB(rRegI dst, memory mem)
 4033 %{
 4034   match(Set dst (LoadUB mem));
 4035 
 4036   ins_cost(125);
 4037   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4038 
 4039   ins_encode %{
 4040     __ movzbl($dst$$Register, $mem$$Address);
 4041   %}
 4042 
 4043   ins_pipe(ialu_reg_mem);
 4044 %}
 4045 
 4046 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4047 instruct loadUB2L(rRegL dst, memory mem)
 4048 %{
 4049   match(Set dst (ConvI2L (LoadUB mem)));
 4050 
 4051   ins_cost(125);
 4052   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4053 
 4054   ins_encode %{
 4055     __ movzbq($dst$$Register, $mem$$Address);
 4056   %}
 4057 
 4058   ins_pipe(ialu_reg_mem);
 4059 %}
 4060 
 4061 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4062 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4063   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4064   effect(KILL cr);
 4065 
 4066   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4067             "andl    $dst, right_n_bits($mask, 8)" %}
 4068   ins_encode %{
 4069     Register Rdst = $dst$$Register;
 4070     __ movzbq(Rdst, $mem$$Address);
 4071     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4072   %}
 4073   ins_pipe(ialu_reg_mem);
 4074 %}
 4075 
 4076 // Load Short (16 bit signed)
 4077 instruct loadS(rRegI dst, memory mem)
 4078 %{
 4079   match(Set dst (LoadS mem));
 4080 
 4081   ins_cost(125);
 4082   format %{ "movswl $dst, $mem\t# short" %}
 4083 
 4084   ins_encode %{
 4085     __ movswl($dst$$Register, $mem$$Address);
 4086   %}
 4087 
 4088   ins_pipe(ialu_reg_mem);
 4089 %}
 4090 
 4091 // Load Short (16 bit signed) to Byte (8 bit signed)
 4092 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4093   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4094 
 4095   ins_cost(125);
 4096   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4097   ins_encode %{
 4098     __ movsbl($dst$$Register, $mem$$Address);
 4099   %}
 4100   ins_pipe(ialu_reg_mem);
 4101 %}
 4102 
 4103 // Load Short (16 bit signed) into Long Register
 4104 instruct loadS2L(rRegL dst, memory mem)
 4105 %{
 4106   match(Set dst (ConvI2L (LoadS mem)));
 4107 
 4108   ins_cost(125);
 4109   format %{ "movswq $dst, $mem\t# short -> long" %}
 4110 
 4111   ins_encode %{
 4112     __ movswq($dst$$Register, $mem$$Address);
 4113   %}
 4114 
 4115   ins_pipe(ialu_reg_mem);
 4116 %}
 4117 
 4118 // Load Unsigned Short/Char (16 bit UNsigned)
 4119 instruct loadUS(rRegI dst, memory mem)
 4120 %{
 4121   match(Set dst (LoadUS mem));
 4122 
 4123   ins_cost(125);
 4124   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4125 
 4126   ins_encode %{
 4127     __ movzwl($dst$$Register, $mem$$Address);
 4128   %}
 4129 
 4130   ins_pipe(ialu_reg_mem);
 4131 %}
 4132 
 4133 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4134 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4135   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4136 
 4137   ins_cost(125);
 4138   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4139   ins_encode %{
 4140     __ movsbl($dst$$Register, $mem$$Address);
 4141   %}
 4142   ins_pipe(ialu_reg_mem);
 4143 %}
 4144 
 4145 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4146 instruct loadUS2L(rRegL dst, memory mem)
 4147 %{
 4148   match(Set dst (ConvI2L (LoadUS mem)));
 4149 
 4150   ins_cost(125);
 4151   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4152 
 4153   ins_encode %{
 4154     __ movzwq($dst$$Register, $mem$$Address);
 4155   %}
 4156 
 4157   ins_pipe(ialu_reg_mem);
 4158 %}
 4159 
 4160 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4161 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4162   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4163 
 4164   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4165   ins_encode %{
 4166     __ movzbq($dst$$Register, $mem$$Address);
 4167   %}
 4168   ins_pipe(ialu_reg_mem);
 4169 %}
 4170 
 4171 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4172 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4173   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4174   effect(KILL cr);
 4175 
 4176   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4177             "andl    $dst, right_n_bits($mask, 16)" %}
 4178   ins_encode %{
 4179     Register Rdst = $dst$$Register;
 4180     __ movzwq(Rdst, $mem$$Address);
 4181     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4182   %}
 4183   ins_pipe(ialu_reg_mem);
 4184 %}
 4185 
 4186 // Load Integer
 4187 instruct loadI(rRegI dst, memory mem)
 4188 %{
 4189   match(Set dst (LoadI mem));
 4190 
 4191   ins_cost(125);
 4192   format %{ "movl    $dst, $mem\t# int" %}
 4193 
 4194   ins_encode %{
 4195     __ movl($dst$$Register, $mem$$Address);
 4196   %}
 4197 
 4198   ins_pipe(ialu_reg_mem);
 4199 %}
 4200 
 4201 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4202 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4203   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4204 
 4205   ins_cost(125);
 4206   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4207   ins_encode %{
 4208     __ movsbl($dst$$Register, $mem$$Address);
 4209   %}
 4210   ins_pipe(ialu_reg_mem);
 4211 %}
 4212 
 4213 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4214 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4215   match(Set dst (AndI (LoadI mem) mask));
 4216 
 4217   ins_cost(125);
 4218   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4219   ins_encode %{
 4220     __ movzbl($dst$$Register, $mem$$Address);
 4221   %}
 4222   ins_pipe(ialu_reg_mem);
 4223 %}
 4224 
 4225 // Load Integer (32 bit signed) to Short (16 bit signed)
 4226 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4227   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4228 
 4229   ins_cost(125);
 4230   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4231   ins_encode %{
 4232     __ movswl($dst$$Register, $mem$$Address);
 4233   %}
 4234   ins_pipe(ialu_reg_mem);
 4235 %}
 4236 
 4237 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4238 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4239   match(Set dst (AndI (LoadI mem) mask));
 4240 
 4241   ins_cost(125);
 4242   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4243   ins_encode %{
 4244     __ movzwl($dst$$Register, $mem$$Address);
 4245   %}
 4246   ins_pipe(ialu_reg_mem);
 4247 %}
 4248 
 4249 // Load Integer into Long Register
 4250 instruct loadI2L(rRegL dst, memory mem)
 4251 %{
 4252   match(Set dst (ConvI2L (LoadI mem)));
 4253 
 4254   ins_cost(125);
 4255   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4256 
 4257   ins_encode %{
 4258     __ movslq($dst$$Register, $mem$$Address);
 4259   %}
 4260 
 4261   ins_pipe(ialu_reg_mem);
 4262 %}
 4263 
 4264 // Load Integer with mask 0xFF into Long Register
 4265 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4266   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4267 
 4268   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4269   ins_encode %{
 4270     __ movzbq($dst$$Register, $mem$$Address);
 4271   %}
 4272   ins_pipe(ialu_reg_mem);
 4273 %}
 4274 
 4275 // Load Integer with mask 0xFFFF into Long Register
 4276 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4277   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4278 
 4279   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4280   ins_encode %{
 4281     __ movzwq($dst$$Register, $mem$$Address);
 4282   %}
 4283   ins_pipe(ialu_reg_mem);
 4284 %}
 4285 
 4286 // Load Integer with a 31-bit mask into Long Register
 4287 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4288   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4289   effect(KILL cr);
 4290 
 4291   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4292             "andl    $dst, $mask" %}
 4293   ins_encode %{
 4294     Register Rdst = $dst$$Register;
 4295     __ movl(Rdst, $mem$$Address);
 4296     __ andl(Rdst, $mask$$constant);
 4297   %}
 4298   ins_pipe(ialu_reg_mem);
 4299 %}
 4300 
 4301 // Load Unsigned Integer into Long Register
 4302 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4303 %{
 4304   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4305 
 4306   ins_cost(125);
 4307   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4308 
 4309   ins_encode %{
 4310     __ movl($dst$$Register, $mem$$Address);
 4311   %}
 4312 
 4313   ins_pipe(ialu_reg_mem);
 4314 %}
 4315 
 4316 // Load Long
 4317 instruct loadL(rRegL dst, memory mem)
 4318 %{
 4319   match(Set dst (LoadL mem));
 4320 
 4321   ins_cost(125);
 4322   format %{ "movq    $dst, $mem\t# long" %}
 4323 
 4324   ins_encode %{
 4325     __ movq($dst$$Register, $mem$$Address);
 4326   %}
 4327 
 4328   ins_pipe(ialu_reg_mem); // XXX
 4329 %}
 4330 
 4331 // Load Range
 4332 instruct loadRange(rRegI dst, memory mem)
 4333 %{
 4334   match(Set dst (LoadRange mem));
 4335 
 4336   ins_cost(125); // XXX
 4337   format %{ "movl    $dst, $mem\t# range" %}
 4338   ins_encode %{
 4339     __ movl($dst$$Register, $mem$$Address);
 4340   %}
 4341   ins_pipe(ialu_reg_mem);
 4342 %}
 4343 
 4344 // Load Pointer
 4345 instruct loadP(rRegP dst, memory mem)
 4346 %{
 4347   match(Set dst (LoadP mem));
 4348   predicate(n->as_Load()->barrier_data() == 0);
 4349 
 4350   ins_cost(125); // XXX
 4351   format %{ "movq    $dst, $mem\t# ptr" %}
 4352   ins_encode %{
 4353     __ movq($dst$$Register, $mem$$Address);
 4354   %}
 4355   ins_pipe(ialu_reg_mem); // XXX
 4356 %}
 4357 
 4358 // Load Compressed Pointer
 4359 instruct loadN(rRegN dst, memory mem)
 4360 %{
 4361    predicate(n->as_Load()->barrier_data() == 0);
 4362    match(Set dst (LoadN mem));
 4363 
 4364    ins_cost(125); // XXX
 4365    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4366    ins_encode %{
 4367      __ movl($dst$$Register, $mem$$Address);
 4368    %}
 4369    ins_pipe(ialu_reg_mem); // XXX
 4370 %}
 4371 
 4372 
 4373 // Load Klass Pointer
 4374 instruct loadKlass(rRegP dst, memory mem)
 4375 %{
 4376   match(Set dst (LoadKlass mem));
 4377 
 4378   ins_cost(125); // XXX
 4379   format %{ "movq    $dst, $mem\t# class" %}
 4380   ins_encode %{
 4381     __ movq($dst$$Register, $mem$$Address);
 4382   %}
 4383   ins_pipe(ialu_reg_mem); // XXX
 4384 %}
 4385 
 4386 // Load narrow Klass Pointer
 4387 instruct loadNKlass(rRegN dst, memory mem)
 4388 %{
 4389   predicate(!UseCompactObjectHeaders);
 4390   match(Set dst (LoadNKlass mem));
 4391 
 4392   ins_cost(125); // XXX
 4393   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4394   ins_encode %{
 4395     __ movl($dst$$Register, $mem$$Address);
 4396   %}
 4397   ins_pipe(ialu_reg_mem); // XXX
 4398 %}
 4399 
 4400 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4401 %{
 4402   predicate(UseCompactObjectHeaders);
 4403   match(Set dst (LoadNKlass mem));
 4404   effect(KILL cr);
 4405   ins_cost(125); // XXX
 4406   format %{
 4407     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4408     "shrl    $dst, markWord::klass_shift_at_offset"
 4409   %}
 4410   ins_encode %{
 4411     __ movl($dst$$Register, $mem$$Address);
 4412     __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4413   %}
 4414   ins_pipe(ialu_reg_mem); // XXX
 4415 %}
 4416 
 4417 // Load Float
 4418 instruct loadF(regF dst, memory mem)
 4419 %{
 4420   match(Set dst (LoadF mem));
 4421 
 4422   ins_cost(145); // XXX
 4423   format %{ "movss   $dst, $mem\t# float" %}
 4424   ins_encode %{
 4425     __ movflt($dst$$XMMRegister, $mem$$Address);
 4426   %}
 4427   ins_pipe(pipe_slow); // XXX
 4428 %}
 4429 
 4430 // Load Double
 4431 instruct loadD_partial(regD dst, memory mem)
 4432 %{
 4433   predicate(!UseXmmLoadAndClearUpper);
 4434   match(Set dst (LoadD mem));
 4435 
 4436   ins_cost(145); // XXX
 4437   format %{ "movlpd  $dst, $mem\t# double" %}
 4438   ins_encode %{
 4439     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4440   %}
 4441   ins_pipe(pipe_slow); // XXX
 4442 %}
 4443 
 4444 instruct loadD(regD dst, memory mem)
 4445 %{
 4446   predicate(UseXmmLoadAndClearUpper);
 4447   match(Set dst (LoadD mem));
 4448 
 4449   ins_cost(145); // XXX
 4450   format %{ "movsd   $dst, $mem\t# double" %}
 4451   ins_encode %{
 4452     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4453   %}
 4454   ins_pipe(pipe_slow); // XXX
 4455 %}
 4456 
 4457 // max = java.lang.Math.max(float a, float b)
 4458 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4459   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4460   match(Set dst (MaxF a b));
 4461   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4462   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4463   ins_encode %{
 4464     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4465   %}
 4466   ins_pipe( pipe_slow );
 4467 %}
 4468 
 4469 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4470   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4471   match(Set dst (MaxF a b));
 4472   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4473 
 4474   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4475   ins_encode %{
 4476     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4477                     false /*min*/, true /*single*/);
 4478   %}
 4479   ins_pipe( pipe_slow );
 4480 %}
 4481 
 4482 // max = java.lang.Math.max(double a, double b)
 4483 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4484   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4485   match(Set dst (MaxD a b));
 4486   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4487   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4488   ins_encode %{
 4489     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4490   %}
 4491   ins_pipe( pipe_slow );
 4492 %}
 4493 
 4494 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4495   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4496   match(Set dst (MaxD a b));
 4497   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4498 
 4499   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4500   ins_encode %{
 4501     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4502                     false /*min*/, false /*single*/);
 4503   %}
 4504   ins_pipe( pipe_slow );
 4505 %}
 4506 
 4507 // min = java.lang.Math.min(float a, float b)
 4508 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4509   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4510   match(Set dst (MinF a b));
 4511   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4512   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4513   ins_encode %{
 4514     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4515   %}
 4516   ins_pipe( pipe_slow );
 4517 %}
 4518 
 4519 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4520   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4521   match(Set dst (MinF a b));
 4522   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4523 
 4524   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4525   ins_encode %{
 4526     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4527                     true /*min*/, true /*single*/);
 4528   %}
 4529   ins_pipe( pipe_slow );
 4530 %}
 4531 
 4532 // min = java.lang.Math.min(double a, double b)
 4533 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4534   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4535   match(Set dst (MinD a b));
 4536   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4537     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4538   ins_encode %{
 4539     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4540   %}
 4541   ins_pipe( pipe_slow );
 4542 %}
 4543 
 4544 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4545   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4546   match(Set dst (MinD a b));
 4547   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4548 
 4549   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4550   ins_encode %{
 4551     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4552                     true /*min*/, false /*single*/);
 4553   %}
 4554   ins_pipe( pipe_slow );
 4555 %}
 4556 
 4557 // Load Effective Address
 4558 instruct leaP8(rRegP dst, indOffset8 mem)
 4559 %{
 4560   match(Set dst mem);
 4561 
 4562   ins_cost(110); // XXX
 4563   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4564   ins_encode %{
 4565     __ leaq($dst$$Register, $mem$$Address);
 4566   %}
 4567   ins_pipe(ialu_reg_reg_fat);
 4568 %}
 4569 
 4570 instruct leaP32(rRegP dst, indOffset32 mem)
 4571 %{
 4572   match(Set dst mem);
 4573 
 4574   ins_cost(110);
 4575   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4576   ins_encode %{
 4577     __ leaq($dst$$Register, $mem$$Address);
 4578   %}
 4579   ins_pipe(ialu_reg_reg_fat);
 4580 %}
 4581 
 4582 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4583 %{
 4584   match(Set dst mem);
 4585 
 4586   ins_cost(110);
 4587   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4588   ins_encode %{
 4589     __ leaq($dst$$Register, $mem$$Address);
 4590   %}
 4591   ins_pipe(ialu_reg_reg_fat);
 4592 %}
 4593 
 4594 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4595 %{
 4596   match(Set dst mem);
 4597 
 4598   ins_cost(110);
 4599   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4600   ins_encode %{
 4601     __ leaq($dst$$Register, $mem$$Address);
 4602   %}
 4603   ins_pipe(ialu_reg_reg_fat);
 4604 %}
 4605 
 4606 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4607 %{
 4608   match(Set dst mem);
 4609 
 4610   ins_cost(110);
 4611   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4612   ins_encode %{
 4613     __ leaq($dst$$Register, $mem$$Address);
 4614   %}
 4615   ins_pipe(ialu_reg_reg_fat);
 4616 %}
 4617 
 4618 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4619 %{
 4620   match(Set dst mem);
 4621 
 4622   ins_cost(110);
 4623   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4624   ins_encode %{
 4625     __ leaq($dst$$Register, $mem$$Address);
 4626   %}
 4627   ins_pipe(ialu_reg_reg_fat);
 4628 %}
 4629 
 4630 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4631 %{
 4632   match(Set dst mem);
 4633 
 4634   ins_cost(110);
 4635   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4636   ins_encode %{
 4637     __ leaq($dst$$Register, $mem$$Address);
 4638   %}
 4639   ins_pipe(ialu_reg_reg_fat);
 4640 %}
 4641 
 4642 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4643 %{
 4644   match(Set dst mem);
 4645 
 4646   ins_cost(110);
 4647   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4648   ins_encode %{
 4649     __ leaq($dst$$Register, $mem$$Address);
 4650   %}
 4651   ins_pipe(ialu_reg_reg_fat);
 4652 %}
 4653 
 4654 // Load Effective Address which uses Narrow (32-bits) oop
 4655 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4656 %{
 4657   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4658   match(Set dst mem);
 4659 
 4660   ins_cost(110);
 4661   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4662   ins_encode %{
 4663     __ leaq($dst$$Register, $mem$$Address);
 4664   %}
 4665   ins_pipe(ialu_reg_reg_fat);
 4666 %}
 4667 
 4668 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4669 %{
 4670   predicate(CompressedOops::shift() == 0);
 4671   match(Set dst mem);
 4672 
 4673   ins_cost(110); // XXX
 4674   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4675   ins_encode %{
 4676     __ leaq($dst$$Register, $mem$$Address);
 4677   %}
 4678   ins_pipe(ialu_reg_reg_fat);
 4679 %}
 4680 
 4681 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4682 %{
 4683   predicate(CompressedOops::shift() == 0);
 4684   match(Set dst mem);
 4685 
 4686   ins_cost(110);
 4687   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4688   ins_encode %{
 4689     __ leaq($dst$$Register, $mem$$Address);
 4690   %}
 4691   ins_pipe(ialu_reg_reg_fat);
 4692 %}
 4693 
 4694 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4695 %{
 4696   predicate(CompressedOops::shift() == 0);
 4697   match(Set dst mem);
 4698 
 4699   ins_cost(110);
 4700   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4701   ins_encode %{
 4702     __ leaq($dst$$Register, $mem$$Address);
 4703   %}
 4704   ins_pipe(ialu_reg_reg_fat);
 4705 %}
 4706 
 4707 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4708 %{
 4709   predicate(CompressedOops::shift() == 0);
 4710   match(Set dst mem);
 4711 
 4712   ins_cost(110);
 4713   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4714   ins_encode %{
 4715     __ leaq($dst$$Register, $mem$$Address);
 4716   %}
 4717   ins_pipe(ialu_reg_reg_fat);
 4718 %}
 4719 
 4720 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4721 %{
 4722   predicate(CompressedOops::shift() == 0);
 4723   match(Set dst mem);
 4724 
 4725   ins_cost(110);
 4726   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4727   ins_encode %{
 4728     __ leaq($dst$$Register, $mem$$Address);
 4729   %}
 4730   ins_pipe(ialu_reg_reg_fat);
 4731 %}
 4732 
 4733 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4734 %{
 4735   predicate(CompressedOops::shift() == 0);
 4736   match(Set dst mem);
 4737 
 4738   ins_cost(110);
 4739   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4740   ins_encode %{
 4741     __ leaq($dst$$Register, $mem$$Address);
 4742   %}
 4743   ins_pipe(ialu_reg_reg_fat);
 4744 %}
 4745 
 4746 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4747 %{
 4748   predicate(CompressedOops::shift() == 0);
 4749   match(Set dst mem);
 4750 
 4751   ins_cost(110);
 4752   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4753   ins_encode %{
 4754     __ leaq($dst$$Register, $mem$$Address);
 4755   %}
 4756   ins_pipe(ialu_reg_reg_fat);
 4757 %}
 4758 
 4759 instruct loadConI(rRegI dst, immI src)
 4760 %{
 4761   match(Set dst src);
 4762 
 4763   format %{ "movl    $dst, $src\t# int" %}
 4764   ins_encode %{
 4765     __ movl($dst$$Register, $src$$constant);
 4766   %}
 4767   ins_pipe(ialu_reg_fat); // XXX
 4768 %}
 4769 
 4770 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4771 %{
 4772   match(Set dst src);
 4773   effect(KILL cr);
 4774 
 4775   ins_cost(50);
 4776   format %{ "xorl    $dst, $dst\t# int" %}
 4777   ins_encode %{
 4778     __ xorl($dst$$Register, $dst$$Register);
 4779   %}
 4780   ins_pipe(ialu_reg);
 4781 %}
 4782 
 4783 instruct loadConL(rRegL dst, immL src)
 4784 %{
 4785   match(Set dst src);
 4786 
 4787   ins_cost(150);
 4788   format %{ "movq    $dst, $src\t# long" %}
 4789   ins_encode %{
 4790     __ mov64($dst$$Register, $src$$constant);
 4791   %}
 4792   ins_pipe(ialu_reg);
 4793 %}
 4794 
 4795 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4796 %{
 4797   match(Set dst src);
 4798   effect(KILL cr);
 4799 
 4800   ins_cost(50);
 4801   format %{ "xorl    $dst, $dst\t# long" %}
 4802   ins_encode %{
 4803     __ xorl($dst$$Register, $dst$$Register);
 4804   %}
 4805   ins_pipe(ialu_reg); // XXX
 4806 %}
 4807 
 4808 instruct loadConUL32(rRegL dst, immUL32 src)
 4809 %{
 4810   match(Set dst src);
 4811 
 4812   ins_cost(60);
 4813   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4814   ins_encode %{
 4815     __ movl($dst$$Register, $src$$constant);
 4816   %}
 4817   ins_pipe(ialu_reg);
 4818 %}
 4819 
 4820 instruct loadConL32(rRegL dst, immL32 src)
 4821 %{
 4822   match(Set dst src);
 4823 
 4824   ins_cost(70);
 4825   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4826   ins_encode %{
 4827     __ movq($dst$$Register, $src$$constant);
 4828   %}
 4829   ins_pipe(ialu_reg);
 4830 %}
 4831 
 4832 instruct loadConP(rRegP dst, immP con) %{
 4833   match(Set dst con);
 4834 
 4835   format %{ "movq    $dst, $con\t# ptr" %}
 4836   ins_encode %{
 4837     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4838   %}
 4839   ins_pipe(ialu_reg_fat); // XXX
 4840 %}
 4841 
 4842 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4843 %{
 4844   match(Set dst src);
 4845   effect(KILL cr);
 4846 
 4847   ins_cost(50);
 4848   format %{ "xorl    $dst, $dst\t# ptr" %}
 4849   ins_encode %{
 4850     __ xorl($dst$$Register, $dst$$Register);
 4851   %}
 4852   ins_pipe(ialu_reg);
 4853 %}
 4854 
 4855 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4856 %{
 4857   match(Set dst src);
 4858   effect(KILL cr);
 4859 
 4860   ins_cost(60);
 4861   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4862   ins_encode %{
 4863     __ movl($dst$$Register, $src$$constant);
 4864   %}
 4865   ins_pipe(ialu_reg);
 4866 %}
 4867 
 4868 instruct loadConF(regF dst, immF con) %{
 4869   match(Set dst con);
 4870   ins_cost(125);
 4871   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4872   ins_encode %{
 4873     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4874   %}
 4875   ins_pipe(pipe_slow);
 4876 %}
 4877 
 4878 instruct loadConH(regF dst, immH con) %{
 4879   match(Set dst con);
 4880   ins_cost(125);
 4881   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4882   ins_encode %{
 4883     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4884   %}
 4885   ins_pipe(pipe_slow);
 4886 %}
 4887 
 4888 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4889   match(Set dst src);
 4890   effect(KILL cr);
 4891   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4892   ins_encode %{
 4893     __ xorq($dst$$Register, $dst$$Register);
 4894   %}
 4895   ins_pipe(ialu_reg);
 4896 %}
 4897 
 4898 instruct loadConN(rRegN dst, immN src) %{
 4899   match(Set dst src);
 4900 
 4901   ins_cost(125);
 4902   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4903   ins_encode %{
 4904     address con = (address)$src$$constant;
 4905     if (con == nullptr) {
 4906       ShouldNotReachHere();
 4907     } else {
 4908       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4909     }
 4910   %}
 4911   ins_pipe(ialu_reg_fat); // XXX
 4912 %}
 4913 
 4914 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4915   match(Set dst src);
 4916 
 4917   ins_cost(125);
 4918   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4919   ins_encode %{
 4920     address con = (address)$src$$constant;
 4921     if (con == nullptr) {
 4922       ShouldNotReachHere();
 4923     } else {
 4924       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4925     }
 4926   %}
 4927   ins_pipe(ialu_reg_fat); // XXX
 4928 %}
 4929 
 4930 instruct loadConF0(regF dst, immF0 src)
 4931 %{
 4932   match(Set dst src);
 4933   ins_cost(100);
 4934 
 4935   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4936   ins_encode %{
 4937     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4938   %}
 4939   ins_pipe(pipe_slow);
 4940 %}
 4941 
 4942 // Use the same format since predicate() can not be used here.
 4943 instruct loadConD(regD dst, immD con) %{
 4944   match(Set dst con);
 4945   ins_cost(125);
 4946   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4947   ins_encode %{
 4948     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4949   %}
 4950   ins_pipe(pipe_slow);
 4951 %}
 4952 
 4953 instruct loadConD0(regD dst, immD0 src)
 4954 %{
 4955   match(Set dst src);
 4956   ins_cost(100);
 4957 
 4958   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4959   ins_encode %{
 4960     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4961   %}
 4962   ins_pipe(pipe_slow);
 4963 %}
 4964 
 4965 instruct loadSSI(rRegI dst, stackSlotI src)
 4966 %{
 4967   match(Set dst src);
 4968 
 4969   ins_cost(125);
 4970   format %{ "movl    $dst, $src\t# int stk" %}
 4971   ins_encode %{
 4972     __ movl($dst$$Register, $src$$Address);
 4973   %}
 4974   ins_pipe(ialu_reg_mem);
 4975 %}
 4976 
 4977 instruct loadSSL(rRegL dst, stackSlotL src)
 4978 %{
 4979   match(Set dst src);
 4980 
 4981   ins_cost(125);
 4982   format %{ "movq    $dst, $src\t# long stk" %}
 4983   ins_encode %{
 4984     __ movq($dst$$Register, $src$$Address);
 4985   %}
 4986   ins_pipe(ialu_reg_mem);
 4987 %}
 4988 
 4989 instruct loadSSP(rRegP dst, stackSlotP src)
 4990 %{
 4991   match(Set dst src);
 4992 
 4993   ins_cost(125);
 4994   format %{ "movq    $dst, $src\t# ptr stk" %}
 4995   ins_encode %{
 4996     __ movq($dst$$Register, $src$$Address);
 4997   %}
 4998   ins_pipe(ialu_reg_mem);
 4999 %}
 5000 
 5001 instruct loadSSF(regF dst, stackSlotF src)
 5002 %{
 5003   match(Set dst src);
 5004 
 5005   ins_cost(125);
 5006   format %{ "movss   $dst, $src\t# float stk" %}
 5007   ins_encode %{
 5008     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5009   %}
 5010   ins_pipe(pipe_slow); // XXX
 5011 %}
 5012 
 5013 // Use the same format since predicate() can not be used here.
 5014 instruct loadSSD(regD dst, stackSlotD src)
 5015 %{
 5016   match(Set dst src);
 5017 
 5018   ins_cost(125);
 5019   format %{ "movsd   $dst, $src\t# double stk" %}
 5020   ins_encode  %{
 5021     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5022   %}
 5023   ins_pipe(pipe_slow); // XXX
 5024 %}
 5025 
 5026 // Prefetch instructions for allocation.
 5027 // Must be safe to execute with invalid address (cannot fault).
 5028 
 5029 instruct prefetchAlloc( memory mem ) %{
 5030   predicate(AllocatePrefetchInstr==3);
 5031   match(PrefetchAllocation mem);
 5032   ins_cost(125);
 5033 
 5034   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5035   ins_encode %{
 5036     __ prefetchw($mem$$Address);
 5037   %}
 5038   ins_pipe(ialu_mem);
 5039 %}
 5040 
 5041 instruct prefetchAllocNTA( memory mem ) %{
 5042   predicate(AllocatePrefetchInstr==0);
 5043   match(PrefetchAllocation mem);
 5044   ins_cost(125);
 5045 
 5046   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5047   ins_encode %{
 5048     __ prefetchnta($mem$$Address);
 5049   %}
 5050   ins_pipe(ialu_mem);
 5051 %}
 5052 
 5053 instruct prefetchAllocT0( memory mem ) %{
 5054   predicate(AllocatePrefetchInstr==1);
 5055   match(PrefetchAllocation mem);
 5056   ins_cost(125);
 5057 
 5058   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5059   ins_encode %{
 5060     __ prefetcht0($mem$$Address);
 5061   %}
 5062   ins_pipe(ialu_mem);
 5063 %}
 5064 
 5065 instruct prefetchAllocT2( memory mem ) %{
 5066   predicate(AllocatePrefetchInstr==2);
 5067   match(PrefetchAllocation mem);
 5068   ins_cost(125);
 5069 
 5070   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5071   ins_encode %{
 5072     __ prefetcht2($mem$$Address);
 5073   %}
 5074   ins_pipe(ialu_mem);
 5075 %}
 5076 
 5077 //----------Store Instructions-------------------------------------------------
 5078 
 5079 // Store Byte
 5080 instruct storeB(memory mem, rRegI src)
 5081 %{
 5082   match(Set mem (StoreB mem src));
 5083 
 5084   ins_cost(125); // XXX
 5085   format %{ "movb    $mem, $src\t# byte" %}
 5086   ins_encode %{
 5087     __ movb($mem$$Address, $src$$Register);
 5088   %}
 5089   ins_pipe(ialu_mem_reg);
 5090 %}
 5091 
 5092 // Store Char/Short
 5093 instruct storeC(memory mem, rRegI src)
 5094 %{
 5095   match(Set mem (StoreC mem src));
 5096 
 5097   ins_cost(125); // XXX
 5098   format %{ "movw    $mem, $src\t# char/short" %}
 5099   ins_encode %{
 5100     __ movw($mem$$Address, $src$$Register);
 5101   %}
 5102   ins_pipe(ialu_mem_reg);
 5103 %}
 5104 
 5105 // Store Integer
 5106 instruct storeI(memory mem, rRegI src)
 5107 %{
 5108   match(Set mem (StoreI mem src));
 5109 
 5110   ins_cost(125); // XXX
 5111   format %{ "movl    $mem, $src\t# int" %}
 5112   ins_encode %{
 5113     __ movl($mem$$Address, $src$$Register);
 5114   %}
 5115   ins_pipe(ialu_mem_reg);
 5116 %}
 5117 
 5118 // Store Long
 5119 instruct storeL(memory mem, rRegL src)
 5120 %{
 5121   match(Set mem (StoreL mem src));
 5122 
 5123   ins_cost(125); // XXX
 5124   format %{ "movq    $mem, $src\t# long" %}
 5125   ins_encode %{
 5126     __ movq($mem$$Address, $src$$Register);
 5127   %}
 5128   ins_pipe(ialu_mem_reg); // XXX
 5129 %}
 5130 
 5131 // Store Pointer
 5132 instruct storeP(memory mem, any_RegP src)
 5133 %{
 5134   predicate(n->as_Store()->barrier_data() == 0);
 5135   match(Set mem (StoreP mem src));
 5136 
 5137   ins_cost(125); // XXX
 5138   format %{ "movq    $mem, $src\t# ptr" %}
 5139   ins_encode %{
 5140     __ movq($mem$$Address, $src$$Register);
 5141   %}
 5142   ins_pipe(ialu_mem_reg);
 5143 %}
 5144 
 5145 instruct storeImmP0(memory mem, immP0 zero)
 5146 %{
 5147   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5148   match(Set mem (StoreP mem zero));
 5149 
 5150   ins_cost(125); // XXX
 5151   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5152   ins_encode %{
 5153     __ movq($mem$$Address, r12);
 5154   %}
 5155   ins_pipe(ialu_mem_reg);
 5156 %}
 5157 
 5158 // Store Null Pointer, mark word, or other simple pointer constant.
 5159 instruct storeImmP(memory mem, immP31 src)
 5160 %{
 5161   predicate(n->as_Store()->barrier_data() == 0);
 5162   match(Set mem (StoreP mem src));
 5163 
 5164   ins_cost(150); // XXX
 5165   format %{ "movq    $mem, $src\t# ptr" %}
 5166   ins_encode %{
 5167     __ movq($mem$$Address, $src$$constant);
 5168   %}
 5169   ins_pipe(ialu_mem_imm);
 5170 %}
 5171 
 5172 // Store Compressed Pointer
 5173 instruct storeN(memory mem, rRegN src)
 5174 %{
 5175   predicate(n->as_Store()->barrier_data() == 0);
 5176   match(Set mem (StoreN mem src));
 5177 
 5178   ins_cost(125); // XXX
 5179   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5180   ins_encode %{
 5181     __ movl($mem$$Address, $src$$Register);
 5182   %}
 5183   ins_pipe(ialu_mem_reg);
 5184 %}
 5185 
 5186 instruct storeNKlass(memory mem, rRegN src)
 5187 %{
 5188   match(Set mem (StoreNKlass mem src));
 5189 
 5190   ins_cost(125); // XXX
 5191   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5192   ins_encode %{
 5193     __ movl($mem$$Address, $src$$Register);
 5194   %}
 5195   ins_pipe(ialu_mem_reg);
 5196 %}
 5197 
 5198 instruct storeImmN0(memory mem, immN0 zero)
 5199 %{
 5200   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5201   match(Set mem (StoreN mem zero));
 5202 
 5203   ins_cost(125); // XXX
 5204   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5205   ins_encode %{
 5206     __ movl($mem$$Address, r12);
 5207   %}
 5208   ins_pipe(ialu_mem_reg);
 5209 %}
 5210 
 5211 instruct storeImmN(memory mem, immN src)
 5212 %{
 5213   predicate(n->as_Store()->barrier_data() == 0);
 5214   match(Set mem (StoreN mem src));
 5215 
 5216   ins_cost(150); // XXX
 5217   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5218   ins_encode %{
 5219     address con = (address)$src$$constant;
 5220     if (con == nullptr) {
 5221       __ movl($mem$$Address, 0);
 5222     } else {
 5223       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5224     }
 5225   %}
 5226   ins_pipe(ialu_mem_imm);
 5227 %}
 5228 
 5229 instruct storeImmNKlass(memory mem, immNKlass src)
 5230 %{
 5231   match(Set mem (StoreNKlass mem src));
 5232 
 5233   ins_cost(150); // XXX
 5234   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5235   ins_encode %{
 5236     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5237   %}
 5238   ins_pipe(ialu_mem_imm);
 5239 %}
 5240 
 5241 // Store Integer Immediate
 5242 instruct storeImmI0(memory mem, immI_0 zero)
 5243 %{
 5244   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5245   match(Set mem (StoreI mem zero));
 5246 
 5247   ins_cost(125); // XXX
 5248   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5249   ins_encode %{
 5250     __ movl($mem$$Address, r12);
 5251   %}
 5252   ins_pipe(ialu_mem_reg);
 5253 %}
 5254 
 5255 instruct storeImmI(memory mem, immI src)
 5256 %{
 5257   match(Set mem (StoreI mem src));
 5258 
 5259   ins_cost(150);
 5260   format %{ "movl    $mem, $src\t# int" %}
 5261   ins_encode %{
 5262     __ movl($mem$$Address, $src$$constant);
 5263   %}
 5264   ins_pipe(ialu_mem_imm);
 5265 %}
 5266 
 5267 // Store Long Immediate
 5268 instruct storeImmL0(memory mem, immL0 zero)
 5269 %{
 5270   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5271   match(Set mem (StoreL mem zero));
 5272 
 5273   ins_cost(125); // XXX
 5274   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5275   ins_encode %{
 5276     __ movq($mem$$Address, r12);
 5277   %}
 5278   ins_pipe(ialu_mem_reg);
 5279 %}
 5280 
 5281 instruct storeImmL(memory mem, immL32 src)
 5282 %{
 5283   match(Set mem (StoreL mem src));
 5284 
 5285   ins_cost(150);
 5286   format %{ "movq    $mem, $src\t# long" %}
 5287   ins_encode %{
 5288     __ movq($mem$$Address, $src$$constant);
 5289   %}
 5290   ins_pipe(ialu_mem_imm);
 5291 %}
 5292 
 5293 // Store Short/Char Immediate
 5294 instruct storeImmC0(memory mem, immI_0 zero)
 5295 %{
 5296   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5297   match(Set mem (StoreC mem zero));
 5298 
 5299   ins_cost(125); // XXX
 5300   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5301   ins_encode %{
 5302     __ movw($mem$$Address, r12);
 5303   %}
 5304   ins_pipe(ialu_mem_reg);
 5305 %}
 5306 
 5307 instruct storeImmI16(memory mem, immI16 src)
 5308 %{
 5309   predicate(UseStoreImmI16);
 5310   match(Set mem (StoreC mem src));
 5311 
 5312   ins_cost(150);
 5313   format %{ "movw    $mem, $src\t# short/char" %}
 5314   ins_encode %{
 5315     __ movw($mem$$Address, $src$$constant);
 5316   %}
 5317   ins_pipe(ialu_mem_imm);
 5318 %}
 5319 
 5320 // Store Byte Immediate
 5321 instruct storeImmB0(memory mem, immI_0 zero)
 5322 %{
 5323   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5324   match(Set mem (StoreB mem zero));
 5325 
 5326   ins_cost(125); // XXX
 5327   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5328   ins_encode %{
 5329     __ movb($mem$$Address, r12);
 5330   %}
 5331   ins_pipe(ialu_mem_reg);
 5332 %}
 5333 
 5334 instruct storeImmB(memory mem, immI8 src)
 5335 %{
 5336   match(Set mem (StoreB mem src));
 5337 
 5338   ins_cost(150); // XXX
 5339   format %{ "movb    $mem, $src\t# byte" %}
 5340   ins_encode %{
 5341     __ movb($mem$$Address, $src$$constant);
 5342   %}
 5343   ins_pipe(ialu_mem_imm);
 5344 %}
 5345 
 5346 // Store Float
 5347 instruct storeF(memory mem, regF src)
 5348 %{
 5349   match(Set mem (StoreF mem src));
 5350 
 5351   ins_cost(95); // XXX
 5352   format %{ "movss   $mem, $src\t# float" %}
 5353   ins_encode %{
 5354     __ movflt($mem$$Address, $src$$XMMRegister);
 5355   %}
 5356   ins_pipe(pipe_slow); // XXX
 5357 %}
 5358 
 5359 // Store immediate Float value (it is faster than store from XMM register)
 5360 instruct storeF0(memory mem, immF0 zero)
 5361 %{
 5362   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5363   match(Set mem (StoreF mem zero));
 5364 
 5365   ins_cost(25); // XXX
 5366   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5367   ins_encode %{
 5368     __ movl($mem$$Address, r12);
 5369   %}
 5370   ins_pipe(ialu_mem_reg);
 5371 %}
 5372 
 5373 instruct storeF_imm(memory mem, immF src)
 5374 %{
 5375   match(Set mem (StoreF mem src));
 5376 
 5377   ins_cost(50);
 5378   format %{ "movl    $mem, $src\t# float" %}
 5379   ins_encode %{
 5380     __ movl($mem$$Address, jint_cast($src$$constant));
 5381   %}
 5382   ins_pipe(ialu_mem_imm);
 5383 %}
 5384 
 5385 // Store Double
 5386 instruct storeD(memory mem, regD src)
 5387 %{
 5388   match(Set mem (StoreD mem src));
 5389 
 5390   ins_cost(95); // XXX
 5391   format %{ "movsd   $mem, $src\t# double" %}
 5392   ins_encode %{
 5393     __ movdbl($mem$$Address, $src$$XMMRegister);
 5394   %}
 5395   ins_pipe(pipe_slow); // XXX
 5396 %}
 5397 
 5398 // Store immediate double 0.0 (it is faster than store from XMM register)
 5399 instruct storeD0_imm(memory mem, immD0 src)
 5400 %{
 5401   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5402   match(Set mem (StoreD mem src));
 5403 
 5404   ins_cost(50);
 5405   format %{ "movq    $mem, $src\t# double 0." %}
 5406   ins_encode %{
 5407     __ movq($mem$$Address, $src$$constant);
 5408   %}
 5409   ins_pipe(ialu_mem_imm);
 5410 %}
 5411 
 5412 instruct storeD0(memory mem, immD0 zero)
 5413 %{
 5414   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5415   match(Set mem (StoreD mem zero));
 5416 
 5417   ins_cost(25); // XXX
 5418   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5419   ins_encode %{
 5420     __ movq($mem$$Address, r12);
 5421   %}
 5422   ins_pipe(ialu_mem_reg);
 5423 %}
 5424 
 5425 instruct storeSSI(stackSlotI dst, rRegI src)
 5426 %{
 5427   match(Set dst src);
 5428 
 5429   ins_cost(100);
 5430   format %{ "movl    $dst, $src\t# int stk" %}
 5431   ins_encode %{
 5432     __ movl($dst$$Address, $src$$Register);
 5433   %}
 5434   ins_pipe( ialu_mem_reg );
 5435 %}
 5436 
 5437 instruct storeSSL(stackSlotL dst, rRegL src)
 5438 %{
 5439   match(Set dst src);
 5440 
 5441   ins_cost(100);
 5442   format %{ "movq    $dst, $src\t# long stk" %}
 5443   ins_encode %{
 5444     __ movq($dst$$Address, $src$$Register);
 5445   %}
 5446   ins_pipe(ialu_mem_reg);
 5447 %}
 5448 
 5449 instruct storeSSP(stackSlotP dst, rRegP src)
 5450 %{
 5451   match(Set dst src);
 5452 
 5453   ins_cost(100);
 5454   format %{ "movq    $dst, $src\t# ptr stk" %}
 5455   ins_encode %{
 5456     __ movq($dst$$Address, $src$$Register);
 5457   %}
 5458   ins_pipe(ialu_mem_reg);
 5459 %}
 5460 
 5461 instruct storeSSF(stackSlotF dst, regF src)
 5462 %{
 5463   match(Set dst src);
 5464 
 5465   ins_cost(95); // XXX
 5466   format %{ "movss   $dst, $src\t# float stk" %}
 5467   ins_encode %{
 5468     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5469   %}
 5470   ins_pipe(pipe_slow); // XXX
 5471 %}
 5472 
 5473 instruct storeSSD(stackSlotD dst, regD src)
 5474 %{
 5475   match(Set dst src);
 5476 
 5477   ins_cost(95); // XXX
 5478   format %{ "movsd   $dst, $src\t# double stk" %}
 5479   ins_encode %{
 5480     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5481   %}
 5482   ins_pipe(pipe_slow); // XXX
 5483 %}
 5484 
 5485 instruct cacheWB(indirect addr)
 5486 %{
 5487   predicate(VM_Version::supports_data_cache_line_flush());
 5488   match(CacheWB addr);
 5489 
 5490   ins_cost(100);
 5491   format %{"cache wb $addr" %}
 5492   ins_encode %{
 5493     assert($addr->index_position() < 0, "should be");
 5494     assert($addr$$disp == 0, "should be");
 5495     __ cache_wb(Address($addr$$base$$Register, 0));
 5496   %}
 5497   ins_pipe(pipe_slow); // XXX
 5498 %}
 5499 
 5500 instruct cacheWBPreSync()
 5501 %{
 5502   predicate(VM_Version::supports_data_cache_line_flush());
 5503   match(CacheWBPreSync);
 5504 
 5505   ins_cost(100);
 5506   format %{"cache wb presync" %}
 5507   ins_encode %{
 5508     __ cache_wbsync(true);
 5509   %}
 5510   ins_pipe(pipe_slow); // XXX
 5511 %}
 5512 
 5513 instruct cacheWBPostSync()
 5514 %{
 5515   predicate(VM_Version::supports_data_cache_line_flush());
 5516   match(CacheWBPostSync);
 5517 
 5518   ins_cost(100);
 5519   format %{"cache wb postsync" %}
 5520   ins_encode %{
 5521     __ cache_wbsync(false);
 5522   %}
 5523   ins_pipe(pipe_slow); // XXX
 5524 %}
 5525 
 5526 //----------BSWAP Instructions-------------------------------------------------
 5527 instruct bytes_reverse_int(rRegI dst) %{
 5528   match(Set dst (ReverseBytesI dst));
 5529 
 5530   format %{ "bswapl  $dst" %}
 5531   ins_encode %{
 5532     __ bswapl($dst$$Register);
 5533   %}
 5534   ins_pipe( ialu_reg );
 5535 %}
 5536 
 5537 instruct bytes_reverse_long(rRegL dst) %{
 5538   match(Set dst (ReverseBytesL dst));
 5539 
 5540   format %{ "bswapq  $dst" %}
 5541   ins_encode %{
 5542     __ bswapq($dst$$Register);
 5543   %}
 5544   ins_pipe( ialu_reg);
 5545 %}
 5546 
 5547 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5548   match(Set dst (ReverseBytesUS dst));
 5549   effect(KILL cr);
 5550 
 5551   format %{ "bswapl  $dst\n\t"
 5552             "shrl    $dst,16\n\t" %}
 5553   ins_encode %{
 5554     __ bswapl($dst$$Register);
 5555     __ shrl($dst$$Register, 16);
 5556   %}
 5557   ins_pipe( ialu_reg );
 5558 %}
 5559 
 5560 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5561   match(Set dst (ReverseBytesS dst));
 5562   effect(KILL cr);
 5563 
 5564   format %{ "bswapl  $dst\n\t"
 5565             "sar     $dst,16\n\t" %}
 5566   ins_encode %{
 5567     __ bswapl($dst$$Register);
 5568     __ sarl($dst$$Register, 16);
 5569   %}
 5570   ins_pipe( ialu_reg );
 5571 %}
 5572 
 5573 //---------- Zeros Count Instructions ------------------------------------------
 5574 
 5575 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5576   predicate(UseCountLeadingZerosInstruction);
 5577   match(Set dst (CountLeadingZerosI src));
 5578   effect(KILL cr);
 5579 
 5580   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5581   ins_encode %{
 5582     __ lzcntl($dst$$Register, $src$$Register);
 5583   %}
 5584   ins_pipe(ialu_reg);
 5585 %}
 5586 
 5587 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5588   predicate(UseCountLeadingZerosInstruction);
 5589   match(Set dst (CountLeadingZerosI (LoadI src)));
 5590   effect(KILL cr);
 5591   ins_cost(175);
 5592   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5593   ins_encode %{
 5594     __ lzcntl($dst$$Register, $src$$Address);
 5595   %}
 5596   ins_pipe(ialu_reg_mem);
 5597 %}
 5598 
 5599 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5600   predicate(!UseCountLeadingZerosInstruction);
 5601   match(Set dst (CountLeadingZerosI src));
 5602   effect(KILL cr);
 5603 
 5604   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5605             "jnz     skip\n\t"
 5606             "movl    $dst, -1\n"
 5607       "skip:\n\t"
 5608             "negl    $dst\n\t"
 5609             "addl    $dst, 31" %}
 5610   ins_encode %{
 5611     Register Rdst = $dst$$Register;
 5612     Register Rsrc = $src$$Register;
 5613     Label skip;
 5614     __ bsrl(Rdst, Rsrc);
 5615     __ jccb(Assembler::notZero, skip);
 5616     __ movl(Rdst, -1);
 5617     __ bind(skip);
 5618     __ negl(Rdst);
 5619     __ addl(Rdst, BitsPerInt - 1);
 5620   %}
 5621   ins_pipe(ialu_reg);
 5622 %}
 5623 
 5624 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5625   predicate(UseCountLeadingZerosInstruction);
 5626   match(Set dst (CountLeadingZerosL src));
 5627   effect(KILL cr);
 5628 
 5629   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5630   ins_encode %{
 5631     __ lzcntq($dst$$Register, $src$$Register);
 5632   %}
 5633   ins_pipe(ialu_reg);
 5634 %}
 5635 
 5636 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5637   predicate(UseCountLeadingZerosInstruction);
 5638   match(Set dst (CountLeadingZerosL (LoadL src)));
 5639   effect(KILL cr);
 5640   ins_cost(175);
 5641   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5642   ins_encode %{
 5643     __ lzcntq($dst$$Register, $src$$Address);
 5644   %}
 5645   ins_pipe(ialu_reg_mem);
 5646 %}
 5647 
 5648 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5649   predicate(!UseCountLeadingZerosInstruction);
 5650   match(Set dst (CountLeadingZerosL src));
 5651   effect(KILL cr);
 5652 
 5653   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5654             "jnz     skip\n\t"
 5655             "movl    $dst, -1\n"
 5656       "skip:\n\t"
 5657             "negl    $dst\n\t"
 5658             "addl    $dst, 63" %}
 5659   ins_encode %{
 5660     Register Rdst = $dst$$Register;
 5661     Register Rsrc = $src$$Register;
 5662     Label skip;
 5663     __ bsrq(Rdst, Rsrc);
 5664     __ jccb(Assembler::notZero, skip);
 5665     __ movl(Rdst, -1);
 5666     __ bind(skip);
 5667     __ negl(Rdst);
 5668     __ addl(Rdst, BitsPerLong - 1);
 5669   %}
 5670   ins_pipe(ialu_reg);
 5671 %}
 5672 
 5673 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5674   predicate(UseCountTrailingZerosInstruction);
 5675   match(Set dst (CountTrailingZerosI src));
 5676   effect(KILL cr);
 5677 
 5678   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5679   ins_encode %{
 5680     __ tzcntl($dst$$Register, $src$$Register);
 5681   %}
 5682   ins_pipe(ialu_reg);
 5683 %}
 5684 
 5685 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5686   predicate(UseCountTrailingZerosInstruction);
 5687   match(Set dst (CountTrailingZerosI (LoadI src)));
 5688   effect(KILL cr);
 5689   ins_cost(175);
 5690   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5691   ins_encode %{
 5692     __ tzcntl($dst$$Register, $src$$Address);
 5693   %}
 5694   ins_pipe(ialu_reg_mem);
 5695 %}
 5696 
 5697 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5698   predicate(!UseCountTrailingZerosInstruction);
 5699   match(Set dst (CountTrailingZerosI src));
 5700   effect(KILL cr);
 5701 
 5702   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5703             "jnz     done\n\t"
 5704             "movl    $dst, 32\n"
 5705       "done:" %}
 5706   ins_encode %{
 5707     Register Rdst = $dst$$Register;
 5708     Label done;
 5709     __ bsfl(Rdst, $src$$Register);
 5710     __ jccb(Assembler::notZero, done);
 5711     __ movl(Rdst, BitsPerInt);
 5712     __ bind(done);
 5713   %}
 5714   ins_pipe(ialu_reg);
 5715 %}
 5716 
 5717 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5718   predicate(UseCountTrailingZerosInstruction);
 5719   match(Set dst (CountTrailingZerosL src));
 5720   effect(KILL cr);
 5721 
 5722   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5723   ins_encode %{
 5724     __ tzcntq($dst$$Register, $src$$Register);
 5725   %}
 5726   ins_pipe(ialu_reg);
 5727 %}
 5728 
 5729 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5730   predicate(UseCountTrailingZerosInstruction);
 5731   match(Set dst (CountTrailingZerosL (LoadL src)));
 5732   effect(KILL cr);
 5733   ins_cost(175);
 5734   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5735   ins_encode %{
 5736     __ tzcntq($dst$$Register, $src$$Address);
 5737   %}
 5738   ins_pipe(ialu_reg_mem);
 5739 %}
 5740 
 5741 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5742   predicate(!UseCountTrailingZerosInstruction);
 5743   match(Set dst (CountTrailingZerosL src));
 5744   effect(KILL cr);
 5745 
 5746   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5747             "jnz     done\n\t"
 5748             "movl    $dst, 64\n"
 5749       "done:" %}
 5750   ins_encode %{
 5751     Register Rdst = $dst$$Register;
 5752     Label done;
 5753     __ bsfq(Rdst, $src$$Register);
 5754     __ jccb(Assembler::notZero, done);
 5755     __ movl(Rdst, BitsPerLong);
 5756     __ bind(done);
 5757   %}
 5758   ins_pipe(ialu_reg);
 5759 %}
 5760 
 5761 //--------------- Reverse Operation Instructions ----------------
 5762 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5763   predicate(!VM_Version::supports_gfni());
 5764   match(Set dst (ReverseI src));
 5765   effect(TEMP dst, TEMP rtmp, KILL cr);
 5766   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5767   ins_encode %{
 5768     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5769   %}
 5770   ins_pipe( ialu_reg );
 5771 %}
 5772 
 5773 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5774   predicate(VM_Version::supports_gfni());
 5775   match(Set dst (ReverseI src));
 5776   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5777   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5778   ins_encode %{
 5779     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5780   %}
 5781   ins_pipe( ialu_reg );
 5782 %}
 5783 
 5784 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5785   predicate(!VM_Version::supports_gfni());
 5786   match(Set dst (ReverseL src));
 5787   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5788   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5789   ins_encode %{
 5790     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5791   %}
 5792   ins_pipe( ialu_reg );
 5793 %}
 5794 
 5795 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5796   predicate(VM_Version::supports_gfni());
 5797   match(Set dst (ReverseL src));
 5798   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5799   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5800   ins_encode %{
 5801     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5802   %}
 5803   ins_pipe( ialu_reg );
 5804 %}
 5805 
 5806 //---------- Population Count Instructions -------------------------------------
 5807 
 5808 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5809   predicate(UsePopCountInstruction);
 5810   match(Set dst (PopCountI src));
 5811   effect(KILL cr);
 5812 
 5813   format %{ "popcnt  $dst, $src" %}
 5814   ins_encode %{
 5815     __ popcntl($dst$$Register, $src$$Register);
 5816   %}
 5817   ins_pipe(ialu_reg);
 5818 %}
 5819 
 5820 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5821   predicate(UsePopCountInstruction);
 5822   match(Set dst (PopCountI (LoadI mem)));
 5823   effect(KILL cr);
 5824 
 5825   format %{ "popcnt  $dst, $mem" %}
 5826   ins_encode %{
 5827     __ popcntl($dst$$Register, $mem$$Address);
 5828   %}
 5829   ins_pipe(ialu_reg);
 5830 %}
 5831 
 5832 // Note: Long.bitCount(long) returns an int.
 5833 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5834   predicate(UsePopCountInstruction);
 5835   match(Set dst (PopCountL src));
 5836   effect(KILL cr);
 5837 
 5838   format %{ "popcnt  $dst, $src" %}
 5839   ins_encode %{
 5840     __ popcntq($dst$$Register, $src$$Register);
 5841   %}
 5842   ins_pipe(ialu_reg);
 5843 %}
 5844 
 5845 // Note: Long.bitCount(long) returns an int.
 5846 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5847   predicate(UsePopCountInstruction);
 5848   match(Set dst (PopCountL (LoadL mem)));
 5849   effect(KILL cr);
 5850 
 5851   format %{ "popcnt  $dst, $mem" %}
 5852   ins_encode %{
 5853     __ popcntq($dst$$Register, $mem$$Address);
 5854   %}
 5855   ins_pipe(ialu_reg);
 5856 %}
 5857 
 5858 
 5859 //----------MemBar Instructions-----------------------------------------------
 5860 // Memory barrier flavors
 5861 
 5862 instruct membar_acquire()
 5863 %{
 5864   match(MemBarAcquire);
 5865   match(LoadFence);
 5866   ins_cost(0);
 5867 
 5868   size(0);
 5869   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5870   ins_encode();
 5871   ins_pipe(empty);
 5872 %}
 5873 
 5874 instruct membar_acquire_lock()
 5875 %{
 5876   match(MemBarAcquireLock);
 5877   ins_cost(0);
 5878 
 5879   size(0);
 5880   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5881   ins_encode();
 5882   ins_pipe(empty);
 5883 %}
 5884 
 5885 instruct membar_release()
 5886 %{
 5887   match(MemBarRelease);
 5888   match(StoreFence);
 5889   ins_cost(0);
 5890 
 5891   size(0);
 5892   format %{ "MEMBAR-release ! (empty encoding)" %}
 5893   ins_encode();
 5894   ins_pipe(empty);
 5895 %}
 5896 
 5897 instruct membar_release_lock()
 5898 %{
 5899   match(MemBarReleaseLock);
 5900   ins_cost(0);
 5901 
 5902   size(0);
 5903   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5904   ins_encode();
 5905   ins_pipe(empty);
 5906 %}
 5907 
 5908 instruct membar_volatile(rFlagsReg cr) %{
 5909   match(MemBarVolatile);
 5910   effect(KILL cr);
 5911   ins_cost(400);
 5912 
 5913   format %{
 5914     $$template
 5915     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5916   %}
 5917   ins_encode %{
 5918     __ membar(Assembler::StoreLoad);
 5919   %}
 5920   ins_pipe(pipe_slow);
 5921 %}
 5922 
 5923 instruct unnecessary_membar_volatile()
 5924 %{
 5925   match(MemBarVolatile);
 5926   predicate(Matcher::post_store_load_barrier(n));
 5927   ins_cost(0);
 5928 
 5929   size(0);
 5930   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5931   ins_encode();
 5932   ins_pipe(empty);
 5933 %}
 5934 
 5935 instruct membar_storestore() %{
 5936   match(MemBarStoreStore);
 5937   match(StoreStoreFence);
 5938   ins_cost(0);
 5939 
 5940   size(0);
 5941   format %{ "MEMBAR-storestore (empty encoding)" %}
 5942   ins_encode( );
 5943   ins_pipe(empty);
 5944 %}
 5945 
 5946 //----------Move Instructions--------------------------------------------------
 5947 
 5948 instruct castX2P(rRegP dst, rRegL src)
 5949 %{
 5950   match(Set dst (CastX2P src));
 5951 
 5952   format %{ "movq    $dst, $src\t# long->ptr" %}
 5953   ins_encode %{
 5954     if ($dst$$reg != $src$$reg) {
 5955       __ movptr($dst$$Register, $src$$Register);
 5956     }
 5957   %}
 5958   ins_pipe(ialu_reg_reg); // XXX
 5959 %}
 5960 
 5961 instruct castI2N(rRegN dst, rRegI src)
 5962 %{
 5963   match(Set dst (CastI2N src));
 5964 
 5965   format %{ "movq    $dst, $src\t# int -> narrow ptr" %}
 5966   ins_encode %{
 5967     if ($dst$$reg != $src$$reg) {
 5968       __ movl($dst$$Register, $src$$Register);
 5969     }
 5970   %}
 5971   ins_pipe(ialu_reg_reg); // XXX
 5972 %}
 5973 
 5974 instruct castN2X(rRegL dst, rRegN src)
 5975 %{
 5976   match(Set dst (CastP2X src));
 5977 
 5978   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5979   ins_encode %{
 5980     if ($dst$$reg != $src$$reg) {
 5981       __ movptr($dst$$Register, $src$$Register);
 5982     }
 5983   %}
 5984   ins_pipe(ialu_reg_reg); // XXX
 5985 %}
 5986 
 5987 instruct castP2X(rRegL dst, rRegP src)
 5988 %{
 5989   match(Set dst (CastP2X src));
 5990 
 5991   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5992   ins_encode %{
 5993     if ($dst$$reg != $src$$reg) {
 5994       __ movptr($dst$$Register, $src$$Register);
 5995     }
 5996   %}
 5997   ins_pipe(ialu_reg_reg); // XXX
 5998 %}
 5999 
 6000 // Convert oop into int for vectors alignment masking
 6001 instruct convP2I(rRegI dst, rRegP src)
 6002 %{
 6003   match(Set dst (ConvL2I (CastP2X src)));
 6004 
 6005   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6006   ins_encode %{
 6007     __ movl($dst$$Register, $src$$Register);
 6008   %}
 6009   ins_pipe(ialu_reg_reg); // XXX
 6010 %}
 6011 
 6012 // Convert compressed oop into int for vectors alignment masking
 6013 // in case of 32bit oops (heap < 4Gb).
 6014 instruct convN2I(rRegI dst, rRegN src)
 6015 %{
 6016   predicate(CompressedOops::shift() == 0);
 6017   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6018 
 6019   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6020   ins_encode %{
 6021     __ movl($dst$$Register, $src$$Register);
 6022   %}
 6023   ins_pipe(ialu_reg_reg); // XXX
 6024 %}
 6025 
 6026 // Convert oop pointer into compressed form
 6027 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6028   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6029   match(Set dst (EncodeP src));
 6030   effect(KILL cr);
 6031   format %{ "encode_heap_oop $dst,$src" %}
 6032   ins_encode %{
 6033     Register s = $src$$Register;
 6034     Register d = $dst$$Register;
 6035     if (s != d) {
 6036       __ movq(d, s);
 6037     }
 6038     __ encode_heap_oop(d);
 6039   %}
 6040   ins_pipe(ialu_reg_long);
 6041 %}
 6042 
 6043 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6044   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6045   match(Set dst (EncodeP src));
 6046   effect(KILL cr);
 6047   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6048   ins_encode %{
 6049     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6050   %}
 6051   ins_pipe(ialu_reg_long);
 6052 %}
 6053 
 6054 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6055   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6056             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6057   match(Set dst (DecodeN src));
 6058   effect(KILL cr);
 6059   format %{ "decode_heap_oop $dst,$src" %}
 6060   ins_encode %{
 6061     Register s = $src$$Register;
 6062     Register d = $dst$$Register;
 6063     if (s != d) {
 6064       __ movq(d, s);
 6065     }
 6066     __ decode_heap_oop(d);
 6067   %}
 6068   ins_pipe(ialu_reg_long);
 6069 %}
 6070 
 6071 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6072   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6073             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6074   match(Set dst (DecodeN src));
 6075   effect(KILL cr);
 6076   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6077   ins_encode %{
 6078     Register s = $src$$Register;
 6079     Register d = $dst$$Register;
 6080     if (s != d) {
 6081       __ decode_heap_oop_not_null(d, s);
 6082     } else {
 6083       __ decode_heap_oop_not_null(d);
 6084     }
 6085   %}
 6086   ins_pipe(ialu_reg_long);
 6087 %}
 6088 
 6089 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6090   match(Set dst (EncodePKlass src));
 6091   effect(TEMP dst, KILL cr);
 6092   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6093   ins_encode %{
 6094     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6095   %}
 6096   ins_pipe(ialu_reg_long);
 6097 %}
 6098 
 6099 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6100   match(Set dst (DecodeNKlass src));
 6101   effect(TEMP dst, KILL cr);
 6102   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6103   ins_encode %{
 6104     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6105   %}
 6106   ins_pipe(ialu_reg_long);
 6107 %}
 6108 
 6109 //----------Conditional Move---------------------------------------------------
 6110 // Jump
 6111 // dummy instruction for generating temp registers
 6112 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6113   match(Jump (LShiftL switch_val shift));
 6114   ins_cost(350);
 6115   predicate(false);
 6116   effect(TEMP dest);
 6117 
 6118   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6119             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6120   ins_encode %{
 6121     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6122     // to do that and the compiler is using that register as one it can allocate.
 6123     // So we build it all by hand.
 6124     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6125     // ArrayAddress dispatch(table, index);
 6126     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6127     __ lea($dest$$Register, $constantaddress);
 6128     __ jmp(dispatch);
 6129   %}
 6130   ins_pipe(pipe_jmp);
 6131 %}
 6132 
 6133 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6134   match(Jump (AddL (LShiftL switch_val shift) offset));
 6135   ins_cost(350);
 6136   effect(TEMP dest);
 6137 
 6138   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6139             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6140   ins_encode %{
 6141     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6142     // to do that and the compiler is using that register as one it can allocate.
 6143     // So we build it all by hand.
 6144     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6145     // ArrayAddress dispatch(table, index);
 6146     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6147     __ lea($dest$$Register, $constantaddress);
 6148     __ jmp(dispatch);
 6149   %}
 6150   ins_pipe(pipe_jmp);
 6151 %}
 6152 
 6153 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6154   match(Jump switch_val);
 6155   ins_cost(350);
 6156   effect(TEMP dest);
 6157 
 6158   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6159             "jmp     [$dest + $switch_val]\n\t" %}
 6160   ins_encode %{
 6161     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6162     // to do that and the compiler is using that register as one it can allocate.
 6163     // So we build it all by hand.
 6164     // Address index(noreg, switch_reg, Address::times_1);
 6165     // ArrayAddress dispatch(table, index);
 6166     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6167     __ lea($dest$$Register, $constantaddress);
 6168     __ jmp(dispatch);
 6169   %}
 6170   ins_pipe(pipe_jmp);
 6171 %}
 6172 
 6173 // Conditional move
 6174 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6175 %{
 6176   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6177   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6178 
 6179   ins_cost(100); // XXX
 6180   format %{ "setbn$cop $dst\t# signed, int" %}
 6181   ins_encode %{
 6182     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6183     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6184   %}
 6185   ins_pipe(ialu_reg);
 6186 %}
 6187 
 6188 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6189 %{
 6190   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6191 
 6192   ins_cost(200); // XXX
 6193   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6194   ins_encode %{
 6195     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6196   %}
 6197   ins_pipe(pipe_cmov_reg);
 6198 %}
 6199 
 6200 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6201 %{
 6202   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6203   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6204 
 6205   ins_cost(100); // XXX
 6206   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6207   ins_encode %{
 6208     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6209     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6210   %}
 6211   ins_pipe(ialu_reg);
 6212 %}
 6213 
 6214 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6215   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6216 
 6217   ins_cost(200); // XXX
 6218   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6219   ins_encode %{
 6220     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6221   %}
 6222   ins_pipe(pipe_cmov_reg);
 6223 %}
 6224 
 6225 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6226 %{
 6227   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6228   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6229 
 6230   ins_cost(100); // XXX
 6231   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6232   ins_encode %{
 6233     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6234     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6235   %}
 6236   ins_pipe(ialu_reg);
 6237 %}
 6238 
 6239 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6240   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6241   ins_cost(200);
 6242   expand %{
 6243     cmovI_regU(cop, cr, dst, src);
 6244   %}
 6245 %}
 6246 
 6247 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6248   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6249   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6250 
 6251   ins_cost(200); // XXX
 6252   format %{ "cmovpl  $dst, $src\n\t"
 6253             "cmovnel $dst, $src" %}
 6254   ins_encode %{
 6255     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6256     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6257   %}
 6258   ins_pipe(pipe_cmov_reg);
 6259 %}
 6260 
 6261 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6262 // inputs of the CMove
 6263 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6264   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6265   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6266 
 6267   ins_cost(200); // XXX
 6268   format %{ "cmovpl  $dst, $src\n\t"
 6269             "cmovnel $dst, $src" %}
 6270   ins_encode %{
 6271     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6272     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6273   %}
 6274   ins_pipe(pipe_cmov_reg);
 6275 %}
 6276 
 6277 // Conditional move
 6278 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6279   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6280 
 6281   ins_cost(250); // XXX
 6282   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6283   ins_encode %{
 6284     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6285   %}
 6286   ins_pipe(pipe_cmov_mem);
 6287 %}
 6288 
 6289 // Conditional move
 6290 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6291 %{
 6292   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6293 
 6294   ins_cost(250); // XXX
 6295   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6296   ins_encode %{
 6297     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6298   %}
 6299   ins_pipe(pipe_cmov_mem);
 6300 %}
 6301 
 6302 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6303   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6304   ins_cost(250);
 6305   expand %{
 6306     cmovI_memU(cop, cr, dst, src);
 6307   %}
 6308 %}
 6309 
 6310 // Conditional move
 6311 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6312 %{
 6313   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6314 
 6315   ins_cost(200); // XXX
 6316   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6317   ins_encode %{
 6318     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6319   %}
 6320   ins_pipe(pipe_cmov_reg);
 6321 %}
 6322 
 6323 // Conditional move
 6324 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6325 %{
 6326   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6327 
 6328   ins_cost(200); // XXX
 6329   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6330   ins_encode %{
 6331     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6332   %}
 6333   ins_pipe(pipe_cmov_reg);
 6334 %}
 6335 
 6336 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6337   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6338   ins_cost(200);
 6339   expand %{
 6340     cmovN_regU(cop, cr, dst, src);
 6341   %}
 6342 %}
 6343 
 6344 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6345   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6346   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6347 
 6348   ins_cost(200); // XXX
 6349   format %{ "cmovpl  $dst, $src\n\t"
 6350             "cmovnel $dst, $src" %}
 6351   ins_encode %{
 6352     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6353     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6354   %}
 6355   ins_pipe(pipe_cmov_reg);
 6356 %}
 6357 
 6358 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6359 // inputs of the CMove
 6360 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6361   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6362   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6363 
 6364   ins_cost(200); // XXX
 6365   format %{ "cmovpl  $dst, $src\n\t"
 6366             "cmovnel $dst, $src" %}
 6367   ins_encode %{
 6368     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6369     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6370   %}
 6371   ins_pipe(pipe_cmov_reg);
 6372 %}
 6373 
 6374 // Conditional move
 6375 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6376 %{
 6377   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6378 
 6379   ins_cost(200); // XXX
 6380   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6381   ins_encode %{
 6382     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6383   %}
 6384   ins_pipe(pipe_cmov_reg);  // XXX
 6385 %}
 6386 
 6387 // Conditional move
 6388 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6389 %{
 6390   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6391 
 6392   ins_cost(200); // XXX
 6393   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6394   ins_encode %{
 6395     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6396   %}
 6397   ins_pipe(pipe_cmov_reg); // XXX
 6398 %}
 6399 
 6400 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6401   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6402   ins_cost(200);
 6403   expand %{
 6404     cmovP_regU(cop, cr, dst, src);
 6405   %}
 6406 %}
 6407 
 6408 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6409   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6410   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6411 
 6412   ins_cost(200); // XXX
 6413   format %{ "cmovpq  $dst, $src\n\t"
 6414             "cmovneq $dst, $src" %}
 6415   ins_encode %{
 6416     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6417     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6418   %}
 6419   ins_pipe(pipe_cmov_reg);
 6420 %}
 6421 
 6422 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6423 // inputs of the CMove
 6424 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6425   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6426   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6427 
 6428   ins_cost(200); // XXX
 6429   format %{ "cmovpq  $dst, $src\n\t"
 6430             "cmovneq $dst, $src" %}
 6431   ins_encode %{
 6432     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6433     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6434   %}
 6435   ins_pipe(pipe_cmov_reg);
 6436 %}
 6437 
 6438 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6439 %{
 6440   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6441   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6442 
 6443   ins_cost(100); // XXX
 6444   format %{ "setbn$cop $dst\t# signed, long" %}
 6445   ins_encode %{
 6446     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6447     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6448   %}
 6449   ins_pipe(ialu_reg);
 6450 %}
 6451 
 6452 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6453 %{
 6454   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6455 
 6456   ins_cost(200); // XXX
 6457   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6458   ins_encode %{
 6459     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6460   %}
 6461   ins_pipe(pipe_cmov_reg);  // XXX
 6462 %}
 6463 
 6464 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6465 %{
 6466   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6467 
 6468   ins_cost(200); // XXX
 6469   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6470   ins_encode %{
 6471     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6472   %}
 6473   ins_pipe(pipe_cmov_mem);  // XXX
 6474 %}
 6475 
 6476 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6477 %{
 6478   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6479   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6480 
 6481   ins_cost(100); // XXX
 6482   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6483   ins_encode %{
 6484     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6485     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6486   %}
 6487   ins_pipe(ialu_reg);
 6488 %}
 6489 
 6490 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6491 %{
 6492   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6493 
 6494   ins_cost(200); // XXX
 6495   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6496   ins_encode %{
 6497     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6498   %}
 6499   ins_pipe(pipe_cmov_reg); // XXX
 6500 %}
 6501 
 6502 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6503 %{
 6504   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6505   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6506 
 6507   ins_cost(100); // XXX
 6508   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6509   ins_encode %{
 6510     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6511     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6512   %}
 6513   ins_pipe(ialu_reg);
 6514 %}
 6515 
 6516 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6517   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6518   ins_cost(200);
 6519   expand %{
 6520     cmovL_regU(cop, cr, dst, src);
 6521   %}
 6522 %}
 6523 
 6524 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6525   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6526   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6527 
 6528   ins_cost(200); // XXX
 6529   format %{ "cmovpq  $dst, $src\n\t"
 6530             "cmovneq $dst, $src" %}
 6531   ins_encode %{
 6532     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6533     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6534   %}
 6535   ins_pipe(pipe_cmov_reg);
 6536 %}
 6537 
 6538 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6539 // inputs of the CMove
 6540 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6541   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6542   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6543 
 6544   ins_cost(200); // XXX
 6545   format %{ "cmovpq  $dst, $src\n\t"
 6546             "cmovneq $dst, $src" %}
 6547   ins_encode %{
 6548     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6549     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6550   %}
 6551   ins_pipe(pipe_cmov_reg);
 6552 %}
 6553 
 6554 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6555 %{
 6556   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6557 
 6558   ins_cost(200); // XXX
 6559   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6560   ins_encode %{
 6561     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6562   %}
 6563   ins_pipe(pipe_cmov_mem); // XXX
 6564 %}
 6565 
 6566 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6567   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6568   ins_cost(200);
 6569   expand %{
 6570     cmovL_memU(cop, cr, dst, src);
 6571   %}
 6572 %}
 6573 
 6574 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6575 %{
 6576   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6577 
 6578   ins_cost(200); // XXX
 6579   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6580             "movss     $dst, $src\n"
 6581     "skip:" %}
 6582   ins_encode %{
 6583     Label Lskip;
 6584     // Invert sense of branch from sense of CMOV
 6585     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6586     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6587     __ bind(Lskip);
 6588   %}
 6589   ins_pipe(pipe_slow);
 6590 %}
 6591 
 6592 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6593 %{
 6594   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6595 
 6596   ins_cost(200); // XXX
 6597   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6598             "movss     $dst, $src\n"
 6599     "skip:" %}
 6600   ins_encode %{
 6601     Label Lskip;
 6602     // Invert sense of branch from sense of CMOV
 6603     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6604     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6605     __ bind(Lskip);
 6606   %}
 6607   ins_pipe(pipe_slow);
 6608 %}
 6609 
 6610 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6611   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6612   ins_cost(200);
 6613   expand %{
 6614     cmovF_regU(cop, cr, dst, src);
 6615   %}
 6616 %}
 6617 
 6618 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6619 %{
 6620   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6621 
 6622   ins_cost(200); // XXX
 6623   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6624             "movsd     $dst, $src\n"
 6625     "skip:" %}
 6626   ins_encode %{
 6627     Label Lskip;
 6628     // Invert sense of branch from sense of CMOV
 6629     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6630     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6631     __ bind(Lskip);
 6632   %}
 6633   ins_pipe(pipe_slow);
 6634 %}
 6635 
 6636 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6637 %{
 6638   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6639 
 6640   ins_cost(200); // XXX
 6641   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6642             "movsd     $dst, $src\n"
 6643     "skip:" %}
 6644   ins_encode %{
 6645     Label Lskip;
 6646     // Invert sense of branch from sense of CMOV
 6647     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6648     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6649     __ bind(Lskip);
 6650   %}
 6651   ins_pipe(pipe_slow);
 6652 %}
 6653 
 6654 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6655   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6656   ins_cost(200);
 6657   expand %{
 6658     cmovD_regU(cop, cr, dst, src);
 6659   %}
 6660 %}
 6661 
 6662 //----------Arithmetic Instructions--------------------------------------------
 6663 //----------Addition Instructions----------------------------------------------
 6664 
 6665 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6666 %{
 6667   match(Set dst (AddI dst src));
 6668   effect(KILL cr);
 6669   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6670   format %{ "addl    $dst, $src\t# int" %}
 6671   ins_encode %{
 6672     __ addl($dst$$Register, $src$$Register);
 6673   %}
 6674   ins_pipe(ialu_reg_reg);
 6675 %}
 6676 
 6677 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 6678 %{
 6679   match(Set dst (AddI dst src));
 6680   effect(KILL cr);
 6681   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6682 
 6683   format %{ "addl    $dst, $src\t# int" %}
 6684   ins_encode %{
 6685     __ addl($dst$$Register, $src$$constant);
 6686   %}
 6687   ins_pipe( ialu_reg );
 6688 %}
 6689 
 6690 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 6691 %{
 6692   match(Set dst (AddI dst (LoadI src)));
 6693   effect(KILL cr);
 6694   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6695 
 6696   ins_cost(150); // XXX
 6697   format %{ "addl    $dst, $src\t# int" %}
 6698   ins_encode %{
 6699     __ addl($dst$$Register, $src$$Address);
 6700   %}
 6701   ins_pipe(ialu_reg_mem);
 6702 %}
 6703 
 6704 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 6705 %{
 6706   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6707   effect(KILL cr);
 6708   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6709 
 6710   ins_cost(150); // XXX
 6711   format %{ "addl    $dst, $src\t# int" %}
 6712   ins_encode %{
 6713     __ addl($dst$$Address, $src$$Register);
 6714   %}
 6715   ins_pipe(ialu_mem_reg);
 6716 %}
 6717 
 6718 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 6719 %{
 6720   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6721   effect(KILL cr);
 6722   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6723 
 6724 
 6725   ins_cost(125); // XXX
 6726   format %{ "addl    $dst, $src\t# int" %}
 6727   ins_encode %{
 6728     __ addl($dst$$Address, $src$$constant);
 6729   %}
 6730   ins_pipe(ialu_mem_imm);
 6731 %}
 6732 
 6733 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 6734 %{
 6735   predicate(UseIncDec);
 6736   match(Set dst (AddI dst src));
 6737   effect(KILL cr);
 6738 
 6739   format %{ "incl    $dst\t# int" %}
 6740   ins_encode %{
 6741     __ incrementl($dst$$Register);
 6742   %}
 6743   ins_pipe(ialu_reg);
 6744 %}
 6745 
 6746 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 6747 %{
 6748   predicate(UseIncDec);
 6749   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6750   effect(KILL cr);
 6751 
 6752   ins_cost(125); // XXX
 6753   format %{ "incl    $dst\t# int" %}
 6754   ins_encode %{
 6755     __ incrementl($dst$$Address);
 6756   %}
 6757   ins_pipe(ialu_mem_imm);
 6758 %}
 6759 
 6760 // XXX why does that use AddI
 6761 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 6762 %{
 6763   predicate(UseIncDec);
 6764   match(Set dst (AddI dst src));
 6765   effect(KILL cr);
 6766 
 6767   format %{ "decl    $dst\t# int" %}
 6768   ins_encode %{
 6769     __ decrementl($dst$$Register);
 6770   %}
 6771   ins_pipe(ialu_reg);
 6772 %}
 6773 
 6774 // XXX why does that use AddI
 6775 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 6776 %{
 6777   predicate(UseIncDec);
 6778   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 6779   effect(KILL cr);
 6780 
 6781   ins_cost(125); // XXX
 6782   format %{ "decl    $dst\t# int" %}
 6783   ins_encode %{
 6784     __ decrementl($dst$$Address);
 6785   %}
 6786   ins_pipe(ialu_mem_imm);
 6787 %}
 6788 
 6789 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 6790 %{
 6791   predicate(VM_Version::supports_fast_2op_lea());
 6792   match(Set dst (AddI (LShiftI index scale) disp));
 6793 
 6794   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 6795   ins_encode %{
 6796     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6797     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6798   %}
 6799   ins_pipe(ialu_reg_reg);
 6800 %}
 6801 
 6802 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 6803 %{
 6804   predicate(VM_Version::supports_fast_3op_lea());
 6805   match(Set dst (AddI (AddI base index) disp));
 6806 
 6807   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 6808   ins_encode %{
 6809     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6810   %}
 6811   ins_pipe(ialu_reg_reg);
 6812 %}
 6813 
 6814 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 6815 %{
 6816   predicate(VM_Version::supports_fast_2op_lea());
 6817   match(Set dst (AddI base (LShiftI index scale)));
 6818 
 6819   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 6820   ins_encode %{
 6821     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6822     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6823   %}
 6824   ins_pipe(ialu_reg_reg);
 6825 %}
 6826 
 6827 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 6828 %{
 6829   predicate(VM_Version::supports_fast_3op_lea());
 6830   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 6831 
 6832   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 6833   ins_encode %{
 6834     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6835     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 6836   %}
 6837   ins_pipe(ialu_reg_reg);
 6838 %}
 6839 
 6840 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 6841 %{
 6842   match(Set dst (AddL dst src));
 6843   effect(KILL cr);
 6844   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6845 
 6846   format %{ "addq    $dst, $src\t# long" %}
 6847   ins_encode %{
 6848     __ addq($dst$$Register, $src$$Register);
 6849   %}
 6850   ins_pipe(ialu_reg_reg);
 6851 %}
 6852 
 6853 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 6854 %{
 6855   match(Set dst (AddL dst src));
 6856   effect(KILL cr);
 6857   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6858 
 6859   format %{ "addq    $dst, $src\t# long" %}
 6860   ins_encode %{
 6861     __ addq($dst$$Register, $src$$constant);
 6862   %}
 6863   ins_pipe( ialu_reg );
 6864 %}
 6865 
 6866 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 6867 %{
 6868   match(Set dst (AddL dst (LoadL src)));
 6869   effect(KILL cr);
 6870   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6871 
 6872   ins_cost(150); // XXX
 6873   format %{ "addq    $dst, $src\t# long" %}
 6874   ins_encode %{
 6875     __ addq($dst$$Register, $src$$Address);
 6876   %}
 6877   ins_pipe(ialu_reg_mem);
 6878 %}
 6879 
 6880 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 6881 %{
 6882   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6883   effect(KILL cr);
 6884   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6885 
 6886   ins_cost(150); // XXX
 6887   format %{ "addq    $dst, $src\t# long" %}
 6888   ins_encode %{
 6889     __ addq($dst$$Address, $src$$Register);
 6890   %}
 6891   ins_pipe(ialu_mem_reg);
 6892 %}
 6893 
 6894 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 6895 %{
 6896   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6897   effect(KILL cr);
 6898   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6899 
 6900   ins_cost(125); // XXX
 6901   format %{ "addq    $dst, $src\t# long" %}
 6902   ins_encode %{
 6903     __ addq($dst$$Address, $src$$constant);
 6904   %}
 6905   ins_pipe(ialu_mem_imm);
 6906 %}
 6907 
 6908 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 6909 %{
 6910   predicate(UseIncDec);
 6911   match(Set dst (AddL dst src));
 6912   effect(KILL cr);
 6913 
 6914   format %{ "incq    $dst\t# long" %}
 6915   ins_encode %{
 6916     __ incrementq($dst$$Register);
 6917   %}
 6918   ins_pipe(ialu_reg);
 6919 %}
 6920 
 6921 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 6922 %{
 6923   predicate(UseIncDec);
 6924   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6925   effect(KILL cr);
 6926 
 6927   ins_cost(125); // XXX
 6928   format %{ "incq    $dst\t# long" %}
 6929   ins_encode %{
 6930     __ incrementq($dst$$Address);
 6931   %}
 6932   ins_pipe(ialu_mem_imm);
 6933 %}
 6934 
 6935 // XXX why does that use AddL
 6936 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 6937 %{
 6938   predicate(UseIncDec);
 6939   match(Set dst (AddL dst src));
 6940   effect(KILL cr);
 6941 
 6942   format %{ "decq    $dst\t# long" %}
 6943   ins_encode %{
 6944     __ decrementq($dst$$Register);
 6945   %}
 6946   ins_pipe(ialu_reg);
 6947 %}
 6948 
 6949 // XXX why does that use AddL
 6950 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 6951 %{
 6952   predicate(UseIncDec);
 6953   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 6954   effect(KILL cr);
 6955 
 6956   ins_cost(125); // XXX
 6957   format %{ "decq    $dst\t# long" %}
 6958   ins_encode %{
 6959     __ decrementq($dst$$Address);
 6960   %}
 6961   ins_pipe(ialu_mem_imm);
 6962 %}
 6963 
 6964 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 6965 %{
 6966   predicate(VM_Version::supports_fast_2op_lea());
 6967   match(Set dst (AddL (LShiftL index scale) disp));
 6968 
 6969   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 6970   ins_encode %{
 6971     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6972     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 6973   %}
 6974   ins_pipe(ialu_reg_reg);
 6975 %}
 6976 
 6977 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 6978 %{
 6979   predicate(VM_Version::supports_fast_3op_lea());
 6980   match(Set dst (AddL (AddL base index) disp));
 6981 
 6982   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 6983   ins_encode %{
 6984     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 6985   %}
 6986   ins_pipe(ialu_reg_reg);
 6987 %}
 6988 
 6989 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 6990 %{
 6991   predicate(VM_Version::supports_fast_2op_lea());
 6992   match(Set dst (AddL base (LShiftL index scale)));
 6993 
 6994   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 6995   ins_encode %{
 6996     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 6997     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 6998   %}
 6999   ins_pipe(ialu_reg_reg);
 7000 %}
 7001 
 7002 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7003 %{
 7004   predicate(VM_Version::supports_fast_3op_lea());
 7005   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7006 
 7007   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7008   ins_encode %{
 7009     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7010     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7011   %}
 7012   ins_pipe(ialu_reg_reg);
 7013 %}
 7014 
 7015 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7016 %{
 7017   match(Set dst (AddP dst src));
 7018   effect(KILL cr);
 7019   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7020 
 7021   format %{ "addq    $dst, $src\t# ptr" %}
 7022   ins_encode %{
 7023     __ addq($dst$$Register, $src$$Register);
 7024   %}
 7025   ins_pipe(ialu_reg_reg);
 7026 %}
 7027 
 7028 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7029 %{
 7030   match(Set dst (AddP dst src));
 7031   effect(KILL cr);
 7032   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7033 
 7034   format %{ "addq    $dst, $src\t# ptr" %}
 7035   ins_encode %{
 7036     __ addq($dst$$Register, $src$$constant);
 7037   %}
 7038   ins_pipe( ialu_reg );
 7039 %}
 7040 
 7041 // XXX addP mem ops ????
 7042 
 7043 instruct checkCastPP(rRegP dst)
 7044 %{
 7045   match(Set dst (CheckCastPP dst));
 7046 
 7047   size(0);
 7048   format %{ "# checkcastPP of $dst" %}
 7049   ins_encode(/* empty encoding */);
 7050   ins_pipe(empty);
 7051 %}
 7052 
 7053 instruct castPP(rRegP dst)
 7054 %{
 7055   match(Set dst (CastPP dst));
 7056 
 7057   size(0);
 7058   format %{ "# castPP of $dst" %}
 7059   ins_encode(/* empty encoding */);
 7060   ins_pipe(empty);
 7061 %}
 7062 
 7063 instruct castII(rRegI dst)
 7064 %{
 7065   match(Set dst (CastII dst));
 7066 
 7067   size(0);
 7068   format %{ "# castII of $dst" %}
 7069   ins_encode(/* empty encoding */);
 7070   ins_cost(0);
 7071   ins_pipe(empty);
 7072 %}
 7073 
 7074 instruct castLL(rRegL dst)
 7075 %{
 7076   match(Set dst (CastLL dst));
 7077 
 7078   size(0);
 7079   format %{ "# castLL of $dst" %}
 7080   ins_encode(/* empty encoding */);
 7081   ins_cost(0);
 7082   ins_pipe(empty);
 7083 %}
 7084 
 7085 instruct castFF(regF dst)
 7086 %{
 7087   match(Set dst (CastFF dst));
 7088 
 7089   size(0);
 7090   format %{ "# castFF of $dst" %}
 7091   ins_encode(/* empty encoding */);
 7092   ins_cost(0);
 7093   ins_pipe(empty);
 7094 %}
 7095 
 7096 instruct castHH(regF dst)
 7097 %{
 7098   match(Set dst (CastHH dst));
 7099 
 7100   size(0);
 7101   format %{ "# castHH of $dst" %}
 7102   ins_encode(/* empty encoding */);
 7103   ins_cost(0);
 7104   ins_pipe(empty);
 7105 %}
 7106 
 7107 instruct castDD(regD dst)
 7108 %{
 7109   match(Set dst (CastDD dst));
 7110 
 7111   size(0);
 7112   format %{ "# castDD of $dst" %}
 7113   ins_encode(/* empty encoding */);
 7114   ins_cost(0);
 7115   ins_pipe(empty);
 7116 %}
 7117 
 7118 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7119 instruct compareAndSwapP(rRegI res,
 7120                          memory mem_ptr,
 7121                          rax_RegP oldval, rRegP newval,
 7122                          rFlagsReg cr)
 7123 %{
 7124   predicate(n->as_LoadStore()->barrier_data() == 0);
 7125   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7126   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7127   effect(KILL cr, KILL oldval);
 7128 
 7129   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7130             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7131             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7132   ins_encode %{
 7133     __ lock();
 7134     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7135     __ setcc(Assembler::equal, $res$$Register);
 7136   %}
 7137   ins_pipe( pipe_cmpxchg );
 7138 %}
 7139 
 7140 instruct compareAndSwapL(rRegI res,
 7141                          memory mem_ptr,
 7142                          rax_RegL oldval, rRegL newval,
 7143                          rFlagsReg cr)
 7144 %{
 7145   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7146   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7147   effect(KILL cr, KILL oldval);
 7148 
 7149   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7150             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7151             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7152   ins_encode %{
 7153     __ lock();
 7154     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7155     __ setcc(Assembler::equal, $res$$Register);
 7156   %}
 7157   ins_pipe( pipe_cmpxchg );
 7158 %}
 7159 
 7160 instruct compareAndSwapI(rRegI res,
 7161                          memory mem_ptr,
 7162                          rax_RegI oldval, rRegI newval,
 7163                          rFlagsReg cr)
 7164 %{
 7165   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7166   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7167   effect(KILL cr, KILL oldval);
 7168 
 7169   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7170             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7171             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7172   ins_encode %{
 7173     __ lock();
 7174     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7175     __ setcc(Assembler::equal, $res$$Register);
 7176   %}
 7177   ins_pipe( pipe_cmpxchg );
 7178 %}
 7179 
 7180 instruct compareAndSwapB(rRegI res,
 7181                          memory mem_ptr,
 7182                          rax_RegI oldval, rRegI newval,
 7183                          rFlagsReg cr)
 7184 %{
 7185   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7186   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7187   effect(KILL cr, KILL oldval);
 7188 
 7189   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7190             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7191             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7192   ins_encode %{
 7193     __ lock();
 7194     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7195     __ setcc(Assembler::equal, $res$$Register);
 7196   %}
 7197   ins_pipe( pipe_cmpxchg );
 7198 %}
 7199 
 7200 instruct compareAndSwapS(rRegI res,
 7201                          memory mem_ptr,
 7202                          rax_RegI oldval, rRegI newval,
 7203                          rFlagsReg cr)
 7204 %{
 7205   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7206   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7207   effect(KILL cr, KILL oldval);
 7208 
 7209   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7210             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7211             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7212   ins_encode %{
 7213     __ lock();
 7214     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7215     __ setcc(Assembler::equal, $res$$Register);
 7216   %}
 7217   ins_pipe( pipe_cmpxchg );
 7218 %}
 7219 
 7220 instruct compareAndSwapN(rRegI res,
 7221                           memory mem_ptr,
 7222                           rax_RegN oldval, rRegN newval,
 7223                           rFlagsReg cr) %{
 7224   predicate(n->as_LoadStore()->barrier_data() == 0);
 7225   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7226   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7227   effect(KILL cr, KILL oldval);
 7228 
 7229   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7230             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7231             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7232   ins_encode %{
 7233     __ lock();
 7234     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7235     __ setcc(Assembler::equal, $res$$Register);
 7236   %}
 7237   ins_pipe( pipe_cmpxchg );
 7238 %}
 7239 
 7240 instruct compareAndExchangeB(
 7241                          memory mem_ptr,
 7242                          rax_RegI oldval, rRegI newval,
 7243                          rFlagsReg cr)
 7244 %{
 7245   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7246   effect(KILL cr);
 7247 
 7248   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7249             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7250   ins_encode %{
 7251     __ lock();
 7252     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7253   %}
 7254   ins_pipe( pipe_cmpxchg );
 7255 %}
 7256 
 7257 instruct compareAndExchangeS(
 7258                          memory mem_ptr,
 7259                          rax_RegI oldval, rRegI newval,
 7260                          rFlagsReg cr)
 7261 %{
 7262   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7263   effect(KILL cr);
 7264 
 7265   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7266             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7267   ins_encode %{
 7268     __ lock();
 7269     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7270   %}
 7271   ins_pipe( pipe_cmpxchg );
 7272 %}
 7273 
 7274 instruct compareAndExchangeI(
 7275                          memory mem_ptr,
 7276                          rax_RegI oldval, rRegI newval,
 7277                          rFlagsReg cr)
 7278 %{
 7279   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7280   effect(KILL cr);
 7281 
 7282   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7283             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7284   ins_encode %{
 7285     __ lock();
 7286     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7287   %}
 7288   ins_pipe( pipe_cmpxchg );
 7289 %}
 7290 
 7291 instruct compareAndExchangeL(
 7292                          memory mem_ptr,
 7293                          rax_RegL oldval, rRegL newval,
 7294                          rFlagsReg cr)
 7295 %{
 7296   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7297   effect(KILL cr);
 7298 
 7299   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7300             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7301   ins_encode %{
 7302     __ lock();
 7303     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7304   %}
 7305   ins_pipe( pipe_cmpxchg );
 7306 %}
 7307 
 7308 instruct compareAndExchangeN(
 7309                           memory mem_ptr,
 7310                           rax_RegN oldval, rRegN newval,
 7311                           rFlagsReg cr) %{
 7312   predicate(n->as_LoadStore()->barrier_data() == 0);
 7313   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7314   effect(KILL cr);
 7315 
 7316   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7317             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7318   ins_encode %{
 7319     __ lock();
 7320     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7321   %}
 7322   ins_pipe( pipe_cmpxchg );
 7323 %}
 7324 
 7325 instruct compareAndExchangeP(
 7326                          memory mem_ptr,
 7327                          rax_RegP oldval, rRegP newval,
 7328                          rFlagsReg cr)
 7329 %{
 7330   predicate(n->as_LoadStore()->barrier_data() == 0);
 7331   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7332   effect(KILL cr);
 7333 
 7334   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7335             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7336   ins_encode %{
 7337     __ lock();
 7338     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7339   %}
 7340   ins_pipe( pipe_cmpxchg );
 7341 %}
 7342 
 7343 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7344   predicate(n->as_LoadStore()->result_not_used());
 7345   match(Set dummy (GetAndAddB mem add));
 7346   effect(KILL cr);
 7347   format %{ "addb_lock   $mem, $add" %}
 7348   ins_encode %{
 7349     __ lock();
 7350     __ addb($mem$$Address, $add$$Register);
 7351   %}
 7352   ins_pipe(pipe_cmpxchg);
 7353 %}
 7354 
 7355 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7356   predicate(n->as_LoadStore()->result_not_used());
 7357   match(Set dummy (GetAndAddB mem add));
 7358   effect(KILL cr);
 7359   format %{ "addb_lock   $mem, $add" %}
 7360   ins_encode %{
 7361     __ lock();
 7362     __ addb($mem$$Address, $add$$constant);
 7363   %}
 7364   ins_pipe(pipe_cmpxchg);
 7365 %}
 7366 
 7367 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7368   predicate(!n->as_LoadStore()->result_not_used());
 7369   match(Set newval (GetAndAddB mem newval));
 7370   effect(KILL cr);
 7371   format %{ "xaddb_lock  $mem, $newval" %}
 7372   ins_encode %{
 7373     __ lock();
 7374     __ xaddb($mem$$Address, $newval$$Register);
 7375   %}
 7376   ins_pipe(pipe_cmpxchg);
 7377 %}
 7378 
 7379 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7380   predicate(n->as_LoadStore()->result_not_used());
 7381   match(Set dummy (GetAndAddS mem add));
 7382   effect(KILL cr);
 7383   format %{ "addw_lock   $mem, $add" %}
 7384   ins_encode %{
 7385     __ lock();
 7386     __ addw($mem$$Address, $add$$Register);
 7387   %}
 7388   ins_pipe(pipe_cmpxchg);
 7389 %}
 7390 
 7391 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7392   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7393   match(Set dummy (GetAndAddS mem add));
 7394   effect(KILL cr);
 7395   format %{ "addw_lock   $mem, $add" %}
 7396   ins_encode %{
 7397     __ lock();
 7398     __ addw($mem$$Address, $add$$constant);
 7399   %}
 7400   ins_pipe(pipe_cmpxchg);
 7401 %}
 7402 
 7403 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7404   predicate(!n->as_LoadStore()->result_not_used());
 7405   match(Set newval (GetAndAddS mem newval));
 7406   effect(KILL cr);
 7407   format %{ "xaddw_lock  $mem, $newval" %}
 7408   ins_encode %{
 7409     __ lock();
 7410     __ xaddw($mem$$Address, $newval$$Register);
 7411   %}
 7412   ins_pipe(pipe_cmpxchg);
 7413 %}
 7414 
 7415 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7416   predicate(n->as_LoadStore()->result_not_used());
 7417   match(Set dummy (GetAndAddI mem add));
 7418   effect(KILL cr);
 7419   format %{ "addl_lock   $mem, $add" %}
 7420   ins_encode %{
 7421     __ lock();
 7422     __ addl($mem$$Address, $add$$Register);
 7423   %}
 7424   ins_pipe(pipe_cmpxchg);
 7425 %}
 7426 
 7427 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7428   predicate(n->as_LoadStore()->result_not_used());
 7429   match(Set dummy (GetAndAddI mem add));
 7430   effect(KILL cr);
 7431   format %{ "addl_lock   $mem, $add" %}
 7432   ins_encode %{
 7433     __ lock();
 7434     __ addl($mem$$Address, $add$$constant);
 7435   %}
 7436   ins_pipe(pipe_cmpxchg);
 7437 %}
 7438 
 7439 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 7440   predicate(!n->as_LoadStore()->result_not_used());
 7441   match(Set newval (GetAndAddI mem newval));
 7442   effect(KILL cr);
 7443   format %{ "xaddl_lock  $mem, $newval" %}
 7444   ins_encode %{
 7445     __ lock();
 7446     __ xaddl($mem$$Address, $newval$$Register);
 7447   %}
 7448   ins_pipe(pipe_cmpxchg);
 7449 %}
 7450 
 7451 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 7452   predicate(n->as_LoadStore()->result_not_used());
 7453   match(Set dummy (GetAndAddL mem add));
 7454   effect(KILL cr);
 7455   format %{ "addq_lock   $mem, $add" %}
 7456   ins_encode %{
 7457     __ lock();
 7458     __ addq($mem$$Address, $add$$Register);
 7459   %}
 7460   ins_pipe(pipe_cmpxchg);
 7461 %}
 7462 
 7463 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 7464   predicate(n->as_LoadStore()->result_not_used());
 7465   match(Set dummy (GetAndAddL mem add));
 7466   effect(KILL cr);
 7467   format %{ "addq_lock   $mem, $add" %}
 7468   ins_encode %{
 7469     __ lock();
 7470     __ addq($mem$$Address, $add$$constant);
 7471   %}
 7472   ins_pipe(pipe_cmpxchg);
 7473 %}
 7474 
 7475 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 7476   predicate(!n->as_LoadStore()->result_not_used());
 7477   match(Set newval (GetAndAddL mem newval));
 7478   effect(KILL cr);
 7479   format %{ "xaddq_lock  $mem, $newval" %}
 7480   ins_encode %{
 7481     __ lock();
 7482     __ xaddq($mem$$Address, $newval$$Register);
 7483   %}
 7484   ins_pipe(pipe_cmpxchg);
 7485 %}
 7486 
 7487 instruct xchgB( memory mem, rRegI newval) %{
 7488   match(Set newval (GetAndSetB mem newval));
 7489   format %{ "XCHGB  $newval,[$mem]" %}
 7490   ins_encode %{
 7491     __ xchgb($newval$$Register, $mem$$Address);
 7492   %}
 7493   ins_pipe( pipe_cmpxchg );
 7494 %}
 7495 
 7496 instruct xchgS( memory mem, rRegI newval) %{
 7497   match(Set newval (GetAndSetS mem newval));
 7498   format %{ "XCHGW  $newval,[$mem]" %}
 7499   ins_encode %{
 7500     __ xchgw($newval$$Register, $mem$$Address);
 7501   %}
 7502   ins_pipe( pipe_cmpxchg );
 7503 %}
 7504 
 7505 instruct xchgI( memory mem, rRegI newval) %{
 7506   match(Set newval (GetAndSetI mem newval));
 7507   format %{ "XCHGL  $newval,[$mem]" %}
 7508   ins_encode %{
 7509     __ xchgl($newval$$Register, $mem$$Address);
 7510   %}
 7511   ins_pipe( pipe_cmpxchg );
 7512 %}
 7513 
 7514 instruct xchgL( memory mem, rRegL newval) %{
 7515   match(Set newval (GetAndSetL mem newval));
 7516   format %{ "XCHGL  $newval,[$mem]" %}
 7517   ins_encode %{
 7518     __ xchgq($newval$$Register, $mem$$Address);
 7519   %}
 7520   ins_pipe( pipe_cmpxchg );
 7521 %}
 7522 
 7523 instruct xchgP( memory mem, rRegP newval) %{
 7524   match(Set newval (GetAndSetP mem newval));
 7525   predicate(n->as_LoadStore()->barrier_data() == 0);
 7526   format %{ "XCHGQ  $newval,[$mem]" %}
 7527   ins_encode %{
 7528     __ xchgq($newval$$Register, $mem$$Address);
 7529   %}
 7530   ins_pipe( pipe_cmpxchg );
 7531 %}
 7532 
 7533 instruct xchgN( memory mem, rRegN newval) %{
 7534   predicate(n->as_LoadStore()->barrier_data() == 0);
 7535   match(Set newval (GetAndSetN mem newval));
 7536   format %{ "XCHGL  $newval,$mem]" %}
 7537   ins_encode %{
 7538     __ xchgl($newval$$Register, $mem$$Address);
 7539   %}
 7540   ins_pipe( pipe_cmpxchg );
 7541 %}
 7542 
 7543 //----------Abs Instructions-------------------------------------------
 7544 
 7545 // Integer Absolute Instructions
 7546 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7547 %{
 7548   match(Set dst (AbsI src));
 7549   effect(TEMP dst, KILL cr);
 7550   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 7551             "subl    $dst, $src\n\t"
 7552             "cmovll  $dst, $src" %}
 7553   ins_encode %{
 7554     __ xorl($dst$$Register, $dst$$Register);
 7555     __ subl($dst$$Register, $src$$Register);
 7556     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 7557   %}
 7558 
 7559   ins_pipe(ialu_reg_reg);
 7560 %}
 7561 
 7562 // Long Absolute Instructions
 7563 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7564 %{
 7565   match(Set dst (AbsL src));
 7566   effect(TEMP dst, KILL cr);
 7567   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 7568             "subq    $dst, $src\n\t"
 7569             "cmovlq  $dst, $src" %}
 7570   ins_encode %{
 7571     __ xorl($dst$$Register, $dst$$Register);
 7572     __ subq($dst$$Register, $src$$Register);
 7573     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 7574   %}
 7575 
 7576   ins_pipe(ialu_reg_reg);
 7577 %}
 7578 
 7579 //----------Subtraction Instructions-------------------------------------------
 7580 
 7581 // Integer Subtraction Instructions
 7582 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7583 %{
 7584   match(Set dst (SubI dst src));
 7585   effect(KILL cr);
 7586   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7587 
 7588   format %{ "subl    $dst, $src\t# int" %}
 7589   ins_encode %{
 7590     __ subl($dst$$Register, $src$$Register);
 7591   %}
 7592   ins_pipe(ialu_reg_reg);
 7593 %}
 7594 
 7595 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7596 %{
 7597   match(Set dst (SubI dst (LoadI src)));
 7598   effect(KILL cr);
 7599   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7600 
 7601   ins_cost(150);
 7602   format %{ "subl    $dst, $src\t# int" %}
 7603   ins_encode %{
 7604     __ subl($dst$$Register, $src$$Address);
 7605   %}
 7606   ins_pipe(ialu_reg_mem);
 7607 %}
 7608 
 7609 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7610 %{
 7611   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 7612   effect(KILL cr);
 7613   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7614 
 7615   ins_cost(150);
 7616   format %{ "subl    $dst, $src\t# int" %}
 7617   ins_encode %{
 7618     __ subl($dst$$Address, $src$$Register);
 7619   %}
 7620   ins_pipe(ialu_mem_reg);
 7621 %}
 7622 
 7623 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7624 %{
 7625   match(Set dst (SubL dst src));
 7626   effect(KILL cr);
 7627   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7628 
 7629   format %{ "subq    $dst, $src\t# long" %}
 7630   ins_encode %{
 7631     __ subq($dst$$Register, $src$$Register);
 7632   %}
 7633   ins_pipe(ialu_reg_reg);
 7634 %}
 7635 
 7636 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7637 %{
 7638   match(Set dst (SubL dst (LoadL src)));
 7639   effect(KILL cr);
 7640   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7641 
 7642   ins_cost(150);
 7643   format %{ "subq    $dst, $src\t# long" %}
 7644   ins_encode %{
 7645     __ subq($dst$$Register, $src$$Address);
 7646   %}
 7647   ins_pipe(ialu_reg_mem);
 7648 %}
 7649 
 7650 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7651 %{
 7652   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 7653   effect(KILL cr);
 7654   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7655 
 7656   ins_cost(150);
 7657   format %{ "subq    $dst, $src\t# long" %}
 7658   ins_encode %{
 7659     __ subq($dst$$Address, $src$$Register);
 7660   %}
 7661   ins_pipe(ialu_mem_reg);
 7662 %}
 7663 
 7664 // Subtract from a pointer
 7665 // XXX hmpf???
 7666 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 7667 %{
 7668   match(Set dst (AddP dst (SubI zero src)));
 7669   effect(KILL cr);
 7670 
 7671   format %{ "subq    $dst, $src\t# ptr - int" %}
 7672   ins_encode %{
 7673     __ subq($dst$$Register, $src$$Register);
 7674   %}
 7675   ins_pipe(ialu_reg_reg);
 7676 %}
 7677 
 7678 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 7679 %{
 7680   match(Set dst (SubI zero dst));
 7681   effect(KILL cr);
 7682   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7683 
 7684   format %{ "negl    $dst\t# int" %}
 7685   ins_encode %{
 7686     __ negl($dst$$Register);
 7687   %}
 7688   ins_pipe(ialu_reg);
 7689 %}
 7690 
 7691 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 7692 %{
 7693   match(Set dst (NegI dst));
 7694   effect(KILL cr);
 7695   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7696 
 7697   format %{ "negl    $dst\t# int" %}
 7698   ins_encode %{
 7699     __ negl($dst$$Register);
 7700   %}
 7701   ins_pipe(ialu_reg);
 7702 %}
 7703 
 7704 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 7705 %{
 7706   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 7707   effect(KILL cr);
 7708   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7709 
 7710   format %{ "negl    $dst\t# int" %}
 7711   ins_encode %{
 7712     __ negl($dst$$Address);
 7713   %}
 7714   ins_pipe(ialu_reg);
 7715 %}
 7716 
 7717 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 7718 %{
 7719   match(Set dst (SubL zero dst));
 7720   effect(KILL cr);
 7721   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7722 
 7723   format %{ "negq    $dst\t# long" %}
 7724   ins_encode %{
 7725     __ negq($dst$$Register);
 7726   %}
 7727   ins_pipe(ialu_reg);
 7728 %}
 7729 
 7730 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 7731 %{
 7732   match(Set dst (NegL dst));
 7733   effect(KILL cr);
 7734   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7735 
 7736   format %{ "negq    $dst\t# int" %}
 7737   ins_encode %{
 7738     __ negq($dst$$Register);
 7739   %}
 7740   ins_pipe(ialu_reg);
 7741 %}
 7742 
 7743 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 7744 %{
 7745   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 7746   effect(KILL cr);
 7747   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 7748 
 7749   format %{ "negq    $dst\t# long" %}
 7750   ins_encode %{
 7751     __ negq($dst$$Address);
 7752   %}
 7753   ins_pipe(ialu_reg);
 7754 %}
 7755 
 7756 //----------Multiplication/Division Instructions-------------------------------
 7757 // Integer Multiplication Instructions
 7758 // Multiply Register
 7759 
 7760 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7761 %{
 7762   match(Set dst (MulI dst src));
 7763   effect(KILL cr);
 7764 
 7765   ins_cost(300);
 7766   format %{ "imull   $dst, $src\t# int" %}
 7767   ins_encode %{
 7768     __ imull($dst$$Register, $src$$Register);
 7769   %}
 7770   ins_pipe(ialu_reg_reg_alu0);
 7771 %}
 7772 
 7773 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 7774 %{
 7775   match(Set dst (MulI src imm));
 7776   effect(KILL cr);
 7777 
 7778   ins_cost(300);
 7779   format %{ "imull   $dst, $src, $imm\t# int" %}
 7780   ins_encode %{
 7781     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 7782   %}
 7783   ins_pipe(ialu_reg_reg_alu0);
 7784 %}
 7785 
 7786 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 7787 %{
 7788   match(Set dst (MulI dst (LoadI src)));
 7789   effect(KILL cr);
 7790 
 7791   ins_cost(350);
 7792   format %{ "imull   $dst, $src\t# int" %}
 7793   ins_encode %{
 7794     __ imull($dst$$Register, $src$$Address);
 7795   %}
 7796   ins_pipe(ialu_reg_mem_alu0);
 7797 %}
 7798 
 7799 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 7800 %{
 7801   match(Set dst (MulI (LoadI src) imm));
 7802   effect(KILL cr);
 7803 
 7804   ins_cost(300);
 7805   format %{ "imull   $dst, $src, $imm\t# int" %}
 7806   ins_encode %{
 7807     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 7808   %}
 7809   ins_pipe(ialu_reg_mem_alu0);
 7810 %}
 7811 
 7812 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 7813 %{
 7814   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 7815   effect(KILL cr, KILL src2);
 7816 
 7817   expand %{ mulI_rReg(dst, src1, cr);
 7818            mulI_rReg(src2, src3, cr);
 7819            addI_rReg(dst, src2, cr); %}
 7820 %}
 7821 
 7822 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7823 %{
 7824   match(Set dst (MulL dst src));
 7825   effect(KILL cr);
 7826 
 7827   ins_cost(300);
 7828   format %{ "imulq   $dst, $src\t# long" %}
 7829   ins_encode %{
 7830     __ imulq($dst$$Register, $src$$Register);
 7831   %}
 7832   ins_pipe(ialu_reg_reg_alu0);
 7833 %}
 7834 
 7835 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 7836 %{
 7837   match(Set dst (MulL src imm));
 7838   effect(KILL cr);
 7839 
 7840   ins_cost(300);
 7841   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7842   ins_encode %{
 7843     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 7844   %}
 7845   ins_pipe(ialu_reg_reg_alu0);
 7846 %}
 7847 
 7848 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 7849 %{
 7850   match(Set dst (MulL dst (LoadL src)));
 7851   effect(KILL cr);
 7852 
 7853   ins_cost(350);
 7854   format %{ "imulq   $dst, $src\t# long" %}
 7855   ins_encode %{
 7856     __ imulq($dst$$Register, $src$$Address);
 7857   %}
 7858   ins_pipe(ialu_reg_mem_alu0);
 7859 %}
 7860 
 7861 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 7862 %{
 7863   match(Set dst (MulL (LoadL src) imm));
 7864   effect(KILL cr);
 7865 
 7866   ins_cost(300);
 7867   format %{ "imulq   $dst, $src, $imm\t# long" %}
 7868   ins_encode %{
 7869     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 7870   %}
 7871   ins_pipe(ialu_reg_mem_alu0);
 7872 %}
 7873 
 7874 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7875 %{
 7876   match(Set dst (MulHiL src rax));
 7877   effect(USE_KILL rax, KILL cr);
 7878 
 7879   ins_cost(300);
 7880   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 7881   ins_encode %{
 7882     __ imulq($src$$Register);
 7883   %}
 7884   ins_pipe(ialu_reg_reg_alu0);
 7885 %}
 7886 
 7887 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 7888 %{
 7889   match(Set dst (UMulHiL src rax));
 7890   effect(USE_KILL rax, KILL cr);
 7891 
 7892   ins_cost(300);
 7893   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 7894   ins_encode %{
 7895     __ mulq($src$$Register);
 7896   %}
 7897   ins_pipe(ialu_reg_reg_alu0);
 7898 %}
 7899 
 7900 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7901                    rFlagsReg cr)
 7902 %{
 7903   match(Set rax (DivI rax div));
 7904   effect(KILL rdx, KILL cr);
 7905 
 7906   ins_cost(30*100+10*100); // XXX
 7907   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7908             "jne,s   normal\n\t"
 7909             "xorl    rdx, rdx\n\t"
 7910             "cmpl    $div, -1\n\t"
 7911             "je,s    done\n"
 7912     "normal: cdql\n\t"
 7913             "idivl   $div\n"
 7914     "done:"        %}
 7915   ins_encode(cdql_enc(div));
 7916   ins_pipe(ialu_reg_reg_alu0);
 7917 %}
 7918 
 7919 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7920                    rFlagsReg cr)
 7921 %{
 7922   match(Set rax (DivL rax div));
 7923   effect(KILL rdx, KILL cr);
 7924 
 7925   ins_cost(30*100+10*100); // XXX
 7926   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7927             "cmpq    rax, rdx\n\t"
 7928             "jne,s   normal\n\t"
 7929             "xorl    rdx, rdx\n\t"
 7930             "cmpq    $div, -1\n\t"
 7931             "je,s    done\n"
 7932     "normal: cdqq\n\t"
 7933             "idivq   $div\n"
 7934     "done:"        %}
 7935   ins_encode(cdqq_enc(div));
 7936   ins_pipe(ialu_reg_reg_alu0);
 7937 %}
 7938 
 7939 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 7940 %{
 7941   match(Set rax (UDivI rax div));
 7942   effect(KILL rdx, KILL cr);
 7943 
 7944   ins_cost(300);
 7945   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 7946   ins_encode %{
 7947     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 7948   %}
 7949   ins_pipe(ialu_reg_reg_alu0);
 7950 %}
 7951 
 7952 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 7953 %{
 7954   match(Set rax (UDivL rax div));
 7955   effect(KILL rdx, KILL cr);
 7956 
 7957   ins_cost(300);
 7958   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 7959   ins_encode %{
 7960      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 7961   %}
 7962   ins_pipe(ialu_reg_reg_alu0);
 7963 %}
 7964 
 7965 // Integer DIVMOD with Register, both quotient and mod results
 7966 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 7967                              rFlagsReg cr)
 7968 %{
 7969   match(DivModI rax div);
 7970   effect(KILL cr);
 7971 
 7972   ins_cost(30*100+10*100); // XXX
 7973   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 7974             "jne,s   normal\n\t"
 7975             "xorl    rdx, rdx\n\t"
 7976             "cmpl    $div, -1\n\t"
 7977             "je,s    done\n"
 7978     "normal: cdql\n\t"
 7979             "idivl   $div\n"
 7980     "done:"        %}
 7981   ins_encode(cdql_enc(div));
 7982   ins_pipe(pipe_slow);
 7983 %}
 7984 
 7985 // Long DIVMOD with Register, both quotient and mod results
 7986 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 7987                              rFlagsReg cr)
 7988 %{
 7989   match(DivModL rax div);
 7990   effect(KILL cr);
 7991 
 7992   ins_cost(30*100+10*100); // XXX
 7993   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 7994             "cmpq    rax, rdx\n\t"
 7995             "jne,s   normal\n\t"
 7996             "xorl    rdx, rdx\n\t"
 7997             "cmpq    $div, -1\n\t"
 7998             "je,s    done\n"
 7999     "normal: cdqq\n\t"
 8000             "idivq   $div\n"
 8001     "done:"        %}
 8002   ins_encode(cdqq_enc(div));
 8003   ins_pipe(pipe_slow);
 8004 %}
 8005 
 8006 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8007 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8008                               no_rax_rdx_RegI div, rFlagsReg cr)
 8009 %{
 8010   match(UDivModI rax div);
 8011   effect(TEMP tmp, KILL cr);
 8012 
 8013   ins_cost(300);
 8014   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8015             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8016           %}
 8017   ins_encode %{
 8018     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8019   %}
 8020   ins_pipe(pipe_slow);
 8021 %}
 8022 
 8023 // Unsigned long DIVMOD with Register, both quotient and mod results
 8024 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8025                               no_rax_rdx_RegL div, rFlagsReg cr)
 8026 %{
 8027   match(UDivModL rax div);
 8028   effect(TEMP tmp, KILL cr);
 8029 
 8030   ins_cost(300);
 8031   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8032             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8033           %}
 8034   ins_encode %{
 8035     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8036   %}
 8037   ins_pipe(pipe_slow);
 8038 %}
 8039 
 8040 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8041                    rFlagsReg cr)
 8042 %{
 8043   match(Set rdx (ModI rax div));
 8044   effect(KILL rax, KILL cr);
 8045 
 8046   ins_cost(300); // XXX
 8047   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8048             "jne,s   normal\n\t"
 8049             "xorl    rdx, rdx\n\t"
 8050             "cmpl    $div, -1\n\t"
 8051             "je,s    done\n"
 8052     "normal: cdql\n\t"
 8053             "idivl   $div\n"
 8054     "done:"        %}
 8055   ins_encode(cdql_enc(div));
 8056   ins_pipe(ialu_reg_reg_alu0);
 8057 %}
 8058 
 8059 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8060                    rFlagsReg cr)
 8061 %{
 8062   match(Set rdx (ModL rax div));
 8063   effect(KILL rax, KILL cr);
 8064 
 8065   ins_cost(300); // XXX
 8066   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8067             "cmpq    rax, rdx\n\t"
 8068             "jne,s   normal\n\t"
 8069             "xorl    rdx, rdx\n\t"
 8070             "cmpq    $div, -1\n\t"
 8071             "je,s    done\n"
 8072     "normal: cdqq\n\t"
 8073             "idivq   $div\n"
 8074     "done:"        %}
 8075   ins_encode(cdqq_enc(div));
 8076   ins_pipe(ialu_reg_reg_alu0);
 8077 %}
 8078 
 8079 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8080 %{
 8081   match(Set rdx (UModI rax div));
 8082   effect(KILL rax, KILL cr);
 8083 
 8084   ins_cost(300);
 8085   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8086   ins_encode %{
 8087     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8088   %}
 8089   ins_pipe(ialu_reg_reg_alu0);
 8090 %}
 8091 
 8092 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8093 %{
 8094   match(Set rdx (UModL rax div));
 8095   effect(KILL rax, KILL cr);
 8096 
 8097   ins_cost(300);
 8098   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8099   ins_encode %{
 8100     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8101   %}
 8102   ins_pipe(ialu_reg_reg_alu0);
 8103 %}
 8104 
 8105 // Integer Shift Instructions
 8106 // Shift Left by one, two, three
 8107 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8108 %{
 8109   match(Set dst (LShiftI dst shift));
 8110   effect(KILL cr);
 8111 
 8112   format %{ "sall    $dst, $shift" %}
 8113   ins_encode %{
 8114     __ sall($dst$$Register, $shift$$constant);
 8115   %}
 8116   ins_pipe(ialu_reg);
 8117 %}
 8118 
 8119 // Shift Left by 8-bit immediate
 8120 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8121 %{
 8122   match(Set dst (LShiftI dst shift));
 8123   effect(KILL cr);
 8124 
 8125   format %{ "sall    $dst, $shift" %}
 8126   ins_encode %{
 8127     __ sall($dst$$Register, $shift$$constant);
 8128   %}
 8129   ins_pipe(ialu_reg);
 8130 %}
 8131 
 8132 // Shift Left by 8-bit immediate
 8133 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8134 %{
 8135   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8136   effect(KILL cr);
 8137 
 8138   format %{ "sall    $dst, $shift" %}
 8139   ins_encode %{
 8140     __ sall($dst$$Address, $shift$$constant);
 8141   %}
 8142   ins_pipe(ialu_mem_imm);
 8143 %}
 8144 
 8145 // Shift Left by variable
 8146 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8147 %{
 8148   predicate(!VM_Version::supports_bmi2());
 8149   match(Set dst (LShiftI dst shift));
 8150   effect(KILL cr);
 8151 
 8152   format %{ "sall    $dst, $shift" %}
 8153   ins_encode %{
 8154     __ sall($dst$$Register);
 8155   %}
 8156   ins_pipe(ialu_reg_reg);
 8157 %}
 8158 
 8159 // Shift Left by variable
 8160 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8161 %{
 8162   predicate(!VM_Version::supports_bmi2());
 8163   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 8164   effect(KILL cr);
 8165 
 8166   format %{ "sall    $dst, $shift" %}
 8167   ins_encode %{
 8168     __ sall($dst$$Address);
 8169   %}
 8170   ins_pipe(ialu_mem_reg);
 8171 %}
 8172 
 8173 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8174 %{
 8175   predicate(VM_Version::supports_bmi2());
 8176   match(Set dst (LShiftI src shift));
 8177 
 8178   format %{ "shlxl   $dst, $src, $shift" %}
 8179   ins_encode %{
 8180     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 8181   %}
 8182   ins_pipe(ialu_reg_reg);
 8183 %}
 8184 
 8185 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8186 %{
 8187   predicate(VM_Version::supports_bmi2());
 8188   match(Set dst (LShiftI (LoadI src) shift));
 8189   ins_cost(175);
 8190   format %{ "shlxl   $dst, $src, $shift" %}
 8191   ins_encode %{
 8192     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 8193   %}
 8194   ins_pipe(ialu_reg_mem);
 8195 %}
 8196 
 8197 // Arithmetic Shift Right by 8-bit immediate
 8198 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8199 %{
 8200   match(Set dst (RShiftI dst shift));
 8201   effect(KILL cr);
 8202 
 8203   format %{ "sarl    $dst, $shift" %}
 8204   ins_encode %{
 8205     __ sarl($dst$$Register, $shift$$constant);
 8206   %}
 8207   ins_pipe(ialu_mem_imm);
 8208 %}
 8209 
 8210 // Arithmetic Shift Right by 8-bit immediate
 8211 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8212 %{
 8213   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8214   effect(KILL cr);
 8215 
 8216   format %{ "sarl    $dst, $shift" %}
 8217   ins_encode %{
 8218     __ sarl($dst$$Address, $shift$$constant);
 8219   %}
 8220   ins_pipe(ialu_mem_imm);
 8221 %}
 8222 
 8223 // Arithmetic Shift Right by variable
 8224 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8225 %{
 8226   predicate(!VM_Version::supports_bmi2());
 8227   match(Set dst (RShiftI dst shift));
 8228   effect(KILL cr);
 8229 
 8230   format %{ "sarl    $dst, $shift" %}
 8231   ins_encode %{
 8232     __ sarl($dst$$Register);
 8233   %}
 8234   ins_pipe(ialu_reg_reg);
 8235 %}
 8236 
 8237 // Arithmetic Shift Right by variable
 8238 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8239 %{
 8240   predicate(!VM_Version::supports_bmi2());
 8241   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 8242   effect(KILL cr);
 8243 
 8244   format %{ "sarl    $dst, $shift" %}
 8245   ins_encode %{
 8246     __ sarl($dst$$Address);
 8247   %}
 8248   ins_pipe(ialu_mem_reg);
 8249 %}
 8250 
 8251 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8252 %{
 8253   predicate(VM_Version::supports_bmi2());
 8254   match(Set dst (RShiftI src shift));
 8255 
 8256   format %{ "sarxl   $dst, $src, $shift" %}
 8257   ins_encode %{
 8258     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 8259   %}
 8260   ins_pipe(ialu_reg_reg);
 8261 %}
 8262 
 8263 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8264 %{
 8265   predicate(VM_Version::supports_bmi2());
 8266   match(Set dst (RShiftI (LoadI src) shift));
 8267   ins_cost(175);
 8268   format %{ "sarxl   $dst, $src, $shift" %}
 8269   ins_encode %{
 8270     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 8271   %}
 8272   ins_pipe(ialu_reg_mem);
 8273 %}
 8274 
 8275 // Logical Shift Right by 8-bit immediate
 8276 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 8277 %{
 8278   match(Set dst (URShiftI dst shift));
 8279   effect(KILL cr);
 8280 
 8281   format %{ "shrl    $dst, $shift" %}
 8282   ins_encode %{
 8283     __ shrl($dst$$Register, $shift$$constant);
 8284   %}
 8285   ins_pipe(ialu_reg);
 8286 %}
 8287 
 8288 // Logical Shift Right by 8-bit immediate
 8289 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8290 %{
 8291   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8292   effect(KILL cr);
 8293 
 8294   format %{ "shrl    $dst, $shift" %}
 8295   ins_encode %{
 8296     __ shrl($dst$$Address, $shift$$constant);
 8297   %}
 8298   ins_pipe(ialu_mem_imm);
 8299 %}
 8300 
 8301 // Logical Shift Right by variable
 8302 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8303 %{
 8304   predicate(!VM_Version::supports_bmi2());
 8305   match(Set dst (URShiftI dst shift));
 8306   effect(KILL cr);
 8307 
 8308   format %{ "shrl    $dst, $shift" %}
 8309   ins_encode %{
 8310     __ shrl($dst$$Register);
 8311   %}
 8312   ins_pipe(ialu_reg_reg);
 8313 %}
 8314 
 8315 // Logical Shift Right by variable
 8316 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8317 %{
 8318   predicate(!VM_Version::supports_bmi2());
 8319   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 8320   effect(KILL cr);
 8321 
 8322   format %{ "shrl    $dst, $shift" %}
 8323   ins_encode %{
 8324     __ shrl($dst$$Address);
 8325   %}
 8326   ins_pipe(ialu_mem_reg);
 8327 %}
 8328 
 8329 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 8330 %{
 8331   predicate(VM_Version::supports_bmi2());
 8332   match(Set dst (URShiftI src shift));
 8333 
 8334   format %{ "shrxl   $dst, $src, $shift" %}
 8335   ins_encode %{
 8336     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 8337   %}
 8338   ins_pipe(ialu_reg_reg);
 8339 %}
 8340 
 8341 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 8342 %{
 8343   predicate(VM_Version::supports_bmi2());
 8344   match(Set dst (URShiftI (LoadI src) shift));
 8345   ins_cost(175);
 8346   format %{ "shrxl   $dst, $src, $shift" %}
 8347   ins_encode %{
 8348     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 8349   %}
 8350   ins_pipe(ialu_reg_mem);
 8351 %}
 8352 
 8353 // Long Shift Instructions
 8354 // Shift Left by one, two, three
 8355 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 8356 %{
 8357   match(Set dst (LShiftL dst shift));
 8358   effect(KILL cr);
 8359 
 8360   format %{ "salq    $dst, $shift" %}
 8361   ins_encode %{
 8362     __ salq($dst$$Register, $shift$$constant);
 8363   %}
 8364   ins_pipe(ialu_reg);
 8365 %}
 8366 
 8367 // Shift Left by 8-bit immediate
 8368 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8369 %{
 8370   match(Set dst (LShiftL dst shift));
 8371   effect(KILL cr);
 8372 
 8373   format %{ "salq    $dst, $shift" %}
 8374   ins_encode %{
 8375     __ salq($dst$$Register, $shift$$constant);
 8376   %}
 8377   ins_pipe(ialu_reg);
 8378 %}
 8379 
 8380 // Shift Left by 8-bit immediate
 8381 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8382 %{
 8383   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8384   effect(KILL cr);
 8385 
 8386   format %{ "salq    $dst, $shift" %}
 8387   ins_encode %{
 8388     __ salq($dst$$Address, $shift$$constant);
 8389   %}
 8390   ins_pipe(ialu_mem_imm);
 8391 %}
 8392 
 8393 // Shift Left by variable
 8394 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8395 %{
 8396   predicate(!VM_Version::supports_bmi2());
 8397   match(Set dst (LShiftL dst shift));
 8398   effect(KILL cr);
 8399 
 8400   format %{ "salq    $dst, $shift" %}
 8401   ins_encode %{
 8402     __ salq($dst$$Register);
 8403   %}
 8404   ins_pipe(ialu_reg_reg);
 8405 %}
 8406 
 8407 // Shift Left by variable
 8408 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8409 %{
 8410   predicate(!VM_Version::supports_bmi2());
 8411   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 8412   effect(KILL cr);
 8413 
 8414   format %{ "salq    $dst, $shift" %}
 8415   ins_encode %{
 8416     __ salq($dst$$Address);
 8417   %}
 8418   ins_pipe(ialu_mem_reg);
 8419 %}
 8420 
 8421 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8422 %{
 8423   predicate(VM_Version::supports_bmi2());
 8424   match(Set dst (LShiftL src shift));
 8425 
 8426   format %{ "shlxq   $dst, $src, $shift" %}
 8427   ins_encode %{
 8428     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 8429   %}
 8430   ins_pipe(ialu_reg_reg);
 8431 %}
 8432 
 8433 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8434 %{
 8435   predicate(VM_Version::supports_bmi2());
 8436   match(Set dst (LShiftL (LoadL src) shift));
 8437   ins_cost(175);
 8438   format %{ "shlxq   $dst, $src, $shift" %}
 8439   ins_encode %{
 8440     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 8441   %}
 8442   ins_pipe(ialu_reg_mem);
 8443 %}
 8444 
 8445 // Arithmetic Shift Right by 8-bit immediate
 8446 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 8447 %{
 8448   match(Set dst (RShiftL dst shift));
 8449   effect(KILL cr);
 8450 
 8451   format %{ "sarq    $dst, $shift" %}
 8452   ins_encode %{
 8453     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 8454   %}
 8455   ins_pipe(ialu_mem_imm);
 8456 %}
 8457 
 8458 // Arithmetic Shift Right by 8-bit immediate
 8459 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 8460 %{
 8461   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8462   effect(KILL cr);
 8463 
 8464   format %{ "sarq    $dst, $shift" %}
 8465   ins_encode %{
 8466     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 8467   %}
 8468   ins_pipe(ialu_mem_imm);
 8469 %}
 8470 
 8471 // Arithmetic Shift Right by variable
 8472 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8473 %{
 8474   predicate(!VM_Version::supports_bmi2());
 8475   match(Set dst (RShiftL dst shift));
 8476   effect(KILL cr);
 8477 
 8478   format %{ "sarq    $dst, $shift" %}
 8479   ins_encode %{
 8480     __ sarq($dst$$Register);
 8481   %}
 8482   ins_pipe(ialu_reg_reg);
 8483 %}
 8484 
 8485 // Arithmetic Shift Right by variable
 8486 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8487 %{
 8488   predicate(!VM_Version::supports_bmi2());
 8489   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 8490   effect(KILL cr);
 8491 
 8492   format %{ "sarq    $dst, $shift" %}
 8493   ins_encode %{
 8494     __ sarq($dst$$Address);
 8495   %}
 8496   ins_pipe(ialu_mem_reg);
 8497 %}
 8498 
 8499 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8500 %{
 8501   predicate(VM_Version::supports_bmi2());
 8502   match(Set dst (RShiftL src shift));
 8503 
 8504   format %{ "sarxq   $dst, $src, $shift" %}
 8505   ins_encode %{
 8506     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 8507   %}
 8508   ins_pipe(ialu_reg_reg);
 8509 %}
 8510 
 8511 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8512 %{
 8513   predicate(VM_Version::supports_bmi2());
 8514   match(Set dst (RShiftL (LoadL src) shift));
 8515   ins_cost(175);
 8516   format %{ "sarxq   $dst, $src, $shift" %}
 8517   ins_encode %{
 8518     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 8519   %}
 8520   ins_pipe(ialu_reg_mem);
 8521 %}
 8522 
 8523 // Logical Shift Right by 8-bit immediate
 8524 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 8525 %{
 8526   match(Set dst (URShiftL dst shift));
 8527   effect(KILL cr);
 8528 
 8529   format %{ "shrq    $dst, $shift" %}
 8530   ins_encode %{
 8531     __ shrq($dst$$Register, $shift$$constant);
 8532   %}
 8533   ins_pipe(ialu_reg);
 8534 %}
 8535 
 8536 // Logical Shift Right by 8-bit immediate
 8537 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 8538 %{
 8539   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8540   effect(KILL cr);
 8541 
 8542   format %{ "shrq    $dst, $shift" %}
 8543   ins_encode %{
 8544     __ shrq($dst$$Address, $shift$$constant);
 8545   %}
 8546   ins_pipe(ialu_mem_imm);
 8547 %}
 8548 
 8549 // Logical Shift Right by variable
 8550 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8551 %{
 8552   predicate(!VM_Version::supports_bmi2());
 8553   match(Set dst (URShiftL dst shift));
 8554   effect(KILL cr);
 8555 
 8556   format %{ "shrq    $dst, $shift" %}
 8557   ins_encode %{
 8558     __ shrq($dst$$Register);
 8559   %}
 8560   ins_pipe(ialu_reg_reg);
 8561 %}
 8562 
 8563 // Logical Shift Right by variable
 8564 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 8565 %{
 8566   predicate(!VM_Version::supports_bmi2());
 8567   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 8568   effect(KILL cr);
 8569 
 8570   format %{ "shrq    $dst, $shift" %}
 8571   ins_encode %{
 8572     __ shrq($dst$$Address);
 8573   %}
 8574   ins_pipe(ialu_mem_reg);
 8575 %}
 8576 
 8577 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 8578 %{
 8579   predicate(VM_Version::supports_bmi2());
 8580   match(Set dst (URShiftL src shift));
 8581 
 8582   format %{ "shrxq   $dst, $src, $shift" %}
 8583   ins_encode %{
 8584     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 8585   %}
 8586   ins_pipe(ialu_reg_reg);
 8587 %}
 8588 
 8589 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 8590 %{
 8591   predicate(VM_Version::supports_bmi2());
 8592   match(Set dst (URShiftL (LoadL src) shift));
 8593   ins_cost(175);
 8594   format %{ "shrxq   $dst, $src, $shift" %}
 8595   ins_encode %{
 8596     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 8597   %}
 8598   ins_pipe(ialu_reg_mem);
 8599 %}
 8600 
 8601 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 8602 // This idiom is used by the compiler for the i2b bytecode.
 8603 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 8604 %{
 8605   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 8606 
 8607   format %{ "movsbl  $dst, $src\t# i2b" %}
 8608   ins_encode %{
 8609     __ movsbl($dst$$Register, $src$$Register);
 8610   %}
 8611   ins_pipe(ialu_reg_reg);
 8612 %}
 8613 
 8614 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 8615 // This idiom is used by the compiler the i2s bytecode.
 8616 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 8617 %{
 8618   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 8619 
 8620   format %{ "movswl  $dst, $src\t# i2s" %}
 8621   ins_encode %{
 8622     __ movswl($dst$$Register, $src$$Register);
 8623   %}
 8624   ins_pipe(ialu_reg_reg);
 8625 %}
 8626 
 8627 // ROL/ROR instructions
 8628 
 8629 // Rotate left by constant.
 8630 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8631 %{
 8632   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8633   match(Set dst (RotateLeft dst shift));
 8634   effect(KILL cr);
 8635   format %{ "roll    $dst, $shift" %}
 8636   ins_encode %{
 8637     __ roll($dst$$Register, $shift$$constant);
 8638   %}
 8639   ins_pipe(ialu_reg);
 8640 %}
 8641 
 8642 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 8643 %{
 8644   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8645   match(Set dst (RotateLeft src shift));
 8646   format %{ "rolxl   $dst, $src, $shift" %}
 8647   ins_encode %{
 8648     int shift = 32 - ($shift$$constant & 31);
 8649     __ rorxl($dst$$Register, $src$$Register, shift);
 8650   %}
 8651   ins_pipe(ialu_reg_reg);
 8652 %}
 8653 
 8654 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8655 %{
 8656   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8657   match(Set dst (RotateLeft (LoadI src) shift));
 8658   ins_cost(175);
 8659   format %{ "rolxl   $dst, $src, $shift" %}
 8660   ins_encode %{
 8661     int shift = 32 - ($shift$$constant & 31);
 8662     __ rorxl($dst$$Register, $src$$Address, shift);
 8663   %}
 8664   ins_pipe(ialu_reg_mem);
 8665 %}
 8666 
 8667 // Rotate Left by variable
 8668 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8669 %{
 8670   predicate(n->bottom_type()->basic_type() == T_INT);
 8671   match(Set dst (RotateLeft dst shift));
 8672   effect(KILL cr);
 8673   format %{ "roll    $dst, $shift" %}
 8674   ins_encode %{
 8675     __ roll($dst$$Register);
 8676   %}
 8677   ins_pipe(ialu_reg_reg);
 8678 %}
 8679 
 8680 // Rotate Right by constant.
 8681 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 8682 %{
 8683   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8684   match(Set dst (RotateRight dst shift));
 8685   effect(KILL cr);
 8686   format %{ "rorl    $dst, $shift" %}
 8687   ins_encode %{
 8688     __ rorl($dst$$Register, $shift$$constant);
 8689   %}
 8690   ins_pipe(ialu_reg);
 8691 %}
 8692 
 8693 // Rotate Right by constant.
 8694 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 8695 %{
 8696   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8697   match(Set dst (RotateRight src shift));
 8698   format %{ "rorxl   $dst, $src, $shift" %}
 8699   ins_encode %{
 8700     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 8701   %}
 8702   ins_pipe(ialu_reg_reg);
 8703 %}
 8704 
 8705 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 8706 %{
 8707   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 8708   match(Set dst (RotateRight (LoadI src) shift));
 8709   ins_cost(175);
 8710   format %{ "rorxl   $dst, $src, $shift" %}
 8711   ins_encode %{
 8712     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 8713   %}
 8714   ins_pipe(ialu_reg_mem);
 8715 %}
 8716 
 8717 // Rotate Right by variable
 8718 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 8719 %{
 8720   predicate(n->bottom_type()->basic_type() == T_INT);
 8721   match(Set dst (RotateRight dst shift));
 8722   effect(KILL cr);
 8723   format %{ "rorl    $dst, $shift" %}
 8724   ins_encode %{
 8725     __ rorl($dst$$Register);
 8726   %}
 8727   ins_pipe(ialu_reg_reg);
 8728 %}
 8729 
 8730 // Rotate Left by constant.
 8731 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8732 %{
 8733   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8734   match(Set dst (RotateLeft dst shift));
 8735   effect(KILL cr);
 8736   format %{ "rolq    $dst, $shift" %}
 8737   ins_encode %{
 8738     __ rolq($dst$$Register, $shift$$constant);
 8739   %}
 8740   ins_pipe(ialu_reg);
 8741 %}
 8742 
 8743 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 8744 %{
 8745   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8746   match(Set dst (RotateLeft src shift));
 8747   format %{ "rolxq   $dst, $src, $shift" %}
 8748   ins_encode %{
 8749     int shift = 64 - ($shift$$constant & 63);
 8750     __ rorxq($dst$$Register, $src$$Register, shift);
 8751   %}
 8752   ins_pipe(ialu_reg_reg);
 8753 %}
 8754 
 8755 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8756 %{
 8757   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8758   match(Set dst (RotateLeft (LoadL src) shift));
 8759   ins_cost(175);
 8760   format %{ "rolxq   $dst, $src, $shift" %}
 8761   ins_encode %{
 8762     int shift = 64 - ($shift$$constant & 63);
 8763     __ rorxq($dst$$Register, $src$$Address, shift);
 8764   %}
 8765   ins_pipe(ialu_reg_mem);
 8766 %}
 8767 
 8768 // Rotate Left by variable
 8769 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8770 %{
 8771   predicate(n->bottom_type()->basic_type() == T_LONG);
 8772   match(Set dst (RotateLeft dst shift));
 8773   effect(KILL cr);
 8774   format %{ "rolq    $dst, $shift" %}
 8775   ins_encode %{
 8776     __ rolq($dst$$Register);
 8777   %}
 8778   ins_pipe(ialu_reg_reg);
 8779 %}
 8780 
 8781 // Rotate Right by constant.
 8782 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 8783 %{
 8784   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8785   match(Set dst (RotateRight dst shift));
 8786   effect(KILL cr);
 8787   format %{ "rorq    $dst, $shift" %}
 8788   ins_encode %{
 8789     __ rorq($dst$$Register, $shift$$constant);
 8790   %}
 8791   ins_pipe(ialu_reg);
 8792 %}
 8793 
 8794 // Rotate Right by constant
 8795 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 8796 %{
 8797   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8798   match(Set dst (RotateRight src shift));
 8799   format %{ "rorxq   $dst, $src, $shift" %}
 8800   ins_encode %{
 8801     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 8802   %}
 8803   ins_pipe(ialu_reg_reg);
 8804 %}
 8805 
 8806 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 8807 %{
 8808   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 8809   match(Set dst (RotateRight (LoadL src) shift));
 8810   ins_cost(175);
 8811   format %{ "rorxq   $dst, $src, $shift" %}
 8812   ins_encode %{
 8813     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 8814   %}
 8815   ins_pipe(ialu_reg_mem);
 8816 %}
 8817 
 8818 // Rotate Right by variable
 8819 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 8820 %{
 8821   predicate(n->bottom_type()->basic_type() == T_LONG);
 8822   match(Set dst (RotateRight dst shift));
 8823   effect(KILL cr);
 8824   format %{ "rorq    $dst, $shift" %}
 8825   ins_encode %{
 8826     __ rorq($dst$$Register);
 8827   %}
 8828   ins_pipe(ialu_reg_reg);
 8829 %}
 8830 
 8831 //----------------------------- CompressBits/ExpandBits ------------------------
 8832 
 8833 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8834   predicate(n->bottom_type()->isa_long());
 8835   match(Set dst (CompressBits src mask));
 8836   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8837   ins_encode %{
 8838     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 8839   %}
 8840   ins_pipe( pipe_slow );
 8841 %}
 8842 
 8843 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 8844   predicate(n->bottom_type()->isa_long());
 8845   match(Set dst (ExpandBits src mask));
 8846   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8847   ins_encode %{
 8848     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 8849   %}
 8850   ins_pipe( pipe_slow );
 8851 %}
 8852 
 8853 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8854   predicate(n->bottom_type()->isa_long());
 8855   match(Set dst (CompressBits src (LoadL mask)));
 8856   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 8857   ins_encode %{
 8858     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 8859   %}
 8860   ins_pipe( pipe_slow );
 8861 %}
 8862 
 8863 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 8864   predicate(n->bottom_type()->isa_long());
 8865   match(Set dst (ExpandBits src (LoadL mask)));
 8866   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 8867   ins_encode %{
 8868     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
 8869   %}
 8870   ins_pipe( pipe_slow );
 8871 %}
 8872 
 8873 
 8874 // Logical Instructions
 8875 
 8876 // Integer Logical Instructions
 8877 
 8878 // And Instructions
 8879 // And Register with Register
 8880 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8881 %{
 8882   match(Set dst (AndI dst src));
 8883   effect(KILL cr);
 8884   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8885 
 8886   format %{ "andl    $dst, $src\t# int" %}
 8887   ins_encode %{
 8888     __ andl($dst$$Register, $src$$Register);
 8889   %}
 8890   ins_pipe(ialu_reg_reg);
 8891 %}
 8892 
 8893 // And Register with Immediate 255
 8894 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
 8895 %{
 8896   match(Set dst (AndI src mask));
 8897 
 8898   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
 8899   ins_encode %{
 8900     __ movzbl($dst$$Register, $src$$Register);
 8901   %}
 8902   ins_pipe(ialu_reg);
 8903 %}
 8904 
 8905 // And Register with Immediate 255 and promote to long
 8906 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
 8907 %{
 8908   match(Set dst (ConvI2L (AndI src mask)));
 8909 
 8910   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
 8911   ins_encode %{
 8912     __ movzbl($dst$$Register, $src$$Register);
 8913   %}
 8914   ins_pipe(ialu_reg);
 8915 %}
 8916 
 8917 // And Register with Immediate 65535
 8918 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
 8919 %{
 8920   match(Set dst (AndI src mask));
 8921 
 8922   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
 8923   ins_encode %{
 8924     __ movzwl($dst$$Register, $src$$Register);
 8925   %}
 8926   ins_pipe(ialu_reg);
 8927 %}
 8928 
 8929 // And Register with Immediate 65535 and promote to long
 8930 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
 8931 %{
 8932   match(Set dst (ConvI2L (AndI src mask)));
 8933 
 8934   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
 8935   ins_encode %{
 8936     __ movzwl($dst$$Register, $src$$Register);
 8937   %}
 8938   ins_pipe(ialu_reg);
 8939 %}
 8940 
 8941 // Can skip int2long conversions after AND with small bitmask
 8942 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
 8943 %{
 8944   predicate(VM_Version::supports_bmi2());
 8945   ins_cost(125);
 8946   effect(TEMP tmp, KILL cr);
 8947   match(Set dst (ConvI2L (AndI src mask)));
 8948   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
 8949   ins_encode %{
 8950     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
 8951     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
 8952   %}
 8953   ins_pipe(ialu_reg_reg);
 8954 %}
 8955 
 8956 // And Register with Immediate
 8957 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 8958 %{
 8959   match(Set dst (AndI dst src));
 8960   effect(KILL cr);
 8961   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8962 
 8963   format %{ "andl    $dst, $src\t# int" %}
 8964   ins_encode %{
 8965     __ andl($dst$$Register, $src$$constant);
 8966   %}
 8967   ins_pipe(ialu_reg);
 8968 %}
 8969 
 8970 // And Register with Memory
 8971 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8972 %{
 8973   match(Set dst (AndI dst (LoadI src)));
 8974   effect(KILL cr);
 8975   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8976 
 8977   ins_cost(150);
 8978   format %{ "andl    $dst, $src\t# int" %}
 8979   ins_encode %{
 8980     __ andl($dst$$Register, $src$$Address);
 8981   %}
 8982   ins_pipe(ialu_reg_mem);
 8983 %}
 8984 
 8985 // And Memory with Register
 8986 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8987 %{
 8988   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
 8989   effect(KILL cr);
 8990   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 8991 
 8992   ins_cost(150);
 8993   format %{ "andb    $dst, $src\t# byte" %}
 8994   ins_encode %{
 8995     __ andb($dst$$Address, $src$$Register);
 8996   %}
 8997   ins_pipe(ialu_mem_reg);
 8998 %}
 8999 
 9000 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9001 %{
 9002   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9003   effect(KILL cr);
 9004   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9005 
 9006   ins_cost(150);
 9007   format %{ "andl    $dst, $src\t# int" %}
 9008   ins_encode %{
 9009     __ andl($dst$$Address, $src$$Register);
 9010   %}
 9011   ins_pipe(ialu_mem_reg);
 9012 %}
 9013 
 9014 // And Memory with Immediate
 9015 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9016 %{
 9017   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
 9018   effect(KILL cr);
 9019   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9020 
 9021   ins_cost(125);
 9022   format %{ "andl    $dst, $src\t# int" %}
 9023   ins_encode %{
 9024     __ andl($dst$$Address, $src$$constant);
 9025   %}
 9026   ins_pipe(ialu_mem_imm);
 9027 %}
 9028 
 9029 // BMI1 instructions
 9030 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
 9031   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
 9032   predicate(UseBMI1Instructions);
 9033   effect(KILL cr);
 9034   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9035 
 9036   ins_cost(125);
 9037   format %{ "andnl  $dst, $src1, $src2" %}
 9038 
 9039   ins_encode %{
 9040     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
 9041   %}
 9042   ins_pipe(ialu_reg_mem);
 9043 %}
 9044 
 9045 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
 9046   match(Set dst (AndI (XorI src1 minus_1) src2));
 9047   predicate(UseBMI1Instructions);
 9048   effect(KILL cr);
 9049   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9050 
 9051   format %{ "andnl  $dst, $src1, $src2" %}
 9052 
 9053   ins_encode %{
 9054     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
 9055   %}
 9056   ins_pipe(ialu_reg);
 9057 %}
 9058 
 9059 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
 9060   match(Set dst (AndI (SubI imm_zero src) src));
 9061   predicate(UseBMI1Instructions);
 9062   effect(KILL cr);
 9063   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9064 
 9065   format %{ "blsil  $dst, $src" %}
 9066 
 9067   ins_encode %{
 9068     __ blsil($dst$$Register, $src$$Register);
 9069   %}
 9070   ins_pipe(ialu_reg);
 9071 %}
 9072 
 9073 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
 9074   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
 9075   predicate(UseBMI1Instructions);
 9076   effect(KILL cr);
 9077   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9078 
 9079   ins_cost(125);
 9080   format %{ "blsil  $dst, $src" %}
 9081 
 9082   ins_encode %{
 9083     __ blsil($dst$$Register, $src$$Address);
 9084   %}
 9085   ins_pipe(ialu_reg_mem);
 9086 %}
 9087 
 9088 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9089 %{
 9090   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9091   predicate(UseBMI1Instructions);
 9092   effect(KILL cr);
 9093   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9094 
 9095   ins_cost(125);
 9096   format %{ "blsmskl $dst, $src" %}
 9097 
 9098   ins_encode %{
 9099     __ blsmskl($dst$$Register, $src$$Address);
 9100   %}
 9101   ins_pipe(ialu_reg_mem);
 9102 %}
 9103 
 9104 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9105 %{
 9106   match(Set dst (XorI (AddI src minus_1) src));
 9107   predicate(UseBMI1Instructions);
 9108   effect(KILL cr);
 9109   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9110 
 9111   format %{ "blsmskl $dst, $src" %}
 9112 
 9113   ins_encode %{
 9114     __ blsmskl($dst$$Register, $src$$Register);
 9115   %}
 9116 
 9117   ins_pipe(ialu_reg);
 9118 %}
 9119 
 9120 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
 9121 %{
 9122   match(Set dst (AndI (AddI src minus_1) src) );
 9123   predicate(UseBMI1Instructions);
 9124   effect(KILL cr);
 9125   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9126 
 9127   format %{ "blsrl  $dst, $src" %}
 9128 
 9129   ins_encode %{
 9130     __ blsrl($dst$$Register, $src$$Register);
 9131   %}
 9132 
 9133   ins_pipe(ialu_reg_mem);
 9134 %}
 9135 
 9136 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
 9137 %{
 9138   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
 9139   predicate(UseBMI1Instructions);
 9140   effect(KILL cr);
 9141   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9142 
 9143   ins_cost(125);
 9144   format %{ "blsrl  $dst, $src" %}
 9145 
 9146   ins_encode %{
 9147     __ blsrl($dst$$Register, $src$$Address);
 9148   %}
 9149 
 9150   ins_pipe(ialu_reg);
 9151 %}
 9152 
 9153 // Or Instructions
 9154 // Or Register with Register
 9155 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9156 %{
 9157   match(Set dst (OrI dst src));
 9158   effect(KILL cr);
 9159   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9160 
 9161   format %{ "orl     $dst, $src\t# int" %}
 9162   ins_encode %{
 9163     __ orl($dst$$Register, $src$$Register);
 9164   %}
 9165   ins_pipe(ialu_reg_reg);
 9166 %}
 9167 
 9168 // Or Register with Immediate
 9169 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9170 %{
 9171   match(Set dst (OrI dst src));
 9172   effect(KILL cr);
 9173   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9174 
 9175   format %{ "orl     $dst, $src\t# int" %}
 9176   ins_encode %{
 9177     __ orl($dst$$Register, $src$$constant);
 9178   %}
 9179   ins_pipe(ialu_reg);
 9180 %}
 9181 
 9182 // Or Register with Memory
 9183 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9184 %{
 9185   match(Set dst (OrI dst (LoadI src)));
 9186   effect(KILL cr);
 9187   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9188 
 9189   ins_cost(150);
 9190   format %{ "orl     $dst, $src\t# int" %}
 9191   ins_encode %{
 9192     __ orl($dst$$Register, $src$$Address);
 9193   %}
 9194   ins_pipe(ialu_reg_mem);
 9195 %}
 9196 
 9197 // Or Memory with Register
 9198 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9199 %{
 9200   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
 9201   effect(KILL cr);
 9202   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9203 
 9204   ins_cost(150);
 9205   format %{ "orb    $dst, $src\t# byte" %}
 9206   ins_encode %{
 9207     __ orb($dst$$Address, $src$$Register);
 9208   %}
 9209   ins_pipe(ialu_mem_reg);
 9210 %}
 9211 
 9212 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9213 %{
 9214   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9215   effect(KILL cr);
 9216   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9217 
 9218   ins_cost(150);
 9219   format %{ "orl     $dst, $src\t# int" %}
 9220   ins_encode %{
 9221     __ orl($dst$$Address, $src$$Register);
 9222   %}
 9223   ins_pipe(ialu_mem_reg);
 9224 %}
 9225 
 9226 // Or Memory with Immediate
 9227 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9228 %{
 9229   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
 9230   effect(KILL cr);
 9231   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9232 
 9233   ins_cost(125);
 9234   format %{ "orl     $dst, $src\t# int" %}
 9235   ins_encode %{
 9236     __ orl($dst$$Address, $src$$constant);
 9237   %}
 9238   ins_pipe(ialu_mem_imm);
 9239 %}
 9240 
 9241 // Xor Instructions
 9242 // Xor Register with Register
 9243 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 9244 %{
 9245   match(Set dst (XorI dst src));
 9246   effect(KILL cr);
 9247   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9248 
 9249   format %{ "xorl    $dst, $src\t# int" %}
 9250   ins_encode %{
 9251     __ xorl($dst$$Register, $src$$Register);
 9252   %}
 9253   ins_pipe(ialu_reg_reg);
 9254 %}
 9255 
 9256 // Xor Register with Immediate -1
 9257 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
 9258   match(Set dst (XorI dst imm));
 9259 
 9260   format %{ "not    $dst" %}
 9261   ins_encode %{
 9262      __ notl($dst$$Register);
 9263   %}
 9264   ins_pipe(ialu_reg);
 9265 %}
 9266 
 9267 // Xor Register with Immediate
 9268 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 9269 %{
 9270   match(Set dst (XorI dst src));
 9271   effect(KILL cr);
 9272   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9273 
 9274   format %{ "xorl    $dst, $src\t# int" %}
 9275   ins_encode %{
 9276     __ xorl($dst$$Register, $src$$constant);
 9277   %}
 9278   ins_pipe(ialu_reg);
 9279 %}
 9280 
 9281 // Xor Register with Memory
 9282 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 9283 %{
 9284   match(Set dst (XorI dst (LoadI src)));
 9285   effect(KILL cr);
 9286   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9287 
 9288   ins_cost(150);
 9289   format %{ "xorl    $dst, $src\t# int" %}
 9290   ins_encode %{
 9291     __ xorl($dst$$Register, $src$$Address);
 9292   %}
 9293   ins_pipe(ialu_reg_mem);
 9294 %}
 9295 
 9296 // Xor Memory with Register
 9297 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9298 %{
 9299   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
 9300   effect(KILL cr);
 9301   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9302 
 9303   ins_cost(150);
 9304   format %{ "xorb    $dst, $src\t# byte" %}
 9305   ins_encode %{
 9306     __ xorb($dst$$Address, $src$$Register);
 9307   %}
 9308   ins_pipe(ialu_mem_reg);
 9309 %}
 9310 
 9311 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 9312 %{
 9313   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9314   effect(KILL cr);
 9315   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9316 
 9317   ins_cost(150);
 9318   format %{ "xorl    $dst, $src\t# int" %}
 9319   ins_encode %{
 9320     __ xorl($dst$$Address, $src$$Register);
 9321   %}
 9322   ins_pipe(ialu_mem_reg);
 9323 %}
 9324 
 9325 // Xor Memory with Immediate
 9326 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
 9327 %{
 9328   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
 9329   effect(KILL cr);
 9330   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9331 
 9332   ins_cost(125);
 9333   format %{ "xorl    $dst, $src\t# int" %}
 9334   ins_encode %{
 9335     __ xorl($dst$$Address, $src$$constant);
 9336   %}
 9337   ins_pipe(ialu_mem_imm);
 9338 %}
 9339 
 9340 
 9341 // Long Logical Instructions
 9342 
 9343 // And Instructions
 9344 // And Register with Register
 9345 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9346 %{
 9347   match(Set dst (AndL dst src));
 9348   effect(KILL cr);
 9349   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9350 
 9351   format %{ "andq    $dst, $src\t# long" %}
 9352   ins_encode %{
 9353     __ andq($dst$$Register, $src$$Register);
 9354   %}
 9355   ins_pipe(ialu_reg_reg);
 9356 %}
 9357 
 9358 // And Register with Immediate 255
 9359 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
 9360 %{
 9361   match(Set dst (AndL src mask));
 9362 
 9363   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
 9364   ins_encode %{
 9365     // movzbl zeroes out the upper 32-bit and does not need REX.W
 9366     __ movzbl($dst$$Register, $src$$Register);
 9367   %}
 9368   ins_pipe(ialu_reg);
 9369 %}
 9370 
 9371 // And Register with Immediate 65535
 9372 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
 9373 %{
 9374   match(Set dst (AndL src mask));
 9375 
 9376   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
 9377   ins_encode %{
 9378     // movzwl zeroes out the upper 32-bit and does not need REX.W
 9379     __ movzwl($dst$$Register, $src$$Register);
 9380   %}
 9381   ins_pipe(ialu_reg);
 9382 %}
 9383 
 9384 // And Register with Immediate
 9385 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9386 %{
 9387   match(Set dst (AndL dst src));
 9388   effect(KILL cr);
 9389   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9390 
 9391   format %{ "andq    $dst, $src\t# long" %}
 9392   ins_encode %{
 9393     __ andq($dst$$Register, $src$$constant);
 9394   %}
 9395   ins_pipe(ialu_reg);
 9396 %}
 9397 
 9398 // And Register with Memory
 9399 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9400 %{
 9401   match(Set dst (AndL dst (LoadL src)));
 9402   effect(KILL cr);
 9403   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9404 
 9405   ins_cost(150);
 9406   format %{ "andq    $dst, $src\t# long" %}
 9407   ins_encode %{
 9408     __ andq($dst$$Register, $src$$Address);
 9409   %}
 9410   ins_pipe(ialu_reg_mem);
 9411 %}
 9412 
 9413 // And Memory with Register
 9414 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9415 %{
 9416   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9417   effect(KILL cr);
 9418   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9419 
 9420   ins_cost(150);
 9421   format %{ "andq    $dst, $src\t# long" %}
 9422   ins_encode %{
 9423     __ andq($dst$$Address, $src$$Register);
 9424   %}
 9425   ins_pipe(ialu_mem_reg);
 9426 %}
 9427 
 9428 // And Memory with Immediate
 9429 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9430 %{
 9431   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
 9432   effect(KILL cr);
 9433   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9434 
 9435   ins_cost(125);
 9436   format %{ "andq    $dst, $src\t# long" %}
 9437   ins_encode %{
 9438     __ andq($dst$$Address, $src$$constant);
 9439   %}
 9440   ins_pipe(ialu_mem_imm);
 9441 %}
 9442 
 9443 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
 9444 %{
 9445   // con should be a pure 64-bit immediate given that not(con) is a power of 2
 9446   // because AND/OR works well enough for 8/32-bit values.
 9447   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
 9448 
 9449   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
 9450   effect(KILL cr);
 9451 
 9452   ins_cost(125);
 9453   format %{ "btrq    $dst, log2(not($con))\t# long" %}
 9454   ins_encode %{
 9455     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
 9456   %}
 9457   ins_pipe(ialu_mem_imm);
 9458 %}
 9459 
 9460 // BMI1 instructions
 9461 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
 9462   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
 9463   predicate(UseBMI1Instructions);
 9464   effect(KILL cr);
 9465   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9466 
 9467   ins_cost(125);
 9468   format %{ "andnq  $dst, $src1, $src2" %}
 9469 
 9470   ins_encode %{
 9471     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
 9472   %}
 9473   ins_pipe(ialu_reg_mem);
 9474 %}
 9475 
 9476 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
 9477   match(Set dst (AndL (XorL src1 minus_1) src2));
 9478   predicate(UseBMI1Instructions);
 9479   effect(KILL cr);
 9480   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9481 
 9482   format %{ "andnq  $dst, $src1, $src2" %}
 9483 
 9484   ins_encode %{
 9485   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
 9486   %}
 9487   ins_pipe(ialu_reg_mem);
 9488 %}
 9489 
 9490 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
 9491   match(Set dst (AndL (SubL imm_zero src) src));
 9492   predicate(UseBMI1Instructions);
 9493   effect(KILL cr);
 9494   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9495 
 9496   format %{ "blsiq  $dst, $src" %}
 9497 
 9498   ins_encode %{
 9499     __ blsiq($dst$$Register, $src$$Register);
 9500   %}
 9501   ins_pipe(ialu_reg);
 9502 %}
 9503 
 9504 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
 9505   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
 9506   predicate(UseBMI1Instructions);
 9507   effect(KILL cr);
 9508   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9509 
 9510   ins_cost(125);
 9511   format %{ "blsiq  $dst, $src" %}
 9512 
 9513   ins_encode %{
 9514     __ blsiq($dst$$Register, $src$$Address);
 9515   %}
 9516   ins_pipe(ialu_reg_mem);
 9517 %}
 9518 
 9519 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9520 %{
 9521   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
 9522   predicate(UseBMI1Instructions);
 9523   effect(KILL cr);
 9524   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9525 
 9526   ins_cost(125);
 9527   format %{ "blsmskq $dst, $src" %}
 9528 
 9529   ins_encode %{
 9530     __ blsmskq($dst$$Register, $src$$Address);
 9531   %}
 9532   ins_pipe(ialu_reg_mem);
 9533 %}
 9534 
 9535 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9536 %{
 9537   match(Set dst (XorL (AddL src minus_1) src));
 9538   predicate(UseBMI1Instructions);
 9539   effect(KILL cr);
 9540   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
 9541 
 9542   format %{ "blsmskq $dst, $src" %}
 9543 
 9544   ins_encode %{
 9545     __ blsmskq($dst$$Register, $src$$Register);
 9546   %}
 9547 
 9548   ins_pipe(ialu_reg);
 9549 %}
 9550 
 9551 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
 9552 %{
 9553   match(Set dst (AndL (AddL src minus_1) src) );
 9554   predicate(UseBMI1Instructions);
 9555   effect(KILL cr);
 9556   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9557 
 9558   format %{ "blsrq  $dst, $src" %}
 9559 
 9560   ins_encode %{
 9561     __ blsrq($dst$$Register, $src$$Register);
 9562   %}
 9563 
 9564   ins_pipe(ialu_reg);
 9565 %}
 9566 
 9567 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
 9568 %{
 9569   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
 9570   predicate(UseBMI1Instructions);
 9571   effect(KILL cr);
 9572   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
 9573 
 9574   ins_cost(125);
 9575   format %{ "blsrq  $dst, $src" %}
 9576 
 9577   ins_encode %{
 9578     __ blsrq($dst$$Register, $src$$Address);
 9579   %}
 9580 
 9581   ins_pipe(ialu_reg);
 9582 %}
 9583 
 9584 // Or Instructions
 9585 // Or Register with Register
 9586 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9587 %{
 9588   match(Set dst (OrL dst src));
 9589   effect(KILL cr);
 9590   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9591 
 9592   format %{ "orq     $dst, $src\t# long" %}
 9593   ins_encode %{
 9594     __ orq($dst$$Register, $src$$Register);
 9595   %}
 9596   ins_pipe(ialu_reg_reg);
 9597 %}
 9598 
 9599 // Use any_RegP to match R15 (TLS register) without spilling.
 9600 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
 9601   match(Set dst (OrL dst (CastP2X src)));
 9602   effect(KILL cr);
 9603   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9604 
 9605   format %{ "orq     $dst, $src\t# long" %}
 9606   ins_encode %{
 9607     __ orq($dst$$Register, $src$$Register);
 9608   %}
 9609   ins_pipe(ialu_reg_reg);
 9610 %}
 9611 
 9612 
 9613 // Or Register with Immediate
 9614 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9615 %{
 9616   match(Set dst (OrL dst src));
 9617   effect(KILL cr);
 9618   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9619 
 9620   format %{ "orq     $dst, $src\t# long" %}
 9621   ins_encode %{
 9622     __ orq($dst$$Register, $src$$constant);
 9623   %}
 9624   ins_pipe(ialu_reg);
 9625 %}
 9626 
 9627 // Or Register with Memory
 9628 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9629 %{
 9630   match(Set dst (OrL dst (LoadL src)));
 9631   effect(KILL cr);
 9632   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9633 
 9634   ins_cost(150);
 9635   format %{ "orq     $dst, $src\t# long" %}
 9636   ins_encode %{
 9637     __ orq($dst$$Register, $src$$Address);
 9638   %}
 9639   ins_pipe(ialu_reg_mem);
 9640 %}
 9641 
 9642 // Or Memory with Register
 9643 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9644 %{
 9645   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9646   effect(KILL cr);
 9647   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9648 
 9649   ins_cost(150);
 9650   format %{ "orq     $dst, $src\t# long" %}
 9651   ins_encode %{
 9652     __ orq($dst$$Address, $src$$Register);
 9653   %}
 9654   ins_pipe(ialu_mem_reg);
 9655 %}
 9656 
 9657 // Or Memory with Immediate
 9658 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9659 %{
 9660   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
 9661   effect(KILL cr);
 9662   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9663 
 9664   ins_cost(125);
 9665   format %{ "orq     $dst, $src\t# long" %}
 9666   ins_encode %{
 9667     __ orq($dst$$Address, $src$$constant);
 9668   %}
 9669   ins_pipe(ialu_mem_imm);
 9670 %}
 9671 
 9672 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
 9673 %{
 9674   // con should be a pure 64-bit power of 2 immediate
 9675   // because AND/OR works well enough for 8/32-bit values.
 9676   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
 9677 
 9678   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
 9679   effect(KILL cr);
 9680 
 9681   ins_cost(125);
 9682   format %{ "btsq    $dst, log2($con)\t# long" %}
 9683   ins_encode %{
 9684     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
 9685   %}
 9686   ins_pipe(ialu_mem_imm);
 9687 %}
 9688 
 9689 // Xor Instructions
 9690 // Xor Register with Register
 9691 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 9692 %{
 9693   match(Set dst (XorL dst src));
 9694   effect(KILL cr);
 9695   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9696 
 9697   format %{ "xorq    $dst, $src\t# long" %}
 9698   ins_encode %{
 9699     __ xorq($dst$$Register, $src$$Register);
 9700   %}
 9701   ins_pipe(ialu_reg_reg);
 9702 %}
 9703 
 9704 // Xor Register with Immediate -1
 9705 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
 9706   match(Set dst (XorL dst imm));
 9707 
 9708   format %{ "notq   $dst" %}
 9709   ins_encode %{
 9710      __ notq($dst$$Register);
 9711   %}
 9712   ins_pipe(ialu_reg);
 9713 %}
 9714 
 9715 // Xor Register with Immediate
 9716 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 9717 %{
 9718   match(Set dst (XorL dst src));
 9719   effect(KILL cr);
 9720   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9721 
 9722   format %{ "xorq    $dst, $src\t# long" %}
 9723   ins_encode %{
 9724     __ xorq($dst$$Register, $src$$constant);
 9725   %}
 9726   ins_pipe(ialu_reg);
 9727 %}
 9728 
 9729 // Xor Register with Memory
 9730 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 9731 %{
 9732   match(Set dst (XorL dst (LoadL src)));
 9733   effect(KILL cr);
 9734   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9735 
 9736   ins_cost(150);
 9737   format %{ "xorq    $dst, $src\t# long" %}
 9738   ins_encode %{
 9739     __ xorq($dst$$Register, $src$$Address);
 9740   %}
 9741   ins_pipe(ialu_reg_mem);
 9742 %}
 9743 
 9744 // Xor Memory with Register
 9745 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 9746 %{
 9747   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9748   effect(KILL cr);
 9749   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9750 
 9751   ins_cost(150);
 9752   format %{ "xorq    $dst, $src\t# long" %}
 9753   ins_encode %{
 9754     __ xorq($dst$$Address, $src$$Register);
 9755   %}
 9756   ins_pipe(ialu_mem_reg);
 9757 %}
 9758 
 9759 // Xor Memory with Immediate
 9760 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 9761 %{
 9762   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
 9763   effect(KILL cr);
 9764   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
 9765 
 9766   ins_cost(125);
 9767   format %{ "xorq    $dst, $src\t# long" %}
 9768   ins_encode %{
 9769     __ xorq($dst$$Address, $src$$constant);
 9770   %}
 9771   ins_pipe(ialu_mem_imm);
 9772 %}
 9773 
 9774 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
 9775 %{
 9776   match(Set dst (CmpLTMask p q));
 9777   effect(KILL cr);
 9778 
 9779   ins_cost(400);
 9780   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
 9781             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
 9782             "negl    $dst" %}
 9783   ins_encode %{
 9784     __ cmpl($p$$Register, $q$$Register);
 9785     __ setcc(Assembler::less, $dst$$Register);
 9786     __ negl($dst$$Register);
 9787   %}
 9788   ins_pipe(pipe_slow);
 9789 %}
 9790 
 9791 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
 9792 %{
 9793   match(Set dst (CmpLTMask dst zero));
 9794   effect(KILL cr);
 9795 
 9796   ins_cost(100);
 9797   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
 9798   ins_encode %{
 9799     __ sarl($dst$$Register, 31);
 9800   %}
 9801   ins_pipe(ialu_reg);
 9802 %}
 9803 
 9804 /* Better to save a register than avoid a branch */
 9805 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9806 %{
 9807   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
 9808   effect(KILL cr);
 9809   ins_cost(300);
 9810   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
 9811             "jge     done\n\t"
 9812             "addl    $p,$y\n"
 9813             "done:   " %}
 9814   ins_encode %{
 9815     Register Rp = $p$$Register;
 9816     Register Rq = $q$$Register;
 9817     Register Ry = $y$$Register;
 9818     Label done;
 9819     __ subl(Rp, Rq);
 9820     __ jccb(Assembler::greaterEqual, done);
 9821     __ addl(Rp, Ry);
 9822     __ bind(done);
 9823   %}
 9824   ins_pipe(pipe_cmplt);
 9825 %}
 9826 
 9827 /* Better to save a register than avoid a branch */
 9828 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
 9829 %{
 9830   match(Set y (AndI (CmpLTMask p q) y));
 9831   effect(KILL cr);
 9832 
 9833   ins_cost(300);
 9834 
 9835   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
 9836             "jlt     done\n\t"
 9837             "xorl    $y, $y\n"
 9838             "done:   " %}
 9839   ins_encode %{
 9840     Register Rp = $p$$Register;
 9841     Register Rq = $q$$Register;
 9842     Register Ry = $y$$Register;
 9843     Label done;
 9844     __ cmpl(Rp, Rq);
 9845     __ jccb(Assembler::less, done);
 9846     __ xorl(Ry, Ry);
 9847     __ bind(done);
 9848   %}
 9849   ins_pipe(pipe_cmplt);
 9850 %}
 9851 
 9852 
 9853 //---------- FP Instructions------------------------------------------------
 9854 
 9855 // Really expensive, avoid
 9856 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
 9857 %{
 9858   match(Set cr (CmpF src1 src2));
 9859 
 9860   ins_cost(500);
 9861   format %{ "ucomiss $src1, $src2\n\t"
 9862             "jnp,s   exit\n\t"
 9863             "pushfq\t# saw NaN, set CF\n\t"
 9864             "andq    [rsp], #0xffffff2b\n\t"
 9865             "popfq\n"
 9866     "exit:" %}
 9867   ins_encode %{
 9868     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9869     emit_cmpfp_fixup(masm);
 9870   %}
 9871   ins_pipe(pipe_slow);
 9872 %}
 9873 
 9874 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 9875   match(Set cr (CmpF src1 src2));
 9876 
 9877   ins_cost(100);
 9878   format %{ "ucomiss $src1, $src2" %}
 9879   ins_encode %{
 9880     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9881   %}
 9882   ins_pipe(pipe_slow);
 9883 %}
 9884 
 9885 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 9886   match(Set cr (CmpF src1 (LoadF src2)));
 9887 
 9888   ins_cost(100);
 9889   format %{ "ucomiss $src1, $src2" %}
 9890   ins_encode %{
 9891     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9892   %}
 9893   ins_pipe(pipe_slow);
 9894 %}
 9895 
 9896 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
 9897   match(Set cr (CmpF src con));
 9898   ins_cost(100);
 9899   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
 9900   ins_encode %{
 9901     __ ucomiss($src$$XMMRegister, $constantaddress($con));
 9902   %}
 9903   ins_pipe(pipe_slow);
 9904 %}
 9905 
 9906 // Really expensive, avoid
 9907 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
 9908 %{
 9909   match(Set cr (CmpD src1 src2));
 9910 
 9911   ins_cost(500);
 9912   format %{ "ucomisd $src1, $src2\n\t"
 9913             "jnp,s   exit\n\t"
 9914             "pushfq\t# saw NaN, set CF\n\t"
 9915             "andq    [rsp], #0xffffff2b\n\t"
 9916             "popfq\n"
 9917     "exit:" %}
 9918   ins_encode %{
 9919     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9920     emit_cmpfp_fixup(masm);
 9921   %}
 9922   ins_pipe(pipe_slow);
 9923 %}
 9924 
 9925 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 9926   match(Set cr (CmpD src1 src2));
 9927 
 9928   ins_cost(100);
 9929   format %{ "ucomisd $src1, $src2 test" %}
 9930   ins_encode %{
 9931     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
 9932   %}
 9933   ins_pipe(pipe_slow);
 9934 %}
 9935 
 9936 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 9937   match(Set cr (CmpD src1 (LoadD src2)));
 9938 
 9939   ins_cost(100);
 9940   format %{ "ucomisd $src1, $src2" %}
 9941   ins_encode %{
 9942     __ ucomisd($src1$$XMMRegister, $src2$$Address);
 9943   %}
 9944   ins_pipe(pipe_slow);
 9945 %}
 9946 
 9947 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
 9948   match(Set cr (CmpD src con));
 9949   ins_cost(100);
 9950   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
 9951   ins_encode %{
 9952     __ ucomisd($src$$XMMRegister, $constantaddress($con));
 9953   %}
 9954   ins_pipe(pipe_slow);
 9955 %}
 9956 
 9957 // Compare into -1,0,1
 9958 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
 9959 %{
 9960   match(Set dst (CmpF3 src1 src2));
 9961   effect(KILL cr);
 9962 
 9963   ins_cost(275);
 9964   format %{ "ucomiss $src1, $src2\n\t"
 9965             "movl    $dst, #-1\n\t"
 9966             "jp,s    done\n\t"
 9967             "jb,s    done\n\t"
 9968             "setne   $dst\n\t"
 9969             "movzbl  $dst, $dst\n"
 9970     "done:" %}
 9971   ins_encode %{
 9972     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 9973     emit_cmpfp3(masm, $dst$$Register);
 9974   %}
 9975   ins_pipe(pipe_slow);
 9976 %}
 9977 
 9978 // Compare into -1,0,1
 9979 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 9980 %{
 9981   match(Set dst (CmpF3 src1 (LoadF src2)));
 9982   effect(KILL cr);
 9983 
 9984   ins_cost(275);
 9985   format %{ "ucomiss $src1, $src2\n\t"
 9986             "movl    $dst, #-1\n\t"
 9987             "jp,s    done\n\t"
 9988             "jb,s    done\n\t"
 9989             "setne   $dst\n\t"
 9990             "movzbl  $dst, $dst\n"
 9991     "done:" %}
 9992   ins_encode %{
 9993     __ ucomiss($src1$$XMMRegister, $src2$$Address);
 9994     emit_cmpfp3(masm, $dst$$Register);
 9995   %}
 9996   ins_pipe(pipe_slow);
 9997 %}
 9998 
 9999 // Compare into -1,0,1
10000 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10001   match(Set dst (CmpF3 src con));
10002   effect(KILL cr);
10003 
10004   ins_cost(275);
10005   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10006             "movl    $dst, #-1\n\t"
10007             "jp,s    done\n\t"
10008             "jb,s    done\n\t"
10009             "setne   $dst\n\t"
10010             "movzbl  $dst, $dst\n"
10011     "done:" %}
10012   ins_encode %{
10013     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10014     emit_cmpfp3(masm, $dst$$Register);
10015   %}
10016   ins_pipe(pipe_slow);
10017 %}
10018 
10019 // Compare into -1,0,1
10020 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10021 %{
10022   match(Set dst (CmpD3 src1 src2));
10023   effect(KILL cr);
10024 
10025   ins_cost(275);
10026   format %{ "ucomisd $src1, $src2\n\t"
10027             "movl    $dst, #-1\n\t"
10028             "jp,s    done\n\t"
10029             "jb,s    done\n\t"
10030             "setne   $dst\n\t"
10031             "movzbl  $dst, $dst\n"
10032     "done:" %}
10033   ins_encode %{
10034     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10035     emit_cmpfp3(masm, $dst$$Register);
10036   %}
10037   ins_pipe(pipe_slow);
10038 %}
10039 
10040 // Compare into -1,0,1
10041 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10042 %{
10043   match(Set dst (CmpD3 src1 (LoadD src2)));
10044   effect(KILL cr);
10045 
10046   ins_cost(275);
10047   format %{ "ucomisd $src1, $src2\n\t"
10048             "movl    $dst, #-1\n\t"
10049             "jp,s    done\n\t"
10050             "jb,s    done\n\t"
10051             "setne   $dst\n\t"
10052             "movzbl  $dst, $dst\n"
10053     "done:" %}
10054   ins_encode %{
10055     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10056     emit_cmpfp3(masm, $dst$$Register);
10057   %}
10058   ins_pipe(pipe_slow);
10059 %}
10060 
10061 // Compare into -1,0,1
10062 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10063   match(Set dst (CmpD3 src con));
10064   effect(KILL cr);
10065 
10066   ins_cost(275);
10067   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10068             "movl    $dst, #-1\n\t"
10069             "jp,s    done\n\t"
10070             "jb,s    done\n\t"
10071             "setne   $dst\n\t"
10072             "movzbl  $dst, $dst\n"
10073     "done:" %}
10074   ins_encode %{
10075     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10076     emit_cmpfp3(masm, $dst$$Register);
10077   %}
10078   ins_pipe(pipe_slow);
10079 %}
10080 
10081 //----------Arithmetic Conversion Instructions---------------------------------
10082 
10083 instruct convF2D_reg_reg(regD dst, regF src)
10084 %{
10085   match(Set dst (ConvF2D src));
10086 
10087   format %{ "cvtss2sd $dst, $src" %}
10088   ins_encode %{
10089     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10090   %}
10091   ins_pipe(pipe_slow); // XXX
10092 %}
10093 
10094 instruct convF2D_reg_mem(regD dst, memory src)
10095 %{
10096   predicate(UseAVX == 0);
10097   match(Set dst (ConvF2D (LoadF src)));
10098 
10099   format %{ "cvtss2sd $dst, $src" %}
10100   ins_encode %{
10101     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10102   %}
10103   ins_pipe(pipe_slow); // XXX
10104 %}
10105 
10106 instruct convD2F_reg_reg(regF dst, regD src)
10107 %{
10108   match(Set dst (ConvD2F src));
10109 
10110   format %{ "cvtsd2ss $dst, $src" %}
10111   ins_encode %{
10112     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10113   %}
10114   ins_pipe(pipe_slow); // XXX
10115 %}
10116 
10117 instruct convD2F_reg_mem(regF dst, memory src)
10118 %{
10119   predicate(UseAVX == 0);
10120   match(Set dst (ConvD2F (LoadD src)));
10121 
10122   format %{ "cvtsd2ss $dst, $src" %}
10123   ins_encode %{
10124     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10125   %}
10126   ins_pipe(pipe_slow); // XXX
10127 %}
10128 
10129 // XXX do mem variants
10130 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10131 %{
10132   match(Set dst (ConvF2I src));
10133   effect(KILL cr);
10134   format %{ "convert_f2i $dst, $src" %}
10135   ins_encode %{
10136     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10137   %}
10138   ins_pipe(pipe_slow);
10139 %}
10140 
10141 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10142 %{
10143   match(Set dst (ConvF2L src));
10144   effect(KILL cr);
10145   format %{ "convert_f2l $dst, $src"%}
10146   ins_encode %{
10147     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
10148   %}
10149   ins_pipe(pipe_slow);
10150 %}
10151 
10152 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10153 %{
10154   match(Set dst (ConvD2I src));
10155   effect(KILL cr);
10156   format %{ "convert_d2i $dst, $src"%}
10157   ins_encode %{
10158     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10159   %}
10160   ins_pipe(pipe_slow);
10161 %}
10162 
10163 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10164 %{
10165   match(Set dst (ConvD2L src));
10166   effect(KILL cr);
10167   format %{ "convert_d2l $dst, $src"%}
10168   ins_encode %{
10169     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
10170   %}
10171   ins_pipe(pipe_slow);
10172 %}
10173 
10174 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10175 %{
10176   match(Set dst (RoundD src));
10177   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10178   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
10179   ins_encode %{
10180     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10181   %}
10182   ins_pipe(pipe_slow);
10183 %}
10184 
10185 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
10186 %{
10187   match(Set dst (RoundF src));
10188   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
10189   format %{ "round_float $dst,$src" %}
10190   ins_encode %{
10191     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
10192   %}
10193   ins_pipe(pipe_slow);
10194 %}
10195 
10196 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
10197 %{
10198   predicate(!UseXmmI2F);
10199   match(Set dst (ConvI2F src));
10200 
10201   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10202   ins_encode %{
10203     if (UseAVX > 0) {
10204       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10205     }
10206     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10207   %}
10208   ins_pipe(pipe_slow); // XXX
10209 %}
10210 
10211 instruct convI2F_reg_mem(regF dst, memory src)
10212 %{
10213   predicate(UseAVX == 0);
10214   match(Set dst (ConvI2F (LoadI src)));
10215 
10216   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10217   ins_encode %{
10218     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10219   %}
10220   ins_pipe(pipe_slow); // XXX
10221 %}
10222 
10223 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
10224 %{
10225   predicate(!UseXmmI2D);
10226   match(Set dst (ConvI2D src));
10227 
10228   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10229   ins_encode %{
10230     if (UseAVX > 0) {
10231       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10232     }
10233     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10234   %}
10235   ins_pipe(pipe_slow); // XXX
10236 %}
10237 
10238 instruct convI2D_reg_mem(regD dst, memory src)
10239 %{
10240   predicate(UseAVX == 0);
10241   match(Set dst (ConvI2D (LoadI src)));
10242 
10243   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10244   ins_encode %{
10245     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10246   %}
10247   ins_pipe(pipe_slow); // XXX
10248 %}
10249 
10250 instruct convXI2F_reg(regF dst, rRegI src)
10251 %{
10252   predicate(UseXmmI2F);
10253   match(Set dst (ConvI2F src));
10254 
10255   format %{ "movdl $dst, $src\n\t"
10256             "cvtdq2psl $dst, $dst\t# i2f" %}
10257   ins_encode %{
10258     __ movdl($dst$$XMMRegister, $src$$Register);
10259     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10260   %}
10261   ins_pipe(pipe_slow); // XXX
10262 %}
10263 
10264 instruct convXI2D_reg(regD dst, rRegI src)
10265 %{
10266   predicate(UseXmmI2D);
10267   match(Set dst (ConvI2D src));
10268 
10269   format %{ "movdl $dst, $src\n\t"
10270             "cvtdq2pdl $dst, $dst\t# i2d" %}
10271   ins_encode %{
10272     __ movdl($dst$$XMMRegister, $src$$Register);
10273     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10274   %}
10275   ins_pipe(pipe_slow); // XXX
10276 %}
10277 
10278 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
10279 %{
10280   match(Set dst (ConvL2F src));
10281 
10282   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10283   ins_encode %{
10284     if (UseAVX > 0) {
10285       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10286     }
10287     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10288   %}
10289   ins_pipe(pipe_slow); // XXX
10290 %}
10291 
10292 instruct convL2F_reg_mem(regF dst, memory src)
10293 %{
10294   predicate(UseAVX == 0);
10295   match(Set dst (ConvL2F (LoadL src)));
10296 
10297   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10298   ins_encode %{
10299     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10300   %}
10301   ins_pipe(pipe_slow); // XXX
10302 %}
10303 
10304 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
10305 %{
10306   match(Set dst (ConvL2D src));
10307 
10308   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10309   ins_encode %{
10310     if (UseAVX > 0) {
10311       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
10312     }
10313     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10314   %}
10315   ins_pipe(pipe_slow); // XXX
10316 %}
10317 
10318 instruct convL2D_reg_mem(regD dst, memory src)
10319 %{
10320   predicate(UseAVX == 0);
10321   match(Set dst (ConvL2D (LoadL src)));
10322 
10323   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10324   ins_encode %{
10325     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10326   %}
10327   ins_pipe(pipe_slow); // XXX
10328 %}
10329 
10330 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10331 %{
10332   match(Set dst (ConvI2L src));
10333 
10334   ins_cost(125);
10335   format %{ "movslq  $dst, $src\t# i2l" %}
10336   ins_encode %{
10337     __ movslq($dst$$Register, $src$$Register);
10338   %}
10339   ins_pipe(ialu_reg_reg);
10340 %}
10341 
10342 // Zero-extend convert int to long
10343 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10344 %{
10345   match(Set dst (AndL (ConvI2L src) mask));
10346 
10347   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10348   ins_encode %{
10349     if ($dst$$reg != $src$$reg) {
10350       __ movl($dst$$Register, $src$$Register);
10351     }
10352   %}
10353   ins_pipe(ialu_reg_reg);
10354 %}
10355 
10356 // Zero-extend convert int to long
10357 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10358 %{
10359   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10360 
10361   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10362   ins_encode %{
10363     __ movl($dst$$Register, $src$$Address);
10364   %}
10365   ins_pipe(ialu_reg_mem);
10366 %}
10367 
10368 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10369 %{
10370   match(Set dst (AndL src mask));
10371 
10372   format %{ "movl    $dst, $src\t# zero-extend long" %}
10373   ins_encode %{
10374     __ movl($dst$$Register, $src$$Register);
10375   %}
10376   ins_pipe(ialu_reg_reg);
10377 %}
10378 
10379 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10380 %{
10381   match(Set dst (ConvL2I src));
10382 
10383   format %{ "movl    $dst, $src\t# l2i" %}
10384   ins_encode %{
10385     __ movl($dst$$Register, $src$$Register);
10386   %}
10387   ins_pipe(ialu_reg_reg);
10388 %}
10389 
10390 
10391 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10392   match(Set dst (MoveF2I src));
10393   effect(DEF dst, USE src);
10394 
10395   ins_cost(125);
10396   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10397   ins_encode %{
10398     __ movl($dst$$Register, Address(rsp, $src$$disp));
10399   %}
10400   ins_pipe(ialu_reg_mem);
10401 %}
10402 
10403 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10404   match(Set dst (MoveI2F src));
10405   effect(DEF dst, USE src);
10406 
10407   ins_cost(125);
10408   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10409   ins_encode %{
10410     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10411   %}
10412   ins_pipe(pipe_slow);
10413 %}
10414 
10415 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10416   match(Set dst (MoveD2L src));
10417   effect(DEF dst, USE src);
10418 
10419   ins_cost(125);
10420   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10421   ins_encode %{
10422     __ movq($dst$$Register, Address(rsp, $src$$disp));
10423   %}
10424   ins_pipe(ialu_reg_mem);
10425 %}
10426 
10427 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10428   predicate(!UseXmmLoadAndClearUpper);
10429   match(Set dst (MoveL2D src));
10430   effect(DEF dst, USE src);
10431 
10432   ins_cost(125);
10433   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10434   ins_encode %{
10435     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10436   %}
10437   ins_pipe(pipe_slow);
10438 %}
10439 
10440 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10441   predicate(UseXmmLoadAndClearUpper);
10442   match(Set dst (MoveL2D src));
10443   effect(DEF dst, USE src);
10444 
10445   ins_cost(125);
10446   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10447   ins_encode %{
10448     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10449   %}
10450   ins_pipe(pipe_slow);
10451 %}
10452 
10453 
10454 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10455   match(Set dst (MoveF2I src));
10456   effect(DEF dst, USE src);
10457 
10458   ins_cost(95); // XXX
10459   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10460   ins_encode %{
10461     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10462   %}
10463   ins_pipe(pipe_slow);
10464 %}
10465 
10466 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10467   match(Set dst (MoveI2F src));
10468   effect(DEF dst, USE src);
10469 
10470   ins_cost(100);
10471   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10472   ins_encode %{
10473     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10474   %}
10475   ins_pipe( ialu_mem_reg );
10476 %}
10477 
10478 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10479   match(Set dst (MoveD2L src));
10480   effect(DEF dst, USE src);
10481 
10482   ins_cost(95); // XXX
10483   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10484   ins_encode %{
10485     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10486   %}
10487   ins_pipe(pipe_slow);
10488 %}
10489 
10490 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10491   match(Set dst (MoveL2D src));
10492   effect(DEF dst, USE src);
10493 
10494   ins_cost(100);
10495   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10496   ins_encode %{
10497     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10498   %}
10499   ins_pipe(ialu_mem_reg);
10500 %}
10501 
10502 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10503   match(Set dst (MoveF2I src));
10504   effect(DEF dst, USE src);
10505   ins_cost(85);
10506   format %{ "movd    $dst,$src\t# MoveF2I" %}
10507   ins_encode %{
10508     __ movdl($dst$$Register, $src$$XMMRegister);
10509   %}
10510   ins_pipe( pipe_slow );
10511 %}
10512 
10513 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10514   match(Set dst (MoveD2L src));
10515   effect(DEF dst, USE src);
10516   ins_cost(85);
10517   format %{ "movd    $dst,$src\t# MoveD2L" %}
10518   ins_encode %{
10519     __ movdq($dst$$Register, $src$$XMMRegister);
10520   %}
10521   ins_pipe( pipe_slow );
10522 %}
10523 
10524 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10525   match(Set dst (MoveI2F src));
10526   effect(DEF dst, USE src);
10527   ins_cost(100);
10528   format %{ "movd    $dst,$src\t# MoveI2F" %}
10529   ins_encode %{
10530     __ movdl($dst$$XMMRegister, $src$$Register);
10531   %}
10532   ins_pipe( pipe_slow );
10533 %}
10534 
10535 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10536   match(Set dst (MoveL2D src));
10537   effect(DEF dst, USE src);
10538   ins_cost(100);
10539   format %{ "movd    $dst,$src\t# MoveL2D" %}
10540   ins_encode %{
10541      __ movdq($dst$$XMMRegister, $src$$Register);
10542   %}
10543   ins_pipe( pipe_slow );
10544 %}
10545 
10546 
10547 // Fast clearing of an array
10548 // Small non-constant lenght ClearArray for non-AVX512 targets.
10549 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
10550                   Universe dummy, rFlagsReg cr)
10551 %{
10552   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
10553   match(Set dummy (ClearArray (Binary cnt base) val));
10554   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
10555 
10556   format %{ $$template
10557     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10558     $$emit$$"jg      LARGE\n\t"
10559     $$emit$$"dec     rcx\n\t"
10560     $$emit$$"js      DONE\t# Zero length\n\t"
10561     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10562     $$emit$$"dec     rcx\n\t"
10563     $$emit$$"jge     LOOP\n\t"
10564     $$emit$$"jmp     DONE\n\t"
10565     $$emit$$"# LARGE:\n\t"
10566     if (UseFastStosb) {
10567        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10568        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10569     } else if (UseXMMForObjInit) {
10570        $$emit$$"movdq   $tmp, $val\n\t"
10571        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
10572        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
10573        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10574        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10575        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10576        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
10577        $$emit$$"add     0x40,rax\n\t"
10578        $$emit$$"# L_zero_64_bytes:\n\t"
10579        $$emit$$"sub     0x8,rcx\n\t"
10580        $$emit$$"jge     L_loop\n\t"
10581        $$emit$$"add     0x4,rcx\n\t"
10582        $$emit$$"jl      L_tail\n\t"
10583        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10584        $$emit$$"add     0x20,rax\n\t"
10585        $$emit$$"sub     0x4,rcx\n\t"
10586        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10587        $$emit$$"add     0x4,rcx\n\t"
10588        $$emit$$"jle     L_end\n\t"
10589        $$emit$$"dec     rcx\n\t"
10590        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10591        $$emit$$"vmovq   xmm0,(rax)\n\t"
10592        $$emit$$"add     0x8,rax\n\t"
10593        $$emit$$"dec     rcx\n\t"
10594        $$emit$$"jge     L_sloop\n\t"
10595        $$emit$$"# L_end:\n\t"
10596     } else {
10597        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10598     }
10599     $$emit$$"# DONE"
10600   %}
10601   ins_encode %{
10602     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10603                  $tmp$$XMMRegister, false, false);
10604   %}
10605   ins_pipe(pipe_slow);
10606 %}
10607 
10608 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
10609                             Universe dummy, rFlagsReg cr)
10610 %{
10611   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
10612   match(Set dummy (ClearArray (Binary cnt base) val));
10613   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
10614 
10615   format %{ $$template
10616     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10617     $$emit$$"jg      LARGE\n\t"
10618     $$emit$$"dec     rcx\n\t"
10619     $$emit$$"js      DONE\t# Zero length\n\t"
10620     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10621     $$emit$$"dec     rcx\n\t"
10622     $$emit$$"jge     LOOP\n\t"
10623     $$emit$$"jmp     DONE\n\t"
10624     $$emit$$"# LARGE:\n\t"
10625     if (UseXMMForObjInit) {
10626        $$emit$$"movdq   $tmp, $val\n\t"
10627        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
10628        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
10629        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10630        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10631        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10632        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
10633        $$emit$$"add     0x40,rax\n\t"
10634        $$emit$$"# L_zero_64_bytes:\n\t"
10635        $$emit$$"sub     0x8,rcx\n\t"
10636        $$emit$$"jge     L_loop\n\t"
10637        $$emit$$"add     0x4,rcx\n\t"
10638        $$emit$$"jl      L_tail\n\t"
10639        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10640        $$emit$$"add     0x20,rax\n\t"
10641        $$emit$$"sub     0x4,rcx\n\t"
10642        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10643        $$emit$$"add     0x4,rcx\n\t"
10644        $$emit$$"jle     L_end\n\t"
10645        $$emit$$"dec     rcx\n\t"
10646        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10647        $$emit$$"vmovq   xmm0,(rax)\n\t"
10648        $$emit$$"add     0x8,rax\n\t"
10649        $$emit$$"dec     rcx\n\t"
10650        $$emit$$"jge     L_sloop\n\t"
10651        $$emit$$"# L_end:\n\t"
10652     } else {
10653        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10654     }
10655     $$emit$$"# DONE"
10656   %}
10657   ins_encode %{
10658     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10659                  $tmp$$XMMRegister, false, true);
10660   %}
10661   ins_pipe(pipe_slow);
10662 %}
10663 
10664 // Small non-constant length ClearArray for AVX512 targets.
10665 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
10666                        Universe dummy, rFlagsReg cr)
10667 %{
10668   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
10669   match(Set dummy (ClearArray (Binary cnt base) val));
10670   ins_cost(125);
10671   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
10672 
10673   format %{ $$template
10674     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10675     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10676     $$emit$$"jg      LARGE\n\t"
10677     $$emit$$"dec     rcx\n\t"
10678     $$emit$$"js      DONE\t# Zero length\n\t"
10679     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10680     $$emit$$"dec     rcx\n\t"
10681     $$emit$$"jge     LOOP\n\t"
10682     $$emit$$"jmp     DONE\n\t"
10683     $$emit$$"# LARGE:\n\t"
10684     if (UseFastStosb) {
10685        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10686        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10687     } else if (UseXMMForObjInit) {
10688        $$emit$$"mov     rdi,rax\n\t"
10689        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10690        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10691        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10692        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10693        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10694        $$emit$$"add     0x40,rax\n\t"
10695        $$emit$$"# L_zero_64_bytes:\n\t"
10696        $$emit$$"sub     0x8,rcx\n\t"
10697        $$emit$$"jge     L_loop\n\t"
10698        $$emit$$"add     0x4,rcx\n\t"
10699        $$emit$$"jl      L_tail\n\t"
10700        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10701        $$emit$$"add     0x20,rax\n\t"
10702        $$emit$$"sub     0x4,rcx\n\t"
10703        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10704        $$emit$$"add     0x4,rcx\n\t"
10705        $$emit$$"jle     L_end\n\t"
10706        $$emit$$"dec     rcx\n\t"
10707        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10708        $$emit$$"vmovq   xmm0,(rax)\n\t"
10709        $$emit$$"add     0x8,rax\n\t"
10710        $$emit$$"dec     rcx\n\t"
10711        $$emit$$"jge     L_sloop\n\t"
10712        $$emit$$"# L_end:\n\t"
10713     } else {
10714        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10715     }
10716     $$emit$$"# DONE"
10717   %}
10718   ins_encode %{
10719     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10720                  $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
10721   %}
10722   ins_pipe(pipe_slow);
10723 %}
10724 
10725 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
10726                                  Universe dummy, rFlagsReg cr)
10727 %{
10728   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
10729   match(Set dummy (ClearArray (Binary cnt base) val));
10730   ins_cost(125);
10731   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
10732 
10733   format %{ $$template
10734     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10735     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10736     $$emit$$"jg      LARGE\n\t"
10737     $$emit$$"dec     rcx\n\t"
10738     $$emit$$"js      DONE\t# Zero length\n\t"
10739     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10740     $$emit$$"dec     rcx\n\t"
10741     $$emit$$"jge     LOOP\n\t"
10742     $$emit$$"jmp     DONE\n\t"
10743     $$emit$$"# LARGE:\n\t"
10744     if (UseFastStosb) {
10745        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10746        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10747     } else if (UseXMMForObjInit) {
10748        $$emit$$"mov     rdi,rax\n\t"
10749        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10750        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10751        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10752        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10753        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10754        $$emit$$"add     0x40,rax\n\t"
10755        $$emit$$"# L_zero_64_bytes:\n\t"
10756        $$emit$$"sub     0x8,rcx\n\t"
10757        $$emit$$"jge     L_loop\n\t"
10758        $$emit$$"add     0x4,rcx\n\t"
10759        $$emit$$"jl      L_tail\n\t"
10760        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10761        $$emit$$"add     0x20,rax\n\t"
10762        $$emit$$"sub     0x4,rcx\n\t"
10763        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10764        $$emit$$"add     0x4,rcx\n\t"
10765        $$emit$$"jle     L_end\n\t"
10766        $$emit$$"dec     rcx\n\t"
10767        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10768        $$emit$$"vmovq   xmm0,(rax)\n\t"
10769        $$emit$$"add     0x8,rax\n\t"
10770        $$emit$$"dec     rcx\n\t"
10771        $$emit$$"jge     L_sloop\n\t"
10772        $$emit$$"# L_end:\n\t"
10773     } else {
10774        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10775     }
10776     $$emit$$"# DONE"
10777   %}
10778   ins_encode %{
10779     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10780                  $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
10781   %}
10782   ins_pipe(pipe_slow);
10783 %}
10784 
10785 // Large non-constant length ClearArray for non-AVX512 targets.
10786 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
10787                         Universe dummy, rFlagsReg cr)
10788 %{
10789   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
10790   match(Set dummy (ClearArray (Binary cnt base) val));
10791   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
10792 
10793   format %{ $$template
10794     if (UseFastStosb) {
10795        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10796        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10797     } else if (UseXMMForObjInit) {
10798        $$emit$$"movdq   $tmp, $val\n\t"
10799        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
10800        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
10801        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10802        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10803        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10804        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
10805        $$emit$$"add     0x40,rax\n\t"
10806        $$emit$$"# L_zero_64_bytes:\n\t"
10807        $$emit$$"sub     0x8,rcx\n\t"
10808        $$emit$$"jge     L_loop\n\t"
10809        $$emit$$"add     0x4,rcx\n\t"
10810        $$emit$$"jl      L_tail\n\t"
10811        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10812        $$emit$$"add     0x20,rax\n\t"
10813        $$emit$$"sub     0x4,rcx\n\t"
10814        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10815        $$emit$$"add     0x4,rcx\n\t"
10816        $$emit$$"jle     L_end\n\t"
10817        $$emit$$"dec     rcx\n\t"
10818        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10819        $$emit$$"vmovq   xmm0,(rax)\n\t"
10820        $$emit$$"add     0x8,rax\n\t"
10821        $$emit$$"dec     rcx\n\t"
10822        $$emit$$"jge     L_sloop\n\t"
10823        $$emit$$"# L_end:\n\t"
10824     } else {
10825        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10826     }
10827   %}
10828   ins_encode %{
10829     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10830                  $tmp$$XMMRegister, true, false);
10831   %}
10832   ins_pipe(pipe_slow);
10833 %}
10834 
10835 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
10836                                   Universe dummy, rFlagsReg cr)
10837 %{
10838   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
10839   match(Set dummy (ClearArray (Binary cnt base) val));
10840   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
10841 
10842   format %{ $$template
10843     if (UseXMMForObjInit) {
10844        $$emit$$"movdq   $tmp, $val\n\t"
10845        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
10846        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
10847        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10848        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10849        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10850        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
10851        $$emit$$"add     0x40,rax\n\t"
10852        $$emit$$"# L_zero_64_bytes:\n\t"
10853        $$emit$$"sub     0x8,rcx\n\t"
10854        $$emit$$"jge     L_loop\n\t"
10855        $$emit$$"add     0x4,rcx\n\t"
10856        $$emit$$"jl      L_tail\n\t"
10857        $$emit$$"vmovdqu $tmp,(rax)\n\t"
10858        $$emit$$"add     0x20,rax\n\t"
10859        $$emit$$"sub     0x4,rcx\n\t"
10860        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10861        $$emit$$"add     0x4,rcx\n\t"
10862        $$emit$$"jle     L_end\n\t"
10863        $$emit$$"dec     rcx\n\t"
10864        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10865        $$emit$$"vmovq   xmm0,(rax)\n\t"
10866        $$emit$$"add     0x8,rax\n\t"
10867        $$emit$$"dec     rcx\n\t"
10868        $$emit$$"jge     L_sloop\n\t"
10869        $$emit$$"# L_end:\n\t"
10870     } else {
10871        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10872     }
10873   %}
10874   ins_encode %{
10875     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10876                  $tmp$$XMMRegister, true, true);
10877   %}
10878   ins_pipe(pipe_slow);
10879 %}
10880 
10881 // Large non-constant length ClearArray for AVX512 targets.
10882 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
10883                              Universe dummy, rFlagsReg cr)
10884 %{
10885   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
10886   match(Set dummy (ClearArray (Binary cnt base) val));
10887   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
10888 
10889   format %{ $$template
10890     if (UseFastStosb) {
10891        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10892        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10893        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10894     } else if (UseXMMForObjInit) {
10895        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10896        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10897        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10898        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10899        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10900        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10901        $$emit$$"add     0x40,rax\n\t"
10902        $$emit$$"# L_zero_64_bytes:\n\t"
10903        $$emit$$"sub     0x8,rcx\n\t"
10904        $$emit$$"jge     L_loop\n\t"
10905        $$emit$$"add     0x4,rcx\n\t"
10906        $$emit$$"jl      L_tail\n\t"
10907        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10908        $$emit$$"add     0x20,rax\n\t"
10909        $$emit$$"sub     0x4,rcx\n\t"
10910        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10911        $$emit$$"add     0x4,rcx\n\t"
10912        $$emit$$"jle     L_end\n\t"
10913        $$emit$$"dec     rcx\n\t"
10914        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10915        $$emit$$"vmovq   xmm0,(rax)\n\t"
10916        $$emit$$"add     0x8,rax\n\t"
10917        $$emit$$"dec     rcx\n\t"
10918        $$emit$$"jge     L_sloop\n\t"
10919        $$emit$$"# L_end:\n\t"
10920     } else {
10921        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10922        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10923     }
10924   %}
10925   ins_encode %{
10926     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10927                  $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
10928   %}
10929   ins_pipe(pipe_slow);
10930 %}
10931 
10932 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
10933                                        Universe dummy, rFlagsReg cr)
10934 %{
10935   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
10936   match(Set dummy (ClearArray (Binary cnt base) val));
10937   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
10938 
10939   format %{ $$template
10940     if (UseFastStosb) {
10941        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10942        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10943        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10944     } else if (UseXMMForObjInit) {
10945        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10946        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10947        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10948        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10949        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10950        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10951        $$emit$$"add     0x40,rax\n\t"
10952        $$emit$$"# L_zero_64_bytes:\n\t"
10953        $$emit$$"sub     0x8,rcx\n\t"
10954        $$emit$$"jge     L_loop\n\t"
10955        $$emit$$"add     0x4,rcx\n\t"
10956        $$emit$$"jl      L_tail\n\t"
10957        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10958        $$emit$$"add     0x20,rax\n\t"
10959        $$emit$$"sub     0x4,rcx\n\t"
10960        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10961        $$emit$$"add     0x4,rcx\n\t"
10962        $$emit$$"jle     L_end\n\t"
10963        $$emit$$"dec     rcx\n\t"
10964        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10965        $$emit$$"vmovq   xmm0,(rax)\n\t"
10966        $$emit$$"add     0x8,rax\n\t"
10967        $$emit$$"dec     rcx\n\t"
10968        $$emit$$"jge     L_sloop\n\t"
10969        $$emit$$"# L_end:\n\t"
10970     } else {
10971        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10972        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10973     }
10974   %}
10975   ins_encode %{
10976     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
10977                  $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
10978   %}
10979   ins_pipe(pipe_slow);
10980 %}
10981 
10982 // Small constant length ClearArray for AVX512 targets.
10983 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
10984 %{
10985   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
10986             ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
10987   match(Set dummy (ClearArray (Binary cnt base) val));
10988   ins_cost(100);
10989   effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
10990   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
10991   ins_encode %{
10992     __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
10993   %}
10994   ins_pipe(pipe_slow);
10995 %}
10996 
10997 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10998                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
10999 %{
11000   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11001   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11002   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11003 
11004   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11005   ins_encode %{
11006     __ string_compare($str1$$Register, $str2$$Register,
11007                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11008                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
11009   %}
11010   ins_pipe( pipe_slow );
11011 %}
11012 
11013 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11014                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11015 %{
11016   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11017   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11018   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11019 
11020   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11021   ins_encode %{
11022     __ string_compare($str1$$Register, $str2$$Register,
11023                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11024                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
11025   %}
11026   ins_pipe( pipe_slow );
11027 %}
11028 
11029 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11030                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
11031 %{
11032   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11033   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11034   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11035 
11036   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11037   ins_encode %{
11038     __ string_compare($str1$$Register, $str2$$Register,
11039                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11040                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
11041   %}
11042   ins_pipe( pipe_slow );
11043 %}
11044 
11045 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11046                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11047 %{
11048   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11049   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11050   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11051 
11052   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11053   ins_encode %{
11054     __ string_compare($str1$$Register, $str2$$Register,
11055                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11056                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
11057   %}
11058   ins_pipe( pipe_slow );
11059 %}
11060 
11061 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11062                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11063 %{
11064   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11065   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11066   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11067 
11068   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11069   ins_encode %{
11070     __ string_compare($str1$$Register, $str2$$Register,
11071                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11072                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
11073   %}
11074   ins_pipe( pipe_slow );
11075 %}
11076 
11077 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11078                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11079 %{
11080   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11081   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11082   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11083 
11084   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11085   ins_encode %{
11086     __ string_compare($str1$$Register, $str2$$Register,
11087                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11088                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
11089   %}
11090   ins_pipe( pipe_slow );
11091 %}
11092 
11093 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11094                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
11095 %{
11096   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11097   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11098   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11099 
11100   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11101   ins_encode %{
11102     __ string_compare($str2$$Register, $str1$$Register,
11103                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11104                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
11105   %}
11106   ins_pipe( pipe_slow );
11107 %}
11108 
11109 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11110                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
11111 %{
11112   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11113   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11114   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11115 
11116   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11117   ins_encode %{
11118     __ string_compare($str2$$Register, $str1$$Register,
11119                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11120                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
11121   %}
11122   ins_pipe( pipe_slow );
11123 %}
11124 
11125 // fast search of substring with known size.
11126 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11127                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11128 %{
11129   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11130   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11131   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11132 
11133   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11134   ins_encode %{
11135     int icnt2 = (int)$int_cnt2$$constant;
11136     if (icnt2 >= 16) {
11137       // IndexOf for constant substrings with size >= 16 elements
11138       // which don't need to be loaded through stack.
11139       __ string_indexofC8($str1$$Register, $str2$$Register,
11140                           $cnt1$$Register, $cnt2$$Register,
11141                           icnt2, $result$$Register,
11142                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11143     } else {
11144       // Small strings are loaded through stack if they cross page boundary.
11145       __ string_indexof($str1$$Register, $str2$$Register,
11146                         $cnt1$$Register, $cnt2$$Register,
11147                         icnt2, $result$$Register,
11148                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11149     }
11150   %}
11151   ins_pipe( pipe_slow );
11152 %}
11153 
11154 // fast search of substring with known size.
11155 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11156                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11157 %{
11158   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11159   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11160   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11161 
11162   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11163   ins_encode %{
11164     int icnt2 = (int)$int_cnt2$$constant;
11165     if (icnt2 >= 8) {
11166       // IndexOf for constant substrings with size >= 8 elements
11167       // which don't need to be loaded through stack.
11168       __ string_indexofC8($str1$$Register, $str2$$Register,
11169                           $cnt1$$Register, $cnt2$$Register,
11170                           icnt2, $result$$Register,
11171                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11172     } else {
11173       // Small strings are loaded through stack if they cross page boundary.
11174       __ string_indexof($str1$$Register, $str2$$Register,
11175                         $cnt1$$Register, $cnt2$$Register,
11176                         icnt2, $result$$Register,
11177                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11178     }
11179   %}
11180   ins_pipe( pipe_slow );
11181 %}
11182 
11183 // fast search of substring with known size.
11184 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11185                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11186 %{
11187   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11188   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11189   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11190 
11191   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
11192   ins_encode %{
11193     int icnt2 = (int)$int_cnt2$$constant;
11194     if (icnt2 >= 8) {
11195       // IndexOf for constant substrings with size >= 8 elements
11196       // which don't need to be loaded through stack.
11197       __ string_indexofC8($str1$$Register, $str2$$Register,
11198                           $cnt1$$Register, $cnt2$$Register,
11199                           icnt2, $result$$Register,
11200                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11201     } else {
11202       // Small strings are loaded through stack if they cross page boundary.
11203       __ string_indexof($str1$$Register, $str2$$Register,
11204                         $cnt1$$Register, $cnt2$$Register,
11205                         icnt2, $result$$Register,
11206                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11207     }
11208   %}
11209   ins_pipe( pipe_slow );
11210 %}
11211 
11212 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11213                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11214 %{
11215   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11216   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11217   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11218 
11219   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11220   ins_encode %{
11221     __ string_indexof($str1$$Register, $str2$$Register,
11222                       $cnt1$$Register, $cnt2$$Register,
11223                       (-1), $result$$Register,
11224                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11225   %}
11226   ins_pipe( pipe_slow );
11227 %}
11228 
11229 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11230                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11231 %{
11232   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11233   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11234   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11235 
11236   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11237   ins_encode %{
11238     __ string_indexof($str1$$Register, $str2$$Register,
11239                       $cnt1$$Register, $cnt2$$Register,
11240                       (-1), $result$$Register,
11241                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11242   %}
11243   ins_pipe( pipe_slow );
11244 %}
11245 
11246 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11247                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
11248 %{
11249   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11250   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11251   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11252 
11253   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11254   ins_encode %{
11255     __ string_indexof($str1$$Register, $str2$$Register,
11256                       $cnt1$$Register, $cnt2$$Register,
11257                       (-1), $result$$Register,
11258                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11259   %}
11260   ins_pipe( pipe_slow );
11261 %}
11262 
11263 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11264                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11265 %{
11266   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
11267   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11268   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11269   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11270   ins_encode %{
11271     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11272                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11273   %}
11274   ins_pipe( pipe_slow );
11275 %}
11276 
11277 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11278                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
11279 %{
11280   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
11281   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11282   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11283   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11284   ins_encode %{
11285     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11286                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
11287   %}
11288   ins_pipe( pipe_slow );
11289 %}
11290 
11291 // fast string equals
11292 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11293                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11294 %{
11295   predicate(!VM_Version::supports_avx512vlbw());
11296   match(Set result (StrEquals (Binary str1 str2) cnt));
11297   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11298 
11299   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11300   ins_encode %{
11301     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11302                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11303                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11304   %}
11305   ins_pipe( pipe_slow );
11306 %}
11307 
11308 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11309                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
11310 %{
11311   predicate(VM_Version::supports_avx512vlbw());
11312   match(Set result (StrEquals (Binary str1 str2) cnt));
11313   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11314 
11315   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11316   ins_encode %{
11317     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11318                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11319                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11320   %}
11321   ins_pipe( pipe_slow );
11322 %}
11323 
11324 // fast array equals
11325 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11326                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11327 %{
11328   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11329   match(Set result (AryEq ary1 ary2));
11330   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11331 
11332   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11333   ins_encode %{
11334     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11335                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11336                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
11337   %}
11338   ins_pipe( pipe_slow );
11339 %}
11340 
11341 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11342                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11343 %{
11344   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11345   match(Set result (AryEq ary1 ary2));
11346   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11347 
11348   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11349   ins_encode %{
11350     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11351                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11352                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
11353   %}
11354   ins_pipe( pipe_slow );
11355 %}
11356 
11357 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11358                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11359 %{
11360   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11361   match(Set result (AryEq ary1 ary2));
11362   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11363 
11364   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11365   ins_encode %{
11366     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11367                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11368                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
11369   %}
11370   ins_pipe( pipe_slow );
11371 %}
11372 
11373 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11374                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11375 %{
11376   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11377   match(Set result (AryEq ary1 ary2));
11378   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11379 
11380   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11381   ins_encode %{
11382     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11383                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11384                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
11385   %}
11386   ins_pipe( pipe_slow );
11387 %}
11388 
11389 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
11390                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
11391                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
11392                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
11393                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
11394 %{
11395   predicate(UseAVX >= 2);
11396   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
11397   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
11398          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
11399          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
11400          USE basic_type, KILL cr);
11401 
11402   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
11403   ins_encode %{
11404     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
11405                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
11406                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
11407                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
11408                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
11409                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
11410                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
11411   %}
11412   ins_pipe( pipe_slow );
11413 %}
11414 
11415 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11416                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
11417 %{
11418   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11419   match(Set result (CountPositives ary1 len));
11420   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11421 
11422   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11423   ins_encode %{
11424     __ count_positives($ary1$$Register, $len$$Register,
11425                        $result$$Register, $tmp3$$Register,
11426                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
11427   %}
11428   ins_pipe( pipe_slow );
11429 %}
11430 
11431 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11432                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
11433 %{
11434   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11435   match(Set result (CountPositives ary1 len));
11436   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11437 
11438   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11439   ins_encode %{
11440     __ count_positives($ary1$$Register, $len$$Register,
11441                        $result$$Register, $tmp3$$Register,
11442                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
11443   %}
11444   ins_pipe( pipe_slow );
11445 %}
11446 
11447 // fast char[] to byte[] compression
11448 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11449                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11450   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11451   match(Set result (StrCompressedCopy src (Binary dst len)));
11452   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
11453          USE_KILL len, KILL tmp5, KILL cr);
11454 
11455   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11456   ins_encode %{
11457     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11458                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11459                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11460                            knoreg, knoreg);
11461   %}
11462   ins_pipe( pipe_slow );
11463 %}
11464 
11465 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
11466                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11467   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11468   match(Set result (StrCompressedCopy src (Binary dst len)));
11469   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
11470          USE_KILL len, KILL tmp5, KILL cr);
11471 
11472   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11473   ins_encode %{
11474     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11475                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11476                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
11477                            $ktmp1$$KRegister, $ktmp2$$KRegister);
11478   %}
11479   ins_pipe( pipe_slow );
11480 %}
11481 // fast byte[] to char[] inflation
11482 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11483                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11484   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
11485   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11486   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11487 
11488   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11489   ins_encode %{
11490     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11491                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
11492   %}
11493   ins_pipe( pipe_slow );
11494 %}
11495 
11496 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11497                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
11498   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
11499   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11500   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11501 
11502   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11503   ins_encode %{
11504     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11505                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
11506   %}
11507   ins_pipe( pipe_slow );
11508 %}
11509 
11510 // encode char[] to byte[] in ISO_8859_1
11511 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11512                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11513                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11514   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
11515   match(Set result (EncodeISOArray src (Binary dst len)));
11516   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11517 
11518   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11519   ins_encode %{
11520     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11521                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11522                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
11523   %}
11524   ins_pipe( pipe_slow );
11525 %}
11526 
11527 // encode char[] to byte[] in ASCII
11528 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11529                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
11530                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11531   predicate(((EncodeISOArrayNode*)n)->is_ascii());
11532   match(Set result (EncodeISOArray src (Binary dst len)));
11533   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11534 
11535   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11536   ins_encode %{
11537     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11538                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11539                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
11540   %}
11541   ins_pipe( pipe_slow );
11542 %}
11543 
11544 //----------Overflow Math Instructions-----------------------------------------
11545 
11546 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11547 %{
11548   match(Set cr (OverflowAddI op1 op2));
11549   effect(DEF cr, USE_KILL op1, USE op2);
11550 
11551   format %{ "addl    $op1, $op2\t# overflow check int" %}
11552 
11553   ins_encode %{
11554     __ addl($op1$$Register, $op2$$Register);
11555   %}
11556   ins_pipe(ialu_reg_reg);
11557 %}
11558 
11559 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11560 %{
11561   match(Set cr (OverflowAddI op1 op2));
11562   effect(DEF cr, USE_KILL op1, USE op2);
11563 
11564   format %{ "addl    $op1, $op2\t# overflow check int" %}
11565 
11566   ins_encode %{
11567     __ addl($op1$$Register, $op2$$constant);
11568   %}
11569   ins_pipe(ialu_reg_reg);
11570 %}
11571 
11572 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11573 %{
11574   match(Set cr (OverflowAddL op1 op2));
11575   effect(DEF cr, USE_KILL op1, USE op2);
11576 
11577   format %{ "addq    $op1, $op2\t# overflow check long" %}
11578   ins_encode %{
11579     __ addq($op1$$Register, $op2$$Register);
11580   %}
11581   ins_pipe(ialu_reg_reg);
11582 %}
11583 
11584 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11585 %{
11586   match(Set cr (OverflowAddL op1 op2));
11587   effect(DEF cr, USE_KILL op1, USE op2);
11588 
11589   format %{ "addq    $op1, $op2\t# overflow check long" %}
11590   ins_encode %{
11591     __ addq($op1$$Register, $op2$$constant);
11592   %}
11593   ins_pipe(ialu_reg_reg);
11594 %}
11595 
11596 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11597 %{
11598   match(Set cr (OverflowSubI op1 op2));
11599 
11600   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11601   ins_encode %{
11602     __ cmpl($op1$$Register, $op2$$Register);
11603   %}
11604   ins_pipe(ialu_reg_reg);
11605 %}
11606 
11607 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11608 %{
11609   match(Set cr (OverflowSubI op1 op2));
11610 
11611   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11612   ins_encode %{
11613     __ cmpl($op1$$Register, $op2$$constant);
11614   %}
11615   ins_pipe(ialu_reg_reg);
11616 %}
11617 
11618 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11619 %{
11620   match(Set cr (OverflowSubL op1 op2));
11621 
11622   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11623   ins_encode %{
11624     __ cmpq($op1$$Register, $op2$$Register);
11625   %}
11626   ins_pipe(ialu_reg_reg);
11627 %}
11628 
11629 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11630 %{
11631   match(Set cr (OverflowSubL op1 op2));
11632 
11633   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11634   ins_encode %{
11635     __ cmpq($op1$$Register, $op2$$constant);
11636   %}
11637   ins_pipe(ialu_reg_reg);
11638 %}
11639 
11640 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
11641 %{
11642   match(Set cr (OverflowSubI zero op2));
11643   effect(DEF cr, USE_KILL op2);
11644 
11645   format %{ "negl    $op2\t# overflow check int" %}
11646   ins_encode %{
11647     __ negl($op2$$Register);
11648   %}
11649   ins_pipe(ialu_reg_reg);
11650 %}
11651 
11652 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11653 %{
11654   match(Set cr (OverflowSubL zero op2));
11655   effect(DEF cr, USE_KILL op2);
11656 
11657   format %{ "negq    $op2\t# overflow check long" %}
11658   ins_encode %{
11659     __ negq($op2$$Register);
11660   %}
11661   ins_pipe(ialu_reg_reg);
11662 %}
11663 
11664 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11665 %{
11666   match(Set cr (OverflowMulI op1 op2));
11667   effect(DEF cr, USE_KILL op1, USE op2);
11668 
11669   format %{ "imull    $op1, $op2\t# overflow check int" %}
11670   ins_encode %{
11671     __ imull($op1$$Register, $op2$$Register);
11672   %}
11673   ins_pipe(ialu_reg_reg_alu0);
11674 %}
11675 
11676 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11677 %{
11678   match(Set cr (OverflowMulI op1 op2));
11679   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11680 
11681   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11682   ins_encode %{
11683     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11684   %}
11685   ins_pipe(ialu_reg_reg_alu0);
11686 %}
11687 
11688 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11689 %{
11690   match(Set cr (OverflowMulL op1 op2));
11691   effect(DEF cr, USE_KILL op1, USE op2);
11692 
11693   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11694   ins_encode %{
11695     __ imulq($op1$$Register, $op2$$Register);
11696   %}
11697   ins_pipe(ialu_reg_reg_alu0);
11698 %}
11699 
11700 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11701 %{
11702   match(Set cr (OverflowMulL op1 op2));
11703   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11704 
11705   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11706   ins_encode %{
11707     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11708   %}
11709   ins_pipe(ialu_reg_reg_alu0);
11710 %}
11711 
11712 
11713 //----------Control Flow Instructions------------------------------------------
11714 // Signed compare Instructions
11715 
11716 // XXX more variants!!
11717 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11718 %{
11719   match(Set cr (CmpI op1 op2));
11720   effect(DEF cr, USE op1, USE op2);
11721 
11722   format %{ "cmpl    $op1, $op2" %}
11723   ins_encode %{
11724     __ cmpl($op1$$Register, $op2$$Register);
11725   %}
11726   ins_pipe(ialu_cr_reg_reg);
11727 %}
11728 
11729 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11730 %{
11731   match(Set cr (CmpI op1 op2));
11732 
11733   format %{ "cmpl    $op1, $op2" %}
11734   ins_encode %{
11735     __ cmpl($op1$$Register, $op2$$constant);
11736   %}
11737   ins_pipe(ialu_cr_reg_imm);
11738 %}
11739 
11740 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11741 %{
11742   match(Set cr (CmpI op1 (LoadI op2)));
11743 
11744   ins_cost(500); // XXX
11745   format %{ "cmpl    $op1, $op2" %}
11746   ins_encode %{
11747     __ cmpl($op1$$Register, $op2$$Address);
11748   %}
11749   ins_pipe(ialu_cr_reg_mem);
11750 %}
11751 
11752 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
11753 %{
11754   match(Set cr (CmpI src zero));
11755 
11756   format %{ "testl   $src, $src" %}
11757   ins_encode %{
11758     __ testl($src$$Register, $src$$Register);
11759   %}
11760   ins_pipe(ialu_cr_reg_imm);
11761 %}
11762 
11763 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
11764 %{
11765   match(Set cr (CmpI (AndI src con) zero));
11766 
11767   format %{ "testl   $src, $con" %}
11768   ins_encode %{
11769     __ testl($src$$Register, $con$$constant);
11770   %}
11771   ins_pipe(ialu_cr_reg_imm);
11772 %}
11773 
11774 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
11775 %{
11776   match(Set cr (CmpI (AndI src1 src2) zero));
11777 
11778   format %{ "testl   $src1, $src2" %}
11779   ins_encode %{
11780     __ testl($src1$$Register, $src2$$Register);
11781   %}
11782   ins_pipe(ialu_cr_reg_imm);
11783 %}
11784 
11785 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
11786 %{
11787   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11788 
11789   format %{ "testl   $src, $mem" %}
11790   ins_encode %{
11791     __ testl($src$$Register, $mem$$Address);
11792   %}
11793   ins_pipe(ialu_cr_reg_mem);
11794 %}
11795 
11796 // Unsigned compare Instructions; really, same as signed except they
11797 // produce an rFlagsRegU instead of rFlagsReg.
11798 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11799 %{
11800   match(Set cr (CmpU op1 op2));
11801 
11802   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11803   ins_encode %{
11804     __ cmpl($op1$$Register, $op2$$Register);
11805   %}
11806   ins_pipe(ialu_cr_reg_reg);
11807 %}
11808 
11809 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11810 %{
11811   match(Set cr (CmpU op1 op2));
11812 
11813   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11814   ins_encode %{
11815     __ cmpl($op1$$Register, $op2$$constant);
11816   %}
11817   ins_pipe(ialu_cr_reg_imm);
11818 %}
11819 
11820 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11821 %{
11822   match(Set cr (CmpU op1 (LoadI op2)));
11823 
11824   ins_cost(500); // XXX
11825   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11826   ins_encode %{
11827     __ cmpl($op1$$Register, $op2$$Address);
11828   %}
11829   ins_pipe(ialu_cr_reg_mem);
11830 %}
11831 
11832 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
11833 %{
11834   match(Set cr (CmpU src zero));
11835 
11836   format %{ "testl   $src, $src\t# unsigned" %}
11837   ins_encode %{
11838     __ testl($src$$Register, $src$$Register);
11839   %}
11840   ins_pipe(ialu_cr_reg_imm);
11841 %}
11842 
11843 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11844 %{
11845   match(Set cr (CmpP op1 op2));
11846 
11847   format %{ "cmpq    $op1, $op2\t# ptr" %}
11848   ins_encode %{
11849     __ cmpq($op1$$Register, $op2$$Register);
11850   %}
11851   ins_pipe(ialu_cr_reg_reg);
11852 %}
11853 
11854 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11855 %{
11856   match(Set cr (CmpP op1 (LoadP op2)));
11857   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11858 
11859   ins_cost(500); // XXX
11860   format %{ "cmpq    $op1, $op2\t# ptr" %}
11861   ins_encode %{
11862     __ cmpq($op1$$Register, $op2$$Address);
11863   %}
11864   ins_pipe(ialu_cr_reg_mem);
11865 %}
11866 
11867 // XXX this is generalized by compP_rReg_mem???
11868 // Compare raw pointer (used in out-of-heap check).
11869 // Only works because non-oop pointers must be raw pointers
11870 // and raw pointers have no anti-dependencies.
11871 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11872 %{
11873   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
11874             n->in(2)->as_Load()->barrier_data() == 0);
11875   match(Set cr (CmpP op1 (LoadP op2)));
11876 
11877   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11878   ins_encode %{
11879     __ cmpq($op1$$Register, $op2$$Address);
11880   %}
11881   ins_pipe(ialu_cr_reg_mem);
11882 %}
11883 
11884 // This will generate a signed flags result. This should be OK since
11885 // any compare to a zero should be eq/neq.
11886 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11887 %{
11888   match(Set cr (CmpP src zero));
11889 
11890   format %{ "testq   $src, $src\t# ptr" %}
11891   ins_encode %{
11892     __ testq($src$$Register, $src$$Register);
11893   %}
11894   ins_pipe(ialu_cr_reg_imm);
11895 %}
11896 
11897 // This will generate a signed flags result. This should be OK since
11898 // any compare to a zero should be eq/neq.
11899 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11900 %{
11901   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
11902             n->in(1)->as_Load()->barrier_data() == 0);
11903   match(Set cr (CmpP (LoadP op) zero));
11904 
11905   ins_cost(500); // XXX
11906   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11907   ins_encode %{
11908     __ testq($op$$Address, 0xFFFFFFFF);
11909   %}
11910   ins_pipe(ialu_cr_reg_imm);
11911 %}
11912 
11913 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11914 %{
11915   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
11916             n->in(1)->as_Load()->barrier_data() == 0);
11917   match(Set cr (CmpP (LoadP mem) zero));
11918 
11919   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11920   ins_encode %{
11921     __ cmpq(r12, $mem$$Address);
11922   %}
11923   ins_pipe(ialu_cr_reg_mem);
11924 %}
11925 
11926 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11927 %{
11928   match(Set cr (CmpN op1 op2));
11929 
11930   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11931   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11932   ins_pipe(ialu_cr_reg_reg);
11933 %}
11934 
11935 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11936 %{
11937   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11938   match(Set cr (CmpN src (LoadN mem)));
11939 
11940   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11941   ins_encode %{
11942     __ cmpl($src$$Register, $mem$$Address);
11943   %}
11944   ins_pipe(ialu_cr_reg_mem);
11945 %}
11946 
11947 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11948   match(Set cr (CmpN op1 op2));
11949 
11950   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11951   ins_encode %{
11952     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11953   %}
11954   ins_pipe(ialu_cr_reg_imm);
11955 %}
11956 
11957 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11958 %{
11959   predicate(n->in(2)->as_Load()->barrier_data() == 0);
11960   match(Set cr (CmpN src (LoadN mem)));
11961 
11962   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11963   ins_encode %{
11964     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11965   %}
11966   ins_pipe(ialu_cr_reg_mem);
11967 %}
11968 
11969 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11970   match(Set cr (CmpN op1 op2));
11971 
11972   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11973   ins_encode %{
11974     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11975   %}
11976   ins_pipe(ialu_cr_reg_imm);
11977 %}
11978 
11979 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11980 %{
11981   predicate(!UseCompactObjectHeaders);
11982   match(Set cr (CmpN src (LoadNKlass mem)));
11983 
11984   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11985   ins_encode %{
11986     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11987   %}
11988   ins_pipe(ialu_cr_reg_mem);
11989 %}
11990 
11991 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11992   match(Set cr (CmpN src zero));
11993 
11994   format %{ "testl   $src, $src\t# compressed ptr" %}
11995   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11996   ins_pipe(ialu_cr_reg_imm);
11997 %}
11998 
11999 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12000 %{
12001   predicate(CompressedOops::base() != nullptr &&
12002             n->in(1)->as_Load()->barrier_data() == 0);
12003   match(Set cr (CmpN (LoadN mem) zero));
12004 
12005   ins_cost(500); // XXX
12006   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12007   ins_encode %{
12008     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12009   %}
12010   ins_pipe(ialu_cr_reg_mem);
12011 %}
12012 
12013 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12014 %{
12015   predicate(CompressedOops::base() == nullptr &&
12016             n->in(1)->as_Load()->barrier_data() == 0);
12017   match(Set cr (CmpN (LoadN mem) zero));
12018 
12019   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12020   ins_encode %{
12021     __ cmpl(r12, $mem$$Address);
12022   %}
12023   ins_pipe(ialu_cr_reg_mem);
12024 %}
12025 
12026 // Yanked all unsigned pointer compare operations.
12027 // Pointer compares are done with CmpP which is already unsigned.
12028 
12029 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12030 %{
12031   match(Set cr (CmpL op1 op2));
12032 
12033   format %{ "cmpq    $op1, $op2" %}
12034   ins_encode %{
12035     __ cmpq($op1$$Register, $op2$$Register);
12036   %}
12037   ins_pipe(ialu_cr_reg_reg);
12038 %}
12039 
12040 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12041 %{
12042   match(Set cr (CmpL op1 op2));
12043 
12044   format %{ "cmpq    $op1, $op2" %}
12045   ins_encode %{
12046     __ cmpq($op1$$Register, $op2$$constant);
12047   %}
12048   ins_pipe(ialu_cr_reg_imm);
12049 %}
12050 
12051 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12052 %{
12053   match(Set cr (CmpL op1 (LoadL op2)));
12054 
12055   format %{ "cmpq    $op1, $op2" %}
12056   ins_encode %{
12057     __ cmpq($op1$$Register, $op2$$Address);
12058   %}
12059   ins_pipe(ialu_cr_reg_mem);
12060 %}
12061 
12062 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12063 %{
12064   match(Set cr (CmpL src zero));
12065 
12066   format %{ "testq   $src, $src" %}
12067   ins_encode %{
12068     __ testq($src$$Register, $src$$Register);
12069   %}
12070   ins_pipe(ialu_cr_reg_imm);
12071 %}
12072 
12073 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12074 %{
12075   match(Set cr (CmpL (AndL src con) zero));
12076 
12077   format %{ "testq   $src, $con\t# long" %}
12078   ins_encode %{
12079     __ testq($src$$Register, $con$$constant);
12080   %}
12081   ins_pipe(ialu_cr_reg_imm);
12082 %}
12083 
12084 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
12085 %{
12086   match(Set cr (CmpL (AndL src1 src2) zero));
12087 
12088   format %{ "testq   $src1, $src2\t# long" %}
12089   ins_encode %{
12090     __ testq($src1$$Register, $src2$$Register);
12091   %}
12092   ins_pipe(ialu_cr_reg_imm);
12093 %}
12094 
12095 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12096 %{
12097   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12098 
12099   format %{ "testq   $src, $mem" %}
12100   ins_encode %{
12101     __ testq($src$$Register, $mem$$Address);
12102   %}
12103   ins_pipe(ialu_cr_reg_mem);
12104 %}
12105 
12106 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12107 %{
12108   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12109 
12110   format %{ "testq   $src, $mem" %}
12111   ins_encode %{
12112     __ testq($src$$Register, $mem$$Address);
12113   %}
12114   ins_pipe(ialu_cr_reg_mem);
12115 %}
12116 
12117 // Manifest a CmpU result in an integer register.  Very painful.
12118 // This is the test to avoid.
12119 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
12120 %{
12121   match(Set dst (CmpU3 src1 src2));
12122   effect(KILL flags);
12123 
12124   ins_cost(275); // XXX
12125   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
12126             "movl    $dst, -1\n\t"
12127             "jb,u    done\n\t"
12128             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
12129     "done:" %}
12130   ins_encode %{
12131     Label done;
12132     __ cmpl($src1$$Register, $src2$$Register);
12133     __ movl($dst$$Register, -1);
12134     __ jccb(Assembler::below, done);
12135     __ setcc(Assembler::notZero, $dst$$Register);
12136     __ bind(done);
12137   %}
12138   ins_pipe(pipe_slow);
12139 %}
12140 
12141 // Manifest a CmpL result in an integer register.  Very painful.
12142 // This is the test to avoid.
12143 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12144 %{
12145   match(Set dst (CmpL3 src1 src2));
12146   effect(KILL flags);
12147 
12148   ins_cost(275); // XXX
12149   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12150             "movl    $dst, -1\n\t"
12151             "jl,s    done\n\t"
12152             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
12153     "done:" %}
12154   ins_encode %{
12155     Label done;
12156     __ cmpq($src1$$Register, $src2$$Register);
12157     __ movl($dst$$Register, -1);
12158     __ jccb(Assembler::less, done);
12159     __ setcc(Assembler::notZero, $dst$$Register);
12160     __ bind(done);
12161   %}
12162   ins_pipe(pipe_slow);
12163 %}
12164 
12165 // Manifest a CmpUL result in an integer register.  Very painful.
12166 // This is the test to avoid.
12167 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12168 %{
12169   match(Set dst (CmpUL3 src1 src2));
12170   effect(KILL flags);
12171 
12172   ins_cost(275); // XXX
12173   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12174             "movl    $dst, -1\n\t"
12175             "jb,u    done\n\t"
12176             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
12177     "done:" %}
12178   ins_encode %{
12179     Label done;
12180     __ cmpq($src1$$Register, $src2$$Register);
12181     __ movl($dst$$Register, -1);
12182     __ jccb(Assembler::below, done);
12183     __ setcc(Assembler::notZero, $dst$$Register);
12184     __ bind(done);
12185   %}
12186   ins_pipe(pipe_slow);
12187 %}
12188 
12189 // Unsigned long compare Instructions; really, same as signed long except they
12190 // produce an rFlagsRegU instead of rFlagsReg.
12191 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12192 %{
12193   match(Set cr (CmpUL op1 op2));
12194 
12195   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12196   ins_encode %{
12197     __ cmpq($op1$$Register, $op2$$Register);
12198   %}
12199   ins_pipe(ialu_cr_reg_reg);
12200 %}
12201 
12202 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12203 %{
12204   match(Set cr (CmpUL op1 op2));
12205 
12206   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12207   ins_encode %{
12208     __ cmpq($op1$$Register, $op2$$constant);
12209   %}
12210   ins_pipe(ialu_cr_reg_imm);
12211 %}
12212 
12213 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12214 %{
12215   match(Set cr (CmpUL op1 (LoadL op2)));
12216 
12217   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12218   ins_encode %{
12219     __ cmpq($op1$$Register, $op2$$Address);
12220   %}
12221   ins_pipe(ialu_cr_reg_mem);
12222 %}
12223 
12224 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12225 %{
12226   match(Set cr (CmpUL src zero));
12227 
12228   format %{ "testq   $src, $src\t# unsigned" %}
12229   ins_encode %{
12230     __ testq($src$$Register, $src$$Register);
12231   %}
12232   ins_pipe(ialu_cr_reg_imm);
12233 %}
12234 
12235 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12236 %{
12237   match(Set cr (CmpI (LoadB mem) imm));
12238 
12239   ins_cost(125);
12240   format %{ "cmpb    $mem, $imm" %}
12241   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12242   ins_pipe(ialu_cr_reg_mem);
12243 %}
12244 
12245 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
12246 %{
12247   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12248 
12249   ins_cost(125);
12250   format %{ "testb   $mem, $imm\t# ubyte" %}
12251   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12252   ins_pipe(ialu_cr_reg_mem);
12253 %}
12254 
12255 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
12256 %{
12257   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12258 
12259   ins_cost(125);
12260   format %{ "testb   $mem, $imm\t# byte" %}
12261   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12262   ins_pipe(ialu_cr_reg_mem);
12263 %}
12264 
12265 //----------Max and Min--------------------------------------------------------
12266 // Min Instructions
12267 
12268 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12269 %{
12270   effect(USE_DEF dst, USE src, USE cr);
12271 
12272   format %{ "cmovlgt $dst, $src\t# min" %}
12273   ins_encode %{
12274     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
12275   %}
12276   ins_pipe(pipe_cmov_reg);
12277 %}
12278 
12279 
12280 instruct minI_rReg(rRegI dst, rRegI src)
12281 %{
12282   match(Set dst (MinI dst src));
12283 
12284   ins_cost(200);
12285   expand %{
12286     rFlagsReg cr;
12287     compI_rReg(cr, dst, src);
12288     cmovI_reg_g(dst, src, cr);
12289   %}
12290 %}
12291 
12292 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12293 %{
12294   effect(USE_DEF dst, USE src, USE cr);
12295 
12296   format %{ "cmovllt $dst, $src\t# max" %}
12297   ins_encode %{
12298     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
12299   %}
12300   ins_pipe(pipe_cmov_reg);
12301 %}
12302 
12303 
12304 instruct maxI_rReg(rRegI dst, rRegI src)
12305 %{
12306   match(Set dst (MaxI dst src));
12307 
12308   ins_cost(200);
12309   expand %{
12310     rFlagsReg cr;
12311     compI_rReg(cr, dst, src);
12312     cmovI_reg_l(dst, src, cr);
12313   %}
12314 %}
12315 
12316 // ============================================================================
12317 // Branch Instructions
12318 
12319 // Jump Direct - Label defines a relative address from JMP+1
12320 instruct jmpDir(label labl)
12321 %{
12322   match(Goto);
12323   effect(USE labl);
12324 
12325   ins_cost(300);
12326   format %{ "jmp     $labl" %}
12327   size(5);
12328   ins_encode %{
12329     Label* L = $labl$$label;
12330     __ jmp(*L, false); // Always long jump
12331   %}
12332   ins_pipe(pipe_jmp);
12333 %}
12334 
12335 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12336 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12337 %{
12338   match(If cop cr);
12339   effect(USE labl);
12340 
12341   ins_cost(300);
12342   format %{ "j$cop     $labl" %}
12343   size(6);
12344   ins_encode %{
12345     Label* L = $labl$$label;
12346     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12347   %}
12348   ins_pipe(pipe_jcc);
12349 %}
12350 
12351 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12352 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12353 %{
12354   match(CountedLoopEnd cop cr);
12355   effect(USE labl);
12356 
12357   ins_cost(300);
12358   format %{ "j$cop     $labl\t# loop end" %}
12359   size(6);
12360   ins_encode %{
12361     Label* L = $labl$$label;
12362     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12363   %}
12364   ins_pipe(pipe_jcc);
12365 %}
12366 
12367 // Jump Direct Conditional - using unsigned comparison
12368 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12369   match(If cop cmp);
12370   effect(USE labl);
12371 
12372   ins_cost(300);
12373   format %{ "j$cop,u   $labl" %}
12374   size(6);
12375   ins_encode %{
12376     Label* L = $labl$$label;
12377     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12378   %}
12379   ins_pipe(pipe_jcc);
12380 %}
12381 
12382 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12383   match(If cop cmp);
12384   effect(USE labl);
12385 
12386   ins_cost(200);
12387   format %{ "j$cop,u   $labl" %}
12388   size(6);
12389   ins_encode %{
12390     Label* L = $labl$$label;
12391     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12392   %}
12393   ins_pipe(pipe_jcc);
12394 %}
12395 
12396 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12397   match(If cop cmp);
12398   effect(USE labl);
12399 
12400   ins_cost(200);
12401   format %{ $$template
12402     if ($cop$$cmpcode == Assembler::notEqual) {
12403       $$emit$$"jp,u    $labl\n\t"
12404       $$emit$$"j$cop,u   $labl"
12405     } else {
12406       $$emit$$"jp,u    done\n\t"
12407       $$emit$$"j$cop,u   $labl\n\t"
12408       $$emit$$"done:"
12409     }
12410   %}
12411   ins_encode %{
12412     Label* l = $labl$$label;
12413     if ($cop$$cmpcode == Assembler::notEqual) {
12414       __ jcc(Assembler::parity, *l, false);
12415       __ jcc(Assembler::notEqual, *l, false);
12416     } else if ($cop$$cmpcode == Assembler::equal) {
12417       Label done;
12418       __ jccb(Assembler::parity, done);
12419       __ jcc(Assembler::equal, *l, false);
12420       __ bind(done);
12421     } else {
12422        ShouldNotReachHere();
12423     }
12424   %}
12425   ins_pipe(pipe_jcc);
12426 %}
12427 
12428 // ============================================================================
12429 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12430 // superklass array for an instance of the superklass.  Set a hidden
12431 // internal cache on a hit (cache is checked with exposed code in
12432 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12433 // encoding ALSO sets flags.
12434 
12435 instruct partialSubtypeCheck(rdi_RegP result,
12436                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12437                              rFlagsReg cr)
12438 %{
12439   match(Set result (PartialSubtypeCheck sub super));
12440   predicate(!UseSecondarySupersTable);
12441   effect(KILL rcx, KILL cr);
12442 
12443   ins_cost(1100);  // slightly larger than the next version
12444   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12445             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12446             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12447             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12448             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12449             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12450             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12451     "miss:\t" %}
12452 
12453   ins_encode %{
12454     Label miss;
12455     // NB: Callers may assume that, when $result is a valid register,
12456     // check_klass_subtype_slow_path_linear sets it to a nonzero
12457     // value.
12458     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
12459                                             $rcx$$Register, $result$$Register,
12460                                             nullptr, &miss,
12461                                             /*set_cond_codes:*/ true);
12462     __ xorptr($result$$Register, $result$$Register);
12463     __ bind(miss);
12464   %}
12465 
12466   ins_pipe(pipe_slow);
12467 %}
12468 
12469 // ============================================================================
12470 // Two versions of hashtable-based partialSubtypeCheck, both used when
12471 // we need to search for a super class in the secondary supers array.
12472 // The first is used when we don't know _a priori_ the class being
12473 // searched for. The second, far more common, is used when we do know:
12474 // this is used for instanceof, checkcast, and any case where C2 can
12475 // determine it by constant propagation.
12476 
12477 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
12478                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
12479                                        rFlagsReg cr)
12480 %{
12481   match(Set result (PartialSubtypeCheck sub super));
12482   predicate(UseSecondarySupersTable);
12483   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
12484 
12485   ins_cost(1000);
12486   format %{ "partialSubtypeCheck $result, $sub, $super" %}
12487 
12488   ins_encode %{
12489     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
12490 					 $temp3$$Register, $temp4$$Register, $result$$Register);
12491   %}
12492 
12493   ins_pipe(pipe_slow);
12494 %}
12495 
12496 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
12497                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
12498                                        rFlagsReg cr)
12499 %{
12500   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
12501   predicate(UseSecondarySupersTable);
12502   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
12503 
12504   ins_cost(700);  // smaller than the next version
12505   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
12506 
12507   ins_encode %{
12508     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
12509     if (InlineSecondarySupersTest) {
12510       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
12511                                        $temp3$$Register, $temp4$$Register, $result$$Register,
12512                                        super_klass_slot);
12513     } else {
12514       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
12515     }
12516   %}
12517 
12518   ins_pipe(pipe_slow);
12519 %}
12520 
12521 // ============================================================================
12522 // Branch Instructions -- short offset versions
12523 //
12524 // These instructions are used to replace jumps of a long offset (the default
12525 // match) with jumps of a shorter offset.  These instructions are all tagged
12526 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12527 // match rules in general matching.  Instead, the ADLC generates a conversion
12528 // method in the MachNode which can be used to do in-place replacement of the
12529 // long variant with the shorter variant.  The compiler will determine if a
12530 // branch can be taken by the is_short_branch_offset() predicate in the machine
12531 // specific code section of the file.
12532 
12533 // Jump Direct - Label defines a relative address from JMP+1
12534 instruct jmpDir_short(label labl) %{
12535   match(Goto);
12536   effect(USE labl);
12537 
12538   ins_cost(300);
12539   format %{ "jmp,s   $labl" %}
12540   size(2);
12541   ins_encode %{
12542     Label* L = $labl$$label;
12543     __ jmpb(*L);
12544   %}
12545   ins_pipe(pipe_jmp);
12546   ins_short_branch(1);
12547 %}
12548 
12549 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12550 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12551   match(If cop cr);
12552   effect(USE labl);
12553 
12554   ins_cost(300);
12555   format %{ "j$cop,s   $labl" %}
12556   size(2);
12557   ins_encode %{
12558     Label* L = $labl$$label;
12559     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12560   %}
12561   ins_pipe(pipe_jcc);
12562   ins_short_branch(1);
12563 %}
12564 
12565 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12566 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12567   match(CountedLoopEnd cop cr);
12568   effect(USE labl);
12569 
12570   ins_cost(300);
12571   format %{ "j$cop,s   $labl\t# loop end" %}
12572   size(2);
12573   ins_encode %{
12574     Label* L = $labl$$label;
12575     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12576   %}
12577   ins_pipe(pipe_jcc);
12578   ins_short_branch(1);
12579 %}
12580 
12581 // Jump Direct Conditional - using unsigned comparison
12582 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12583   match(If cop cmp);
12584   effect(USE labl);
12585 
12586   ins_cost(300);
12587   format %{ "j$cop,us  $labl" %}
12588   size(2);
12589   ins_encode %{
12590     Label* L = $labl$$label;
12591     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12592   %}
12593   ins_pipe(pipe_jcc);
12594   ins_short_branch(1);
12595 %}
12596 
12597 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12598   match(If cop cmp);
12599   effect(USE labl);
12600 
12601   ins_cost(300);
12602   format %{ "j$cop,us  $labl" %}
12603   size(2);
12604   ins_encode %{
12605     Label* L = $labl$$label;
12606     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12607   %}
12608   ins_pipe(pipe_jcc);
12609   ins_short_branch(1);
12610 %}
12611 
12612 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12613   match(If cop cmp);
12614   effect(USE labl);
12615 
12616   ins_cost(300);
12617   format %{ $$template
12618     if ($cop$$cmpcode == Assembler::notEqual) {
12619       $$emit$$"jp,u,s  $labl\n\t"
12620       $$emit$$"j$cop,u,s  $labl"
12621     } else {
12622       $$emit$$"jp,u,s  done\n\t"
12623       $$emit$$"j$cop,u,s  $labl\n\t"
12624       $$emit$$"done:"
12625     }
12626   %}
12627   size(4);
12628   ins_encode %{
12629     Label* l = $labl$$label;
12630     if ($cop$$cmpcode == Assembler::notEqual) {
12631       __ jccb(Assembler::parity, *l);
12632       __ jccb(Assembler::notEqual, *l);
12633     } else if ($cop$$cmpcode == Assembler::equal) {
12634       Label done;
12635       __ jccb(Assembler::parity, done);
12636       __ jccb(Assembler::equal, *l);
12637       __ bind(done);
12638     } else {
12639        ShouldNotReachHere();
12640     }
12641   %}
12642   ins_pipe(pipe_jcc);
12643   ins_short_branch(1);
12644 %}
12645 
12646 // ============================================================================
12647 // inlined locking and unlocking
12648 
12649 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12650   predicate(LockingMode != LM_LIGHTWEIGHT);
12651   match(Set cr (FastLock object box));
12652   effect(TEMP tmp, TEMP scr, USE_KILL box);
12653   ins_cost(300);
12654   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12655   ins_encode %{
12656     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12657                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
12658   %}
12659   ins_pipe(pipe_slow);
12660 %}
12661 
12662 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12663   predicate(LockingMode != LM_LIGHTWEIGHT);
12664   match(Set cr (FastUnlock object box));
12665   effect(TEMP tmp, USE_KILL box);
12666   ins_cost(300);
12667   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12668   ins_encode %{
12669     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
12670   %}
12671   ins_pipe(pipe_slow);
12672 %}
12673 
12674 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
12675   predicate(LockingMode == LM_LIGHTWEIGHT);
12676   match(Set cr (FastLock object box));
12677   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
12678   ins_cost(300);
12679   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
12680   ins_encode %{
12681     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12682   %}
12683   ins_pipe(pipe_slow);
12684 %}
12685 
12686 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
12687   predicate(LockingMode == LM_LIGHTWEIGHT);
12688   match(Set cr (FastUnlock object rax_reg));
12689   effect(TEMP tmp, USE_KILL rax_reg);
12690   ins_cost(300);
12691   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
12692   ins_encode %{
12693     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
12694   %}
12695   ins_pipe(pipe_slow);
12696 %}
12697 
12698 
12699 // ============================================================================
12700 // Safepoint Instructions
12701 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12702 %{
12703   match(SafePoint poll);
12704   effect(KILL cr, USE poll);
12705 
12706   format %{ "testl   rax, [$poll]\t"
12707             "# Safepoint: poll for GC" %}
12708   ins_cost(125);
12709   ins_encode %{
12710     __ relocate(relocInfo::poll_type);
12711     address pre_pc = __ pc();
12712     __ testl(rax, Address($poll$$Register, 0));
12713     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12714   %}
12715   ins_pipe(ialu_reg_mem);
12716 %}
12717 
12718 instruct mask_all_evexL(kReg dst, rRegL src) %{
12719   match(Set dst (MaskAll src));
12720   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
12721   ins_encode %{
12722     int mask_len = Matcher::vector_length(this);
12723     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
12724   %}
12725   ins_pipe( pipe_slow );
12726 %}
12727 
12728 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
12729   predicate(Matcher::vector_length(n) > 32);
12730   match(Set dst (MaskAll src));
12731   effect(TEMP tmp);
12732   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
12733   ins_encode %{
12734     int mask_len = Matcher::vector_length(this);
12735     __ movslq($tmp$$Register, $src$$Register);
12736     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
12737   %}
12738   ins_pipe( pipe_slow );
12739 %}
12740 
12741 // ============================================================================
12742 // Procedure Call/Return Instructions
12743 // Call Java Static Instruction
12744 // Note: If this code changes, the corresponding ret_addr_offset() and
12745 //       compute_padding() functions will have to be adjusted.
12746 instruct CallStaticJavaDirect(method meth) %{
12747   match(CallStaticJava);
12748   effect(USE meth);
12749 
12750   ins_cost(300);
12751   format %{ "call,static " %}
12752   opcode(0xE8); /* E8 cd */
12753   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12754   ins_pipe(pipe_slow);
12755   ins_alignment(4);
12756 %}
12757 
12758 // Call Java Dynamic Instruction
12759 // Note: If this code changes, the corresponding ret_addr_offset() and
12760 //       compute_padding() functions will have to be adjusted.
12761 instruct CallDynamicJavaDirect(method meth)
12762 %{
12763   match(CallDynamicJava);
12764   effect(USE meth);
12765 
12766   ins_cost(300);
12767   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12768             "call,dynamic " %}
12769   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12770   ins_pipe(pipe_slow);
12771   ins_alignment(4);
12772 %}
12773 
12774 // Call Runtime Instruction
12775 instruct CallRuntimeDirect(method meth)
12776 %{
12777   match(CallRuntime);
12778   effect(USE meth);
12779 
12780   ins_cost(300);
12781   format %{ "call,runtime " %}
12782   ins_encode(clear_avx, Java_To_Runtime(meth));
12783   ins_pipe(pipe_slow);
12784 %}
12785 
12786 // Call runtime without safepoint
12787 instruct CallLeafDirect(method meth)
12788 %{
12789   match(CallLeaf);
12790   effect(USE meth);
12791 
12792   ins_cost(300);
12793   format %{ "call_leaf,runtime " %}
12794   ins_encode(clear_avx, Java_To_Runtime(meth));
12795   ins_pipe(pipe_slow);
12796 %}
12797 
12798 // Call runtime without safepoint and with vector arguments
12799 instruct CallLeafDirectVector(method meth)
12800 %{
12801   match(CallLeafVector);
12802   effect(USE meth);
12803 
12804   ins_cost(300);
12805   format %{ "call_leaf,vector " %}
12806   ins_encode(Java_To_Runtime(meth));
12807   ins_pipe(pipe_slow);
12808 %}
12809 
12810 // Call runtime without safepoint
12811 // entry point is null, target holds the address to call
12812 instruct CallLeafNoFPInDirect(rRegP target)
12813 %{
12814   predicate(n->as_Call()->entry_point() == nullptr);
12815   match(CallLeafNoFP target);
12816 
12817   ins_cost(300);
12818   format %{ "call_leaf_nofp,runtime indirect " %}
12819   ins_encode %{
12820      __ call($target$$Register);
12821   %}
12822 
12823   ins_pipe(pipe_slow);
12824 %}
12825 
12826 instruct CallLeafNoFPDirect(method meth)
12827 %{
12828   predicate(n->as_Call()->entry_point() != nullptr);
12829   match(CallLeafNoFP);
12830   effect(USE meth);
12831 
12832   ins_cost(300);
12833   format %{ "call_leaf_nofp,runtime " %}
12834   ins_encode(clear_avx, Java_To_Runtime(meth));
12835   ins_pipe(pipe_slow);
12836 %}
12837 
12838 // Return Instruction
12839 // Remove the return address & jump to it.
12840 // Notice: We always emit a nop after a ret to make sure there is room
12841 // for safepoint patching
12842 instruct Ret()
12843 %{
12844   match(Return);
12845 
12846   format %{ "ret" %}
12847   ins_encode %{
12848     __ ret(0);
12849   %}
12850   ins_pipe(pipe_jmp);
12851 %}
12852 
12853 // Tail Call; Jump from runtime stub to Java code.
12854 // Also known as an 'interprocedural jump'.
12855 // Target of jump will eventually return to caller.
12856 // TailJump below removes the return address.
12857 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
12858 // emitted just above the TailCall which has reset rbp to the caller state.
12859 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
12860 %{
12861   match(TailCall jump_target method_ptr);
12862 
12863   ins_cost(300);
12864   format %{ "jmp     $jump_target\t# rbx holds method" %}
12865   ins_encode %{
12866     __ jmp($jump_target$$Register);
12867   %}
12868   ins_pipe(pipe_jmp);
12869 %}
12870 
12871 // Tail Jump; remove the return address; jump to target.
12872 // TailCall above leaves the return address around.
12873 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12874 %{
12875   match(TailJump jump_target ex_oop);
12876 
12877   ins_cost(300);
12878   format %{ "popq    rdx\t# pop return address\n\t"
12879             "jmp     $jump_target" %}
12880   ins_encode %{
12881     __ popq(as_Register(RDX_enc));
12882     __ jmp($jump_target$$Register);
12883   %}
12884   ins_pipe(pipe_jmp);
12885 %}
12886 
12887 // Forward exception.
12888 instruct ForwardExceptionjmp()
12889 %{
12890   match(ForwardException);
12891 
12892   format %{ "jmp     forward_exception_stub" %}
12893   ins_encode %{
12894     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
12895   %}
12896   ins_pipe(pipe_jmp);
12897 %}
12898 
12899 // Create exception oop: created by stack-crawling runtime code.
12900 // Created exception is now available to this handler, and is setup
12901 // just prior to jumping to this handler.  No code emitted.
12902 instruct CreateException(rax_RegP ex_oop)
12903 %{
12904   match(Set ex_oop (CreateEx));
12905 
12906   size(0);
12907   // use the following format syntax
12908   format %{ "# exception oop is in rax; no code emitted" %}
12909   ins_encode();
12910   ins_pipe(empty);
12911 %}
12912 
12913 // Rethrow exception:
12914 // The exception oop will come in the first argument position.
12915 // Then JUMP (not call) to the rethrow stub code.
12916 instruct RethrowException()
12917 %{
12918   match(Rethrow);
12919 
12920   // use the following format syntax
12921   format %{ "jmp     rethrow_stub" %}
12922   ins_encode %{
12923     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
12924   %}
12925   ins_pipe(pipe_jmp);
12926 %}
12927 
12928 // ============================================================================
12929 // This name is KNOWN by the ADLC and cannot be changed.
12930 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12931 // for this guy.
12932 instruct tlsLoadP(r15_RegP dst) %{
12933   match(Set dst (ThreadLocal));
12934   effect(DEF dst);
12935 
12936   size(0);
12937   format %{ "# TLS is in R15" %}
12938   ins_encode( /*empty encoding*/ );
12939   ins_pipe(ialu_reg_reg);
12940 %}
12941 
12942 
12943 //----------PEEPHOLE RULES-----------------------------------------------------
12944 // These must follow all instruction definitions as they use the names
12945 // defined in the instructions definitions.
12946 //
12947 // peeppredicate ( rule_predicate );
12948 // // the predicate unless which the peephole rule will be ignored
12949 //
12950 // peepmatch ( root_instr_name [preceding_instruction]* );
12951 //
12952 // peepprocedure ( procedure_name );
12953 // // provide a procedure name to perform the optimization, the procedure should
12954 // // reside in the architecture dependent peephole file, the method has the
12955 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
12956 // // with the arguments being the basic block, the current node index inside the
12957 // // block, the register allocator, the functions upon invoked return a new node
12958 // // defined in peepreplace, and the rules of the nodes appearing in the
12959 // // corresponding peepmatch, the function return true if successful, else
12960 // // return false
12961 //
12962 // peepconstraint %{
12963 // (instruction_number.operand_name relational_op instruction_number.operand_name
12964 //  [, ...] );
12965 // // instruction numbers are zero-based using left to right order in peepmatch
12966 //
12967 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12968 // // provide an instruction_number.operand_name for each operand that appears
12969 // // in the replacement instruction's match rule
12970 //
12971 // ---------VM FLAGS---------------------------------------------------------
12972 //
12973 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12974 //
12975 // Each peephole rule is given an identifying number starting with zero and
12976 // increasing by one in the order seen by the parser.  An individual peephole
12977 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12978 // on the command-line.
12979 //
12980 // ---------CURRENT LIMITATIONS----------------------------------------------
12981 //
12982 // Only transformations inside a basic block (do we need more for peephole)
12983 //
12984 // ---------EXAMPLE----------------------------------------------------------
12985 //
12986 // // pertinent parts of existing instructions in architecture description
12987 // instruct movI(rRegI dst, rRegI src)
12988 // %{
12989 //   match(Set dst (CopyI src));
12990 // %}
12991 //
12992 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
12993 // %{
12994 //   match(Set dst (AddI dst src));
12995 //   effect(KILL cr);
12996 // %}
12997 //
12998 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
12999 // %{
13000 //   match(Set dst (AddI dst src));
13001 // %}
13002 //
13003 // 1. Simple replacement
13004 // - Only match adjacent instructions in same basic block
13005 // - Only equality constraints
13006 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
13007 // - Only one replacement instruction
13008 //
13009 // // Change (inc mov) to lea
13010 // peephole %{
13011 //   // lea should only be emitted when beneficial
13012 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13013 //   // increment preceded by register-register move
13014 //   peepmatch ( incI_rReg movI );
13015 //   // require that the destination register of the increment
13016 //   // match the destination register of the move
13017 //   peepconstraint ( 0.dst == 1.dst );
13018 //   // construct a replacement instruction that sets
13019 //   // the destination to ( move's source register + one )
13020 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13021 // %}
13022 //
13023 // 2. Procedural replacement
13024 // - More flexible finding relevent nodes
13025 // - More flexible constraints
13026 // - More flexible transformations
13027 // - May utilise architecture-dependent API more effectively
13028 // - Currently only one replacement instruction due to adlc parsing capabilities
13029 //
13030 // // Change (inc mov) to lea
13031 // peephole %{
13032 //   // lea should only be emitted when beneficial
13033 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
13034 //   // the rule numbers of these nodes inside are passed into the function below
13035 //   peepmatch ( incI_rReg movI );
13036 //   // the method that takes the responsibility of transformation
13037 //   peepprocedure ( inc_mov_to_lea );
13038 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
13039 //   // node is passed into the function above
13040 //   peepreplace ( leaI_rReg_immI() );
13041 // %}
13042 
13043 // These instructions is not matched by the matcher but used by the peephole
13044 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
13045 %{
13046   predicate(false);
13047   match(Set dst (AddI src1 src2));
13048   format %{ "leal    $dst, [$src1 + $src2]" %}
13049   ins_encode %{
13050     Register dst = $dst$$Register;
13051     Register src1 = $src1$$Register;
13052     Register src2 = $src2$$Register;
13053     if (src1 != rbp && src1 != r13) {
13054       __ leal(dst, Address(src1, src2, Address::times_1));
13055     } else {
13056       assert(src2 != rbp && src2 != r13, "");
13057       __ leal(dst, Address(src2, src1, Address::times_1));
13058     }
13059   %}
13060   ins_pipe(ialu_reg_reg);
13061 %}
13062 
13063 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
13064 %{
13065   predicate(false);
13066   match(Set dst (AddI src1 src2));
13067   format %{ "leal    $dst, [$src1 + $src2]" %}
13068   ins_encode %{
13069     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
13070   %}
13071   ins_pipe(ialu_reg_reg);
13072 %}
13073 
13074 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
13075 %{
13076   predicate(false);
13077   match(Set dst (LShiftI src shift));
13078   format %{ "leal    $dst, [$src << $shift]" %}
13079   ins_encode %{
13080     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13081     Register src = $src$$Register;
13082     if (scale == Address::times_2 && src != rbp && src != r13) {
13083       __ leal($dst$$Register, Address(src, src, Address::times_1));
13084     } else {
13085       __ leal($dst$$Register, Address(noreg, src, scale));
13086     }
13087   %}
13088   ins_pipe(ialu_reg_reg);
13089 %}
13090 
13091 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
13092 %{
13093   predicate(false);
13094   match(Set dst (AddL src1 src2));
13095   format %{ "leaq    $dst, [$src1 + $src2]" %}
13096   ins_encode %{
13097     Register dst = $dst$$Register;
13098     Register src1 = $src1$$Register;
13099     Register src2 = $src2$$Register;
13100     if (src1 != rbp && src1 != r13) {
13101       __ leaq(dst, Address(src1, src2, Address::times_1));
13102     } else {
13103       assert(src2 != rbp && src2 != r13, "");
13104       __ leaq(dst, Address(src2, src1, Address::times_1));
13105     }
13106   %}
13107   ins_pipe(ialu_reg_reg);
13108 %}
13109 
13110 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
13111 %{
13112   predicate(false);
13113   match(Set dst (AddL src1 src2));
13114   format %{ "leaq    $dst, [$src1 + $src2]" %}
13115   ins_encode %{
13116     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
13117   %}
13118   ins_pipe(ialu_reg_reg);
13119 %}
13120 
13121 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
13122 %{
13123   predicate(false);
13124   match(Set dst (LShiftL src shift));
13125   format %{ "leaq    $dst, [$src << $shift]" %}
13126   ins_encode %{
13127     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
13128     Register src = $src$$Register;
13129     if (scale == Address::times_2 && src != rbp && src != r13) {
13130       __ leaq($dst$$Register, Address(src, src, Address::times_1));
13131     } else {
13132       __ leaq($dst$$Register, Address(noreg, src, scale));
13133     }
13134   %}
13135   ins_pipe(ialu_reg_reg);
13136 %}
13137 
13138 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
13139 // sal}) with lea instructions. The {add, sal} rules are beneficial in
13140 // processors with at least partial ALU support for lea
13141 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
13142 // beneficial for processors with full ALU support
13143 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
13144 
13145 peephole
13146 %{
13147   peeppredicate(VM_Version::supports_fast_2op_lea());
13148   peepmatch (addI_rReg);
13149   peepprocedure (lea_coalesce_reg);
13150   peepreplace (leaI_rReg_rReg_peep());
13151 %}
13152 
13153 peephole
13154 %{
13155   peeppredicate(VM_Version::supports_fast_2op_lea());
13156   peepmatch (addI_rReg_imm);
13157   peepprocedure (lea_coalesce_imm);
13158   peepreplace (leaI_rReg_immI_peep());
13159 %}
13160 
13161 peephole
13162 %{
13163   peeppredicate(VM_Version::supports_fast_3op_lea() ||
13164                 VM_Version::is_intel_cascade_lake());
13165   peepmatch (incI_rReg);
13166   peepprocedure (lea_coalesce_imm);
13167   peepreplace (leaI_rReg_immI_peep());
13168 %}
13169 
13170 peephole
13171 %{
13172   peeppredicate(VM_Version::supports_fast_3op_lea() ||
13173                 VM_Version::is_intel_cascade_lake());
13174   peepmatch (decI_rReg);
13175   peepprocedure (lea_coalesce_imm);
13176   peepreplace (leaI_rReg_immI_peep());
13177 %}
13178 
13179 peephole
13180 %{
13181   peeppredicate(VM_Version::supports_fast_2op_lea());
13182   peepmatch (salI_rReg_immI2);
13183   peepprocedure (lea_coalesce_imm);
13184   peepreplace (leaI_rReg_immI2_peep());
13185 %}
13186 
13187 peephole
13188 %{
13189   peeppredicate(VM_Version::supports_fast_2op_lea());
13190   peepmatch (addL_rReg);
13191   peepprocedure (lea_coalesce_reg);
13192   peepreplace (leaL_rReg_rReg_peep());
13193 %}
13194 
13195 peephole
13196 %{
13197   peeppredicate(VM_Version::supports_fast_2op_lea());
13198   peepmatch (addL_rReg_imm);
13199   peepprocedure (lea_coalesce_imm);
13200   peepreplace (leaL_rReg_immL32_peep());
13201 %}
13202 
13203 peephole
13204 %{
13205   peeppredicate(VM_Version::supports_fast_3op_lea() ||
13206                 VM_Version::is_intel_cascade_lake());
13207   peepmatch (incL_rReg);
13208   peepprocedure (lea_coalesce_imm);
13209   peepreplace (leaL_rReg_immL32_peep());
13210 %}
13211 
13212 peephole
13213 %{
13214   peeppredicate(VM_Version::supports_fast_3op_lea() ||
13215                 VM_Version::is_intel_cascade_lake());
13216   peepmatch (decL_rReg);
13217   peepprocedure (lea_coalesce_imm);
13218   peepreplace (leaL_rReg_immL32_peep());
13219 %}
13220 
13221 peephole
13222 %{
13223   peeppredicate(VM_Version::supports_fast_2op_lea());
13224   peepmatch (salL_rReg_immI2);
13225   peepprocedure (lea_coalesce_imm);
13226   peepreplace (leaL_rReg_immI2_peep());
13227 %}
13228 
13229 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
13230 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
13231 
13232 //int variant
13233 peephole
13234 %{
13235   peepmatch (testI_reg);
13236   peepprocedure (test_may_remove);
13237 %}
13238 
13239 //long variant
13240 peephole
13241 %{
13242   peepmatch (testL_reg);
13243   peepprocedure (test_may_remove);
13244 %}
13245 
13246 
13247 //----------SMARTSPILL RULES---------------------------------------------------
13248 // These must follow all instruction definitions as they use the names
13249 // defined in the instructions definitions.