1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 %}
  426 
  427 // Register masks
  428 source_hpp %{
  429 
  430 extern RegMask _ANY_REG_mask;
  431 extern RegMask _PTR_REG_mask;
  432 extern RegMask _PTR_REG_NO_RBP_mask;
  433 extern RegMask _PTR_NO_RAX_REG_mask;
  434 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  435 extern RegMask _LONG_REG_mask;
  436 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  437 extern RegMask _LONG_NO_RCX_REG_mask;
  438 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  439 extern RegMask _INT_REG_mask;
  440 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  441 extern RegMask _INT_NO_RCX_REG_mask;
  442 extern RegMask _INT_NO_RBP_R13_REG_mask;
  443 extern RegMask _FLOAT_REG_mask;
  444 
  445 extern RegMask _STACK_OR_PTR_REG_mask;
  446 extern RegMask _STACK_OR_LONG_REG_mask;
  447 extern RegMask _STACK_OR_INT_REG_mask;
  448 
  449 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  450 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  451 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  452 
  453 %}
  454 
  455 source %{
  456 #define   RELOC_IMM64    Assembler::imm_operand
  457 #define   RELOC_DISP32   Assembler::disp32_operand
  458 
  459 #define __ masm->
  460 
  461 RegMask _ANY_REG_mask;
  462 RegMask _PTR_REG_mask;
  463 RegMask _PTR_REG_NO_RBP_mask;
  464 RegMask _PTR_NO_RAX_REG_mask;
  465 RegMask _PTR_NO_RAX_RBX_REG_mask;
  466 RegMask _LONG_REG_mask;
  467 RegMask _LONG_NO_RAX_RDX_REG_mask;
  468 RegMask _LONG_NO_RCX_REG_mask;
  469 RegMask _LONG_NO_RBP_R13_REG_mask;
  470 RegMask _INT_REG_mask;
  471 RegMask _INT_NO_RAX_RDX_REG_mask;
  472 RegMask _INT_NO_RCX_REG_mask;
  473 RegMask _INT_NO_RBP_R13_REG_mask;
  474 RegMask _FLOAT_REG_mask;
  475 RegMask _STACK_OR_PTR_REG_mask;
  476 RegMask _STACK_OR_LONG_REG_mask;
  477 RegMask _STACK_OR_INT_REG_mask;
  478 
  479 static bool need_r12_heapbase() {
  480   return UseCompressedOops;
  481 }
  482 
  483 void reg_mask_init() {
  484   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  485 
  486   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  487   // We derive a number of subsets from it.
  488   _ANY_REG_mask = _ALL_REG_mask;
  489 
  490   if (PreserveFramePointer) {
  491     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  492     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  493   }
  494   if (need_r12_heapbase()) {
  495     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  496     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  497   }
  498 
  499   _PTR_REG_mask = _ANY_REG_mask;
  500   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  501   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  502   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  503   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  504   if (!UseAPX) {
  505     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  506       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  507       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  508     }
  509   }
  510 
  511   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  512   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  513 
  514   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  515   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  516   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  517 
  518   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  519   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  520   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  521 
  522   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  523   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  524   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  525 
  526 
  527   _LONG_REG_mask = _PTR_REG_mask;
  528   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  529   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  530 
  531   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  532   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  533   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  534   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  535   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  536 
  537   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  538   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  539   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  540 
  541   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  542   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  543   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  544   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  545   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  546 
  547   _INT_REG_mask = _ALL_INT_REG_mask;
  548   if (!UseAPX) {
  549     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  550       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  551     }
  552   }
  553 
  554   if (PreserveFramePointer) {
  555     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  556   }
  557   if (need_r12_heapbase()) {
  558     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  559   }
  560 
  561   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  562   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  563 
  564   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  565   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  566   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  567 
  568   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  569   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  570 
  571   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  572   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  573   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  574 
  575   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  576   // from the float_reg_legacy/float_reg_evex register class.
  577   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  578 }
  579 
  580 static bool generate_vzeroupper(Compile* C) {
  581   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  582 }
  583 
  584 static int clear_avx_size() {
  585   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  586 }
  587 
  588 // !!!!! Special hack to get all types of calls to specify the byte offset
  589 //       from the start of the call to the point where the return address
  590 //       will point.
  591 int MachCallStaticJavaNode::ret_addr_offset()
  592 {
  593   int offset = 5; // 5 bytes from start of call to where return address points
  594   offset += clear_avx_size();
  595   return offset;
  596 }
  597 
  598 int MachCallDynamicJavaNode::ret_addr_offset()
  599 {
  600   int offset = 15; // 15 bytes from start of call to where return address points
  601   offset += clear_avx_size();
  602   return offset;
  603 }
  604 
  605 int MachCallRuntimeNode::ret_addr_offset() {
  606   if (_entry_point == nullptr) {
  607     // CallLeafNoFPInDirect
  608     return 3; // callq (register)
  609   }
  610   int offset = 13; // movq r10,#addr; callq (r10)
  611   if (this->ideal_Opcode() != Op_CallLeafVector) {
  612     offset += clear_avx_size();
  613   }
  614   return offset;
  615 }
  616 
  617 //
  618 // Compute padding required for nodes which need alignment
  619 //
  620 
  621 // The address of the call instruction needs to be 4-byte aligned to
  622 // ensure that it does not span a cache line so that it can be patched.
  623 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  624 {
  625   current_offset += clear_avx_size(); // skip vzeroupper
  626   current_offset += 1; // skip call opcode byte
  627   return align_up(current_offset, alignment_required()) - current_offset;
  628 }
  629 
  630 // The address of the call instruction needs to be 4-byte aligned to
  631 // ensure that it does not span a cache line so that it can be patched.
  632 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  633 {
  634   current_offset += clear_avx_size(); // skip vzeroupper
  635   current_offset += 11; // skip movq instruction + call opcode byte
  636   return align_up(current_offset, alignment_required()) - current_offset;
  637 }
  638 
  639 // This could be in MacroAssembler but it's fairly C2 specific
  640 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  641   Label exit;
  642   __ jccb(Assembler::noParity, exit);
  643   __ pushf();
  644   //
  645   // comiss/ucomiss instructions set ZF,PF,CF flags and
  646   // zero OF,AF,SF for NaN values.
  647   // Fixup flags by zeroing ZF,PF so that compare of NaN
  648   // values returns 'less than' result (CF is set).
  649   // Leave the rest of flags unchanged.
  650   //
  651   //    7 6 5 4 3 2 1 0
  652   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  653   //    0 0 1 0 1 0 1 1   (0x2B)
  654   //
  655   __ andq(Address(rsp, 0), 0xffffff2b);
  656   __ popf();
  657   __ bind(exit);
  658 }
  659 
  660 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  661   Label done;
  662   __ movl(dst, -1);
  663   __ jcc(Assembler::parity, done);
  664   __ jcc(Assembler::below, done);
  665   __ setcc(Assembler::notEqual, dst);
  666   __ bind(done);
  667 }
  668 
  669 // Math.min()    # Math.max()
  670 // --------------------------
  671 // ucomis[s/d]   #
  672 // ja   -> b     # a
  673 // jp   -> NaN   # NaN
  674 // jb   -> a     # b
  675 // je            #
  676 // |-jz -> a | b # a & b
  677 // |    -> a     #
  678 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  679                             XMMRegister a, XMMRegister b,
  680                             XMMRegister xmmt, Register rt,
  681                             bool min, bool single) {
  682 
  683   Label nan, zero, below, above, done;
  684 
  685   if (single)
  686     __ ucomiss(a, b);
  687   else
  688     __ ucomisd(a, b);
  689 
  690   if (dst->encoding() != (min ? b : a)->encoding())
  691     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  692   else
  693     __ jccb(Assembler::above, done);
  694 
  695   __ jccb(Assembler::parity, nan);  // PF=1
  696   __ jccb(Assembler::below, below); // CF=1
  697 
  698   // equal
  699   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  700   if (single) {
  701     __ ucomiss(a, xmmt);
  702     __ jccb(Assembler::equal, zero);
  703 
  704     __ movflt(dst, a);
  705     __ jmp(done);
  706   }
  707   else {
  708     __ ucomisd(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movdbl(dst, a);
  712     __ jmp(done);
  713   }
  714 
  715   __ bind(zero);
  716   if (min)
  717     __ vpor(dst, a, b, Assembler::AVX_128bit);
  718   else
  719     __ vpand(dst, a, b, Assembler::AVX_128bit);
  720 
  721   __ jmp(done);
  722 
  723   __ bind(above);
  724   if (single)
  725     __ movflt(dst, min ? b : a);
  726   else
  727     __ movdbl(dst, min ? b : a);
  728 
  729   __ jmp(done);
  730 
  731   __ bind(nan);
  732   if (single) {
  733     __ movl(rt, 0x7fc00000); // Float.NaN
  734     __ movdl(dst, rt);
  735   }
  736   else {
  737     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  738     __ movdq(dst, rt);
  739   }
  740   __ jmp(done);
  741 
  742   __ bind(below);
  743   if (single)
  744     __ movflt(dst, min ? a : b);
  745   else
  746     __ movdbl(dst, min ? a : b);
  747 
  748   __ bind(done);
  749 }
  750 
  751 //=============================================================================
  752 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  753 
  754 int ConstantTable::calculate_table_base_offset() const {
  755   return 0;  // absolute addressing, no offset
  756 }
  757 
  758 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  759 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  760   ShouldNotReachHere();
  761 }
  762 
  763 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  764   // Empty encoding
  765 }
  766 
  767 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  768   return 0;
  769 }
  770 
  771 #ifndef PRODUCT
  772 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  773   st->print("# MachConstantBaseNode (empty encoding)");
  774 }
  775 #endif
  776 
  777 
  778 //=============================================================================
  779 #ifndef PRODUCT
  780 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  781   Compile* C = ra_->C;
  782 
  783   int framesize = C->output()->frame_size_in_bytes();
  784   int bangsize = C->output()->bang_size_in_bytes();
  785   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  786   // Remove wordSize for return addr which is already pushed.
  787   framesize -= wordSize;
  788 
  789   if (C->output()->need_stack_bang(bangsize)) {
  790     framesize -= wordSize;
  791     st->print("# stack bang (%d bytes)", bangsize);
  792     st->print("\n\t");
  793     st->print("pushq   rbp\t# Save rbp");
  794     if (PreserveFramePointer) {
  795         st->print("\n\t");
  796         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  797     }
  798     if (framesize) {
  799       st->print("\n\t");
  800       st->print("subq    rsp, #%d\t# Create frame",framesize);
  801     }
  802   } else {
  803     st->print("subq    rsp, #%d\t# Create frame",framesize);
  804     st->print("\n\t");
  805     framesize -= wordSize;
  806     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  807     if (PreserveFramePointer) {
  808       st->print("\n\t");
  809       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  810       if (framesize > 0) {
  811         st->print("\n\t");
  812         st->print("addq    rbp, #%d", framesize);
  813       }
  814     }
  815   }
  816 
  817   if (VerifyStackAtCalls) {
  818     st->print("\n\t");
  819     framesize -= wordSize;
  820     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  821 #ifdef ASSERT
  822     st->print("\n\t");
  823     st->print("# stack alignment check");
  824 #endif
  825   }
  826   if (C->stub_function() != nullptr) {
  827     st->print("\n\t");
  828     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  829     st->print("\n\t");
  830     st->print("je      fast_entry\t");
  831     st->print("\n\t");
  832     st->print("call    #nmethod_entry_barrier_stub\t");
  833     st->print("\n\tfast_entry:");
  834   }
  835   st->cr();
  836 }
  837 #endif
  838 
  839 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  840   Compile* C = ra_->C;
  841 
  842   __ verified_entry(C);
  843 
  844   if (ra_->C->stub_function() == nullptr) {
  845     __ entry_barrier();
  846   }
  847 
  848   if (!Compile::current()->output()->in_scratch_emit_size()) {
  849     __ bind(*_verified_entry);
  850   }
  851 
  852   C->output()->set_frame_complete(__ offset());
  853 
  854   if (C->has_mach_constant_base_node()) {
  855     // NOTE: We set the table base offset here because users might be
  856     // emitted before MachConstantBaseNode.
  857     ConstantTable& constant_table = C->output()->constant_table();
  858     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  859   }
  860 }
  861 
  862 int MachPrologNode::reloc() const
  863 {
  864   return 0; // a large enough number
  865 }
  866 
  867 //=============================================================================
  868 #ifndef PRODUCT
  869 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  870 {
  871   Compile* C = ra_->C;
  872   if (generate_vzeroupper(C)) {
  873     st->print("vzeroupper");
  874     st->cr(); st->print("\t");
  875   }
  876 
  877   int framesize = C->output()->frame_size_in_bytes();
  878   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  879   // Remove word for return adr already pushed
  880   // and RBP
  881   framesize -= 2*wordSize;
  882 
  883   if (framesize) {
  884     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  885     st->print("\t");
  886   }
  887 
  888   st->print_cr("popq    rbp");
  889   if (do_polling() && C->is_method_compilation()) {
  890     st->print("\t");
  891     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  892                  "ja      #safepoint_stub\t"
  893                  "# Safepoint: poll for GC");
  894   }
  895 }
  896 #endif
  897 
  898 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  899 {
  900   Compile* C = ra_->C;
  901 
  902   if (generate_vzeroupper(C)) {
  903     // Clear upper bits of YMM registers when current compiled code uses
  904     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  905     __ vzeroupper();
  906   }
  907 
  908   // Subtract two words to account for return address and rbp
  909   int initial_framesize = C->output()->frame_size_in_bytes() - 2*wordSize;
  910   __ remove_frame(initial_framesize, C->needs_stack_repair());
  911 
  912   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  913     __ reserved_stack_check();
  914   }
  915 
  916   if (do_polling() && C->is_method_compilation()) {
  917     Label dummy_label;
  918     Label* code_stub = &dummy_label;
  919     if (!C->output()->in_scratch_emit_size()) {
  920       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  921       C->output()->add_stub(stub);
  922       code_stub = &stub->entry();
  923     }
  924     __ relocate(relocInfo::poll_return_type);
  925     __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
  926   }
  927 }
  928 
  929 int MachEpilogNode::reloc() const
  930 {
  931   return 2; // a large enough number
  932 }
  933 
  934 const Pipeline* MachEpilogNode::pipeline() const
  935 {
  936   return MachNode::pipeline_class();
  937 }
  938 
  939 //=============================================================================
  940 
  941 enum RC {
  942   rc_bad,
  943   rc_int,
  944   rc_kreg,
  945   rc_float,
  946   rc_stack
  947 };
  948 
  949 static enum RC rc_class(OptoReg::Name reg)
  950 {
  951   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  952 
  953   if (OptoReg::is_stack(reg)) return rc_stack;
  954 
  955   VMReg r = OptoReg::as_VMReg(reg);
  956 
  957   if (r->is_Register()) return rc_int;
  958 
  959   if (r->is_KRegister()) return rc_kreg;
  960 
  961   assert(r->is_XMMRegister(), "must be");
  962   return rc_float;
  963 }
  964 
  965 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
  966 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
  967                           int src_hi, int dst_hi, uint ireg, outputStream* st);
  968 
  969 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
  970                      int stack_offset, int reg, uint ireg, outputStream* st);
  971 
  972 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
  973                                       int dst_offset, uint ireg, outputStream* st) {
  974   if (masm) {
  975     switch (ireg) {
  976     case Op_VecS:
  977       __ movq(Address(rsp, -8), rax);
  978       __ movl(rax, Address(rsp, src_offset));
  979       __ movl(Address(rsp, dst_offset), rax);
  980       __ movq(rax, Address(rsp, -8));
  981       break;
  982     case Op_VecD:
  983       __ pushq(Address(rsp, src_offset));
  984       __ popq (Address(rsp, dst_offset));
  985       break;
  986     case Op_VecX:
  987       __ pushq(Address(rsp, src_offset));
  988       __ popq (Address(rsp, dst_offset));
  989       __ pushq(Address(rsp, src_offset+8));
  990       __ popq (Address(rsp, dst_offset+8));
  991       break;
  992     case Op_VecY:
  993       __ vmovdqu(Address(rsp, -32), xmm0);
  994       __ vmovdqu(xmm0, Address(rsp, src_offset));
  995       __ vmovdqu(Address(rsp, dst_offset), xmm0);
  996       __ vmovdqu(xmm0, Address(rsp, -32));
  997       break;
  998     case Op_VecZ:
  999       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1000       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1001       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1002       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1003       break;
 1004     default:
 1005       ShouldNotReachHere();
 1006     }
 1007 #ifndef PRODUCT
 1008   } else {
 1009     switch (ireg) {
 1010     case Op_VecS:
 1011       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1012                 "movl    rax, [rsp + #%d]\n\t"
 1013                 "movl    [rsp + #%d], rax\n\t"
 1014                 "movq    rax, [rsp - #8]",
 1015                 src_offset, dst_offset);
 1016       break;
 1017     case Op_VecD:
 1018       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1019                 "popq    [rsp + #%d]",
 1020                 src_offset, dst_offset);
 1021       break;
 1022      case Op_VecX:
 1023       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1024                 "popq    [rsp + #%d]\n\t"
 1025                 "pushq   [rsp + #%d]\n\t"
 1026                 "popq    [rsp + #%d]",
 1027                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1028       break;
 1029     case Op_VecY:
 1030       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1031                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1032                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1033                 "vmovdqu xmm0, [rsp - #32]",
 1034                 src_offset, dst_offset);
 1035       break;
 1036     case Op_VecZ:
 1037       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1038                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1039                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1040                 "vmovdqu xmm0, [rsp - #64]",
 1041                 src_offset, dst_offset);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #endif
 1047   }
 1048 }
 1049 
 1050 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1051                                        PhaseRegAlloc* ra_,
 1052                                        bool do_size,
 1053                                        outputStream* st) const {
 1054   assert(masm != nullptr || st  != nullptr, "sanity");
 1055   // Get registers to move
 1056   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1057   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1058   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1059   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1060 
 1061   enum RC src_second_rc = rc_class(src_second);
 1062   enum RC src_first_rc = rc_class(src_first);
 1063   enum RC dst_second_rc = rc_class(dst_second);
 1064   enum RC dst_first_rc = rc_class(dst_first);
 1065 
 1066   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1067          "must move at least 1 register" );
 1068 
 1069   if (src_first == dst_first && src_second == dst_second) {
 1070     // Self copy, no move
 1071     return 0;
 1072   }
 1073   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1074     uint ireg = ideal_reg();
 1075     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1076     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1077     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1078       // mem -> mem
 1079       int src_offset = ra_->reg2offset(src_first);
 1080       int dst_offset = ra_->reg2offset(dst_first);
 1081       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1082     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1083       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1084     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1085       int stack_offset = ra_->reg2offset(dst_first);
 1086       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1087     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1088       int stack_offset = ra_->reg2offset(src_first);
 1089       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1090     } else {
 1091       ShouldNotReachHere();
 1092     }
 1093     return 0;
 1094   }
 1095   if (src_first_rc == rc_stack) {
 1096     // mem ->
 1097     if (dst_first_rc == rc_stack) {
 1098       // mem -> mem
 1099       assert(src_second != dst_first, "overlap");
 1100       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1101           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1102         // 64-bit
 1103         int src_offset = ra_->reg2offset(src_first);
 1104         int dst_offset = ra_->reg2offset(dst_first);
 1105         if (masm) {
 1106           __ pushq(Address(rsp, src_offset));
 1107           __ popq (Address(rsp, dst_offset));
 1108 #ifndef PRODUCT
 1109         } else {
 1110           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1111                     "popq    [rsp + #%d]",
 1112                      src_offset, dst_offset);
 1113 #endif
 1114         }
 1115       } else {
 1116         // 32-bit
 1117         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1118         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1119         // No pushl/popl, so:
 1120         int src_offset = ra_->reg2offset(src_first);
 1121         int dst_offset = ra_->reg2offset(dst_first);
 1122         if (masm) {
 1123           __ movq(Address(rsp, -8), rax);
 1124           __ movl(rax, Address(rsp, src_offset));
 1125           __ movl(Address(rsp, dst_offset), rax);
 1126           __ movq(rax, Address(rsp, -8));
 1127 #ifndef PRODUCT
 1128         } else {
 1129           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1130                     "movl    rax, [rsp + #%d]\n\t"
 1131                     "movl    [rsp + #%d], rax\n\t"
 1132                     "movq    rax, [rsp - #8]",
 1133                      src_offset, dst_offset);
 1134 #endif
 1135         }
 1136       }
 1137       return 0;
 1138     } else if (dst_first_rc == rc_int) {
 1139       // mem -> gpr
 1140       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1141           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1142         // 64-bit
 1143         int offset = ra_->reg2offset(src_first);
 1144         if (masm) {
 1145           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1146 #ifndef PRODUCT
 1147         } else {
 1148           st->print("movq    %s, [rsp + #%d]\t# spill",
 1149                      Matcher::regName[dst_first],
 1150                      offset);
 1151 #endif
 1152         }
 1153       } else {
 1154         // 32-bit
 1155         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1156         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1157         int offset = ra_->reg2offset(src_first);
 1158         if (masm) {
 1159           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1160 #ifndef PRODUCT
 1161         } else {
 1162           st->print("movl    %s, [rsp + #%d]\t# spill",
 1163                      Matcher::regName[dst_first],
 1164                      offset);
 1165 #endif
 1166         }
 1167       }
 1168       return 0;
 1169     } else if (dst_first_rc == rc_float) {
 1170       // mem-> xmm
 1171       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1172           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1173         // 64-bit
 1174         int offset = ra_->reg2offset(src_first);
 1175         if (masm) {
 1176           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1177 #ifndef PRODUCT
 1178         } else {
 1179           st->print("%s  %s, [rsp + #%d]\t# spill",
 1180                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1181                      Matcher::regName[dst_first],
 1182                      offset);
 1183 #endif
 1184         }
 1185       } else {
 1186         // 32-bit
 1187         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1188         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1189         int offset = ra_->reg2offset(src_first);
 1190         if (masm) {
 1191           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1192 #ifndef PRODUCT
 1193         } else {
 1194           st->print("movss   %s, [rsp + #%d]\t# spill",
 1195                      Matcher::regName[dst_first],
 1196                      offset);
 1197 #endif
 1198         }
 1199       }
 1200       return 0;
 1201     } else if (dst_first_rc == rc_kreg) {
 1202       // mem -> kreg
 1203       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1204           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1205         // 64-bit
 1206         int offset = ra_->reg2offset(src_first);
 1207         if (masm) {
 1208           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1209 #ifndef PRODUCT
 1210         } else {
 1211           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1212                      Matcher::regName[dst_first],
 1213                      offset);
 1214 #endif
 1215         }
 1216       }
 1217       return 0;
 1218     }
 1219   } else if (src_first_rc == rc_int) {
 1220     // gpr ->
 1221     if (dst_first_rc == rc_stack) {
 1222       // gpr -> mem
 1223       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1224           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1225         // 64-bit
 1226         int offset = ra_->reg2offset(dst_first);
 1227         if (masm) {
 1228           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1229 #ifndef PRODUCT
 1230         } else {
 1231           st->print("movq    [rsp + #%d], %s\t# spill",
 1232                      offset,
 1233                      Matcher::regName[src_first]);
 1234 #endif
 1235         }
 1236       } else {
 1237         // 32-bit
 1238         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1239         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1240         int offset = ra_->reg2offset(dst_first);
 1241         if (masm) {
 1242           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1243 #ifndef PRODUCT
 1244         } else {
 1245           st->print("movl    [rsp + #%d], %s\t# spill",
 1246                      offset,
 1247                      Matcher::regName[src_first]);
 1248 #endif
 1249         }
 1250       }
 1251       return 0;
 1252     } else if (dst_first_rc == rc_int) {
 1253       // gpr -> gpr
 1254       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1255           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1256         // 64-bit
 1257         if (masm) {
 1258           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1259                   as_Register(Matcher::_regEncode[src_first]));
 1260 #ifndef PRODUCT
 1261         } else {
 1262           st->print("movq    %s, %s\t# spill",
 1263                      Matcher::regName[dst_first],
 1264                      Matcher::regName[src_first]);
 1265 #endif
 1266         }
 1267         return 0;
 1268       } else {
 1269         // 32-bit
 1270         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1271         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1272         if (masm) {
 1273           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1274                   as_Register(Matcher::_regEncode[src_first]));
 1275 #ifndef PRODUCT
 1276         } else {
 1277           st->print("movl    %s, %s\t# spill",
 1278                      Matcher::regName[dst_first],
 1279                      Matcher::regName[src_first]);
 1280 #endif
 1281         }
 1282         return 0;
 1283       }
 1284     } else if (dst_first_rc == rc_float) {
 1285       // gpr -> xmm
 1286       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1287           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1288         // 64-bit
 1289         if (masm) {
 1290           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1291 #ifndef PRODUCT
 1292         } else {
 1293           st->print("movdq   %s, %s\t# spill",
 1294                      Matcher::regName[dst_first],
 1295                      Matcher::regName[src_first]);
 1296 #endif
 1297         }
 1298       } else {
 1299         // 32-bit
 1300         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1301         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1302         if (masm) {
 1303           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1304 #ifndef PRODUCT
 1305         } else {
 1306           st->print("movdl   %s, %s\t# spill",
 1307                      Matcher::regName[dst_first],
 1308                      Matcher::regName[src_first]);
 1309 #endif
 1310         }
 1311       }
 1312       return 0;
 1313     } else if (dst_first_rc == rc_kreg) {
 1314       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1315           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1316         // 64-bit
 1317         if (masm) {
 1318           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1319   #ifndef PRODUCT
 1320         } else {
 1321            st->print("kmovq   %s, %s\t# spill",
 1322                        Matcher::regName[dst_first],
 1323                        Matcher::regName[src_first]);
 1324   #endif
 1325         }
 1326       }
 1327       Unimplemented();
 1328       return 0;
 1329     }
 1330   } else if (src_first_rc == rc_float) {
 1331     // xmm ->
 1332     if (dst_first_rc == rc_stack) {
 1333       // xmm -> mem
 1334       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1335           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1336         // 64-bit
 1337         int offset = ra_->reg2offset(dst_first);
 1338         if (masm) {
 1339           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1340 #ifndef PRODUCT
 1341         } else {
 1342           st->print("movsd   [rsp + #%d], %s\t# spill",
 1343                      offset,
 1344                      Matcher::regName[src_first]);
 1345 #endif
 1346         }
 1347       } else {
 1348         // 32-bit
 1349         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1350         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1351         int offset = ra_->reg2offset(dst_first);
 1352         if (masm) {
 1353           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1354 #ifndef PRODUCT
 1355         } else {
 1356           st->print("movss   [rsp + #%d], %s\t# spill",
 1357                      offset,
 1358                      Matcher::regName[src_first]);
 1359 #endif
 1360         }
 1361       }
 1362       return 0;
 1363     } else if (dst_first_rc == rc_int) {
 1364       // xmm -> gpr
 1365       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1366           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1367         // 64-bit
 1368         if (masm) {
 1369           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1370 #ifndef PRODUCT
 1371         } else {
 1372           st->print("movdq   %s, %s\t# spill",
 1373                      Matcher::regName[dst_first],
 1374                      Matcher::regName[src_first]);
 1375 #endif
 1376         }
 1377       } else {
 1378         // 32-bit
 1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1381         if (masm) {
 1382           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1383 #ifndef PRODUCT
 1384         } else {
 1385           st->print("movdl   %s, %s\t# spill",
 1386                      Matcher::regName[dst_first],
 1387                      Matcher::regName[src_first]);
 1388 #endif
 1389         }
 1390       }
 1391       return 0;
 1392     } else if (dst_first_rc == rc_float) {
 1393       // xmm -> xmm
 1394       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1395           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1396         // 64-bit
 1397         if (masm) {
 1398           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1399 #ifndef PRODUCT
 1400         } else {
 1401           st->print("%s  %s, %s\t# spill",
 1402                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1403                      Matcher::regName[dst_first],
 1404                      Matcher::regName[src_first]);
 1405 #endif
 1406         }
 1407       } else {
 1408         // 32-bit
 1409         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1410         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1411         if (masm) {
 1412           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1413 #ifndef PRODUCT
 1414         } else {
 1415           st->print("%s  %s, %s\t# spill",
 1416                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1417                      Matcher::regName[dst_first],
 1418                      Matcher::regName[src_first]);
 1419 #endif
 1420         }
 1421       }
 1422       return 0;
 1423     } else if (dst_first_rc == rc_kreg) {
 1424       assert(false, "Illegal spilling");
 1425       return 0;
 1426     }
 1427   } else if (src_first_rc == rc_kreg) {
 1428     if (dst_first_rc == rc_stack) {
 1429       // mem -> kreg
 1430       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1431           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1432         // 64-bit
 1433         int offset = ra_->reg2offset(dst_first);
 1434         if (masm) {
 1435           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1436 #ifndef PRODUCT
 1437         } else {
 1438           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1439                      offset,
 1440                      Matcher::regName[src_first]);
 1441 #endif
 1442         }
 1443       }
 1444       return 0;
 1445     } else if (dst_first_rc == rc_int) {
 1446       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1447           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1448         // 64-bit
 1449         if (masm) {
 1450           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1451 #ifndef PRODUCT
 1452         } else {
 1453          st->print("kmovq   %s, %s\t# spill",
 1454                      Matcher::regName[dst_first],
 1455                      Matcher::regName[src_first]);
 1456 #endif
 1457         }
 1458       }
 1459       Unimplemented();
 1460       return 0;
 1461     } else if (dst_first_rc == rc_kreg) {
 1462       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1463           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1464         // 64-bit
 1465         if (masm) {
 1466           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1467 #ifndef PRODUCT
 1468         } else {
 1469          st->print("kmovq   %s, %s\t# spill",
 1470                      Matcher::regName[dst_first],
 1471                      Matcher::regName[src_first]);
 1472 #endif
 1473         }
 1474       }
 1475       return 0;
 1476     } else if (dst_first_rc == rc_float) {
 1477       assert(false, "Illegal spill");
 1478       return 0;
 1479     }
 1480   }
 1481 
 1482   assert(0," foo ");
 1483   Unimplemented();
 1484   return 0;
 1485 }
 1486 
 1487 #ifndef PRODUCT
 1488 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1489   implementation(nullptr, ra_, false, st);
 1490 }
 1491 #endif
 1492 
 1493 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1494   implementation(masm, ra_, false, nullptr);
 1495 }
 1496 
 1497 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1498   return MachNode::size(ra_);
 1499 }
 1500 
 1501 //=============================================================================
 1502 #ifndef PRODUCT
 1503 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1504 {
 1505   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1506   int reg = ra_->get_reg_first(this);
 1507   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1508             Matcher::regName[reg], offset);
 1509 }
 1510 #endif
 1511 
 1512 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1513 {
 1514   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1515   int reg = ra_->get_encode(this);
 1516 
 1517   __ lea(as_Register(reg), Address(rsp, offset));
 1518 }
 1519 
 1520 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1521 {
 1522   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1523   if (ra_->get_encode(this) > 15) {
 1524     return (offset < 0x80) ? 6 : 9; // REX2
 1525   } else {
 1526     return (offset < 0x80) ? 5 : 8; // REX
 1527   }
 1528 }
 1529 
 1530 //=============================================================================
 1531 #ifndef PRODUCT
 1532 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1533 {
 1534   st->print_cr("MachVEPNode");
 1535 }
 1536 #endif
 1537 
 1538 void MachVEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1539 {
 1540   CodeBuffer* cbuf = masm->code();
 1541   uint insts_size = cbuf->insts_size();
 1542   if (!_verified) {
 1543     __ ic_check(1);
 1544   } else {
 1545     // TODO 8284443 Avoid creation of temporary frame
 1546     if (ra_->C->stub_function() == nullptr) {
 1547       __ verified_entry(ra_->C, 0);
 1548       __ entry_barrier();
 1549       int initial_framesize = ra_->C->output()->frame_size_in_bytes() - 2*wordSize;
 1550       __ remove_frame(initial_framesize, false);
 1551     }
 1552     // Unpack inline type args passed as oop and then jump to
 1553     // the verified entry point (skipping the unverified entry).
 1554     int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
 1555     // Emit code for verified entry and save increment for stack repair on return
 1556     __ verified_entry(ra_->C, sp_inc);
 1557     if (Compile::current()->output()->in_scratch_emit_size()) {
 1558       Label dummy_verified_entry;
 1559       __ jmp(dummy_verified_entry);
 1560     } else {
 1561       __ jmp(*_verified_entry);
 1562     }
 1563   }
 1564   /* WARNING these NOPs are critical so that verified entry point is properly
 1565      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
 1566   int nops_cnt = 4 - ((cbuf->insts_size() - insts_size) & 0x3);
 1567   nops_cnt &= 0x3; // Do not add nops if code is aligned.
 1568   if (nops_cnt > 0) {
 1569     __ nop(nops_cnt);
 1570   }
 1571 }
 1572 
 1573 //=============================================================================
 1574 #ifndef PRODUCT
 1575 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1576 {
 1577   if (UseCompressedClassPointers) {
 1578     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1579     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1580   } else {
 1581     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1582     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1583   }
 1584   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1585 }
 1586 #endif
 1587 
 1588 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1589 {
 1590   __ ic_check(InteriorEntryAlignment);
 1591 }
 1592 
 1593 //=============================================================================
 1594 
 1595 bool Matcher::supports_vector_calling_convention(void) {
 1596   if (EnableVectorSupport && UseVectorStubs) {
 1597     return true;
 1598   }
 1599   return false;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport && UseVectorStubs, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     debug_only(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     debug_only(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     // No relocation needed
 1863     __ mov64(r10, (int64_t) $meth$$method);
 1864     __ call(r10);
 1865     __ post_call_nop();
 1866   %}
 1867 
 1868   enc_class Java_Static_Call(method meth)
 1869   %{
 1870     // JAVA STATIC CALL
 1871     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1872     // determine who we intended to call.
 1873     if (!_method) {
 1874       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1875     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1876       // The NOP here is purely to ensure that eliding a call to
 1877       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1878       __ addr_nop_5();
 1879       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1880     } else {
 1881       int method_index = resolved_method_index(masm);
 1882       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1883                                                   : static_call_Relocation::spec(method_index);
 1884       address mark = __ pc();
 1885       int call_offset = __ offset();
 1886       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1887       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1888         // Calls of the same statically bound method can share
 1889         // a stub to the interpreter.
 1890         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1891       } else {
 1892         // Emit stubs for static call.
 1893         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1894         __ clear_inst_mark();
 1895         if (stub == nullptr) {
 1896           ciEnv::current()->record_failure("CodeCache is full");
 1897           return;
 1898         }
 1899       }
 1900     }
 1901     __ post_call_nop();
 1902   %}
 1903 
 1904   enc_class Java_Dynamic_Call(method meth) %{
 1905     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1906     __ post_call_nop();
 1907   %}
 1908 
 1909 %}
 1910 
 1911 
 1912 
 1913 //----------FRAME--------------------------------------------------------------
 1914 // Definition of frame structure and management information.
 1915 //
 1916 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1917 //                             |   (to get allocators register number
 1918 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1919 //  r   CALLER     |        |
 1920 //  o     |        +--------+      pad to even-align allocators stack-slot
 1921 //  w     V        |  pad0  |        numbers; owned by CALLER
 1922 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1923 //  h     ^        |   in   |  5
 1924 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1925 //  |     |        |        |  3
 1926 //  |     |        +--------+
 1927 //  V     |        | old out|      Empty on Intel, window on Sparc
 1928 //        |    old |preserve|      Must be even aligned.
 1929 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1930 //        |        |   in   |  3   area for Intel ret address
 1931 //     Owned by    |preserve|      Empty on Sparc.
 1932 //       SELF      +--------+
 1933 //        |        |  pad2  |  2   pad to align old SP
 1934 //        |        +--------+  1
 1935 //        |        | locks  |  0
 1936 //        |        +--------+----> OptoReg::stack0(), even aligned
 1937 //        |        |  pad1  | 11   pad to align new SP
 1938 //        |        +--------+
 1939 //        |        |        | 10
 1940 //        |        | spills |  9   spills
 1941 //        V        |        |  8   (pad0 slot for callee)
 1942 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1943 //        ^        |  out   |  7
 1944 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1945 //     Owned by    +--------+
 1946 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1947 //        |    new |preserve|      Must be even-aligned.
 1948 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1949 //        |        |        |
 1950 //
 1951 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1952 //         known from SELF's arguments and the Java calling convention.
 1953 //         Region 6-7 is determined per call site.
 1954 // Note 2: If the calling convention leaves holes in the incoming argument
 1955 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1956 //         are owned by the CALLEE.  Holes should not be necessary in the
 1957 //         incoming area, as the Java calling convention is completely under
 1958 //         the control of the AD file.  Doubles can be sorted and packed to
 1959 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1960 //         varargs C calling conventions.
 1961 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1962 //         even aligned with pad0 as needed.
 1963 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1964 //         region 6-11 is even aligned; it may be padded out more so that
 1965 //         the region from SP to FP meets the minimum stack alignment.
 1966 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1967 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1968 //         SP meets the minimum alignment.
 1969 
 1970 frame
 1971 %{
 1972   // These three registers define part of the calling convention
 1973   // between compiled code and the interpreter.
 1974   inline_cache_reg(RAX);                // Inline Cache Register
 1975 
 1976   // Optional: name the operand used by cisc-spilling to access
 1977   // [stack_pointer + offset]
 1978   cisc_spilling_operand_name(indOffset32);
 1979 
 1980   // Number of stack slots consumed by locking an object
 1981   sync_stack_slots(2);
 1982 
 1983   // Compiled code's Frame Pointer
 1984   frame_pointer(RSP);
 1985 
 1986   // Interpreter stores its frame pointer in a register which is
 1987   // stored to the stack by I2CAdaptors.
 1988   // I2CAdaptors convert from interpreted java to compiled java.
 1989   interpreter_frame_pointer(RBP);
 1990 
 1991   // Stack alignment requirement
 1992   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1993 
 1994   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1995   // for calls to C.  Supports the var-args backing area for register parms.
 1996   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1997 
 1998   // The after-PROLOG location of the return address.  Location of
 1999   // return address specifies a type (REG or STACK) and a number
 2000   // representing the register number (i.e. - use a register name) or
 2001   // stack slot.
 2002   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2003   // Otherwise, it is above the locks and verification slot and alignment word
 2004   return_addr(STACK - 2 +
 2005               align_up((Compile::current()->in_preserve_stack_slots() +
 2006                         Compile::current()->fixed_slots()),
 2007                        stack_alignment_in_slots()));
 2008 
 2009   // Location of compiled Java return values.  Same as C for now.
 2010   return_value
 2011   %{
 2012     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2013            "only return normal values");
 2014 
 2015     static const int lo[Op_RegL + 1] = {
 2016       0,
 2017       0,
 2018       RAX_num,  // Op_RegN
 2019       RAX_num,  // Op_RegI
 2020       RAX_num,  // Op_RegP
 2021       XMM0_num, // Op_RegF
 2022       XMM0_num, // Op_RegD
 2023       RAX_num   // Op_RegL
 2024     };
 2025     static const int hi[Op_RegL + 1] = {
 2026       0,
 2027       0,
 2028       OptoReg::Bad, // Op_RegN
 2029       OptoReg::Bad, // Op_RegI
 2030       RAX_H_num,    // Op_RegP
 2031       OptoReg::Bad, // Op_RegF
 2032       XMM0b_num,    // Op_RegD
 2033       RAX_H_num     // Op_RegL
 2034     };
 2035     // Excluded flags and vector registers.
 2036     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2037     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2038   %}
 2039 %}
 2040 
 2041 //----------ATTRIBUTES---------------------------------------------------------
 2042 //----------Operand Attributes-------------------------------------------------
 2043 op_attrib op_cost(0);        // Required cost attribute
 2044 
 2045 //----------Instruction Attributes---------------------------------------------
 2046 ins_attrib ins_cost(100);       // Required cost attribute
 2047 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2048 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2049                                 // a non-matching short branch variant
 2050                                 // of some long branch?
 2051 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2052                                 // be a power of 2) specifies the
 2053                                 // alignment that some part of the
 2054                                 // instruction (not necessarily the
 2055                                 // start) requires.  If > 1, a
 2056                                 // compute_padding() function must be
 2057                                 // provided for the instruction
 2058 
 2059 //----------OPERANDS-----------------------------------------------------------
 2060 // Operand definitions must precede instruction definitions for correct parsing
 2061 // in the ADLC because operands constitute user defined types which are used in
 2062 // instruction definitions.
 2063 
 2064 //----------Simple Operands----------------------------------------------------
 2065 // Immediate Operands
 2066 // Integer Immediate
 2067 operand immI()
 2068 %{
 2069   match(ConI);
 2070 
 2071   op_cost(10);
 2072   format %{ %}
 2073   interface(CONST_INTER);
 2074 %}
 2075 
 2076 // Constant for test vs zero
 2077 operand immI_0()
 2078 %{
 2079   predicate(n->get_int() == 0);
 2080   match(ConI);
 2081 
 2082   op_cost(0);
 2083   format %{ %}
 2084   interface(CONST_INTER);
 2085 %}
 2086 
 2087 // Constant for increment
 2088 operand immI_1()
 2089 %{
 2090   predicate(n->get_int() == 1);
 2091   match(ConI);
 2092 
 2093   op_cost(0);
 2094   format %{ %}
 2095   interface(CONST_INTER);
 2096 %}
 2097 
 2098 // Constant for decrement
 2099 operand immI_M1()
 2100 %{
 2101   predicate(n->get_int() == -1);
 2102   match(ConI);
 2103 
 2104   op_cost(0);
 2105   format %{ %}
 2106   interface(CONST_INTER);
 2107 %}
 2108 
 2109 operand immI_2()
 2110 %{
 2111   predicate(n->get_int() == 2);
 2112   match(ConI);
 2113 
 2114   op_cost(0);
 2115   format %{ %}
 2116   interface(CONST_INTER);
 2117 %}
 2118 
 2119 operand immI_4()
 2120 %{
 2121   predicate(n->get_int() == 4);
 2122   match(ConI);
 2123 
 2124   op_cost(0);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 operand immI_8()
 2130 %{
 2131   predicate(n->get_int() == 8);
 2132   match(ConI);
 2133 
 2134   op_cost(0);
 2135   format %{ %}
 2136   interface(CONST_INTER);
 2137 %}
 2138 
 2139 // Valid scale values for addressing modes
 2140 operand immI2()
 2141 %{
 2142   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2143   match(ConI);
 2144 
 2145   format %{ %}
 2146   interface(CONST_INTER);
 2147 %}
 2148 
 2149 operand immU7()
 2150 %{
 2151   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2152   match(ConI);
 2153 
 2154   op_cost(5);
 2155   format %{ %}
 2156   interface(CONST_INTER);
 2157 %}
 2158 
 2159 operand immI8()
 2160 %{
 2161   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2162   match(ConI);
 2163 
 2164   op_cost(5);
 2165   format %{ %}
 2166   interface(CONST_INTER);
 2167 %}
 2168 
 2169 operand immU8()
 2170 %{
 2171   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2172   match(ConI);
 2173 
 2174   op_cost(5);
 2175   format %{ %}
 2176   interface(CONST_INTER);
 2177 %}
 2178 
 2179 operand immI16()
 2180 %{
 2181   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2182   match(ConI);
 2183 
 2184   op_cost(10);
 2185   format %{ %}
 2186   interface(CONST_INTER);
 2187 %}
 2188 
 2189 // Int Immediate non-negative
 2190 operand immU31()
 2191 %{
 2192   predicate(n->get_int() >= 0);
 2193   match(ConI);
 2194 
 2195   op_cost(0);
 2196   format %{ %}
 2197   interface(CONST_INTER);
 2198 %}
 2199 
 2200 // Pointer Immediate
 2201 operand immP()
 2202 %{
 2203   match(ConP);
 2204 
 2205   op_cost(10);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 // Null Pointer Immediate
 2211 operand immP0()
 2212 %{
 2213   predicate(n->get_ptr() == 0);
 2214   match(ConP);
 2215 
 2216   op_cost(5);
 2217   format %{ %}
 2218   interface(CONST_INTER);
 2219 %}
 2220 
 2221 // Pointer Immediate
 2222 operand immN() %{
 2223   match(ConN);
 2224 
 2225   op_cost(10);
 2226   format %{ %}
 2227   interface(CONST_INTER);
 2228 %}
 2229 
 2230 operand immNKlass() %{
 2231   match(ConNKlass);
 2232 
 2233   op_cost(10);
 2234   format %{ %}
 2235   interface(CONST_INTER);
 2236 %}
 2237 
 2238 // Null Pointer Immediate
 2239 operand immN0() %{
 2240   predicate(n->get_narrowcon() == 0);
 2241   match(ConN);
 2242 
 2243   op_cost(5);
 2244   format %{ %}
 2245   interface(CONST_INTER);
 2246 %}
 2247 
 2248 operand immP31()
 2249 %{
 2250   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2251             && (n->get_ptr() >> 31) == 0);
 2252   match(ConP);
 2253 
 2254   op_cost(5);
 2255   format %{ %}
 2256   interface(CONST_INTER);
 2257 %}
 2258 
 2259 
 2260 // Long Immediate
 2261 operand immL()
 2262 %{
 2263   match(ConL);
 2264 
 2265   op_cost(20);
 2266   format %{ %}
 2267   interface(CONST_INTER);
 2268 %}
 2269 
 2270 // Long Immediate 8-bit
 2271 operand immL8()
 2272 %{
 2273   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2274   match(ConL);
 2275 
 2276   op_cost(5);
 2277   format %{ %}
 2278   interface(CONST_INTER);
 2279 %}
 2280 
 2281 // Long Immediate 32-bit unsigned
 2282 operand immUL32()
 2283 %{
 2284   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2285   match(ConL);
 2286 
 2287   op_cost(10);
 2288   format %{ %}
 2289   interface(CONST_INTER);
 2290 %}
 2291 
 2292 // Long Immediate 32-bit signed
 2293 operand immL32()
 2294 %{
 2295   predicate(n->get_long() == (int) (n->get_long()));
 2296   match(ConL);
 2297 
 2298   op_cost(15);
 2299   format %{ %}
 2300   interface(CONST_INTER);
 2301 %}
 2302 
 2303 operand immL_Pow2()
 2304 %{
 2305   predicate(is_power_of_2((julong)n->get_long()));
 2306   match(ConL);
 2307 
 2308   op_cost(15);
 2309   format %{ %}
 2310   interface(CONST_INTER);
 2311 %}
 2312 
 2313 operand immL_NotPow2()
 2314 %{
 2315   predicate(is_power_of_2((julong)~n->get_long()));
 2316   match(ConL);
 2317 
 2318   op_cost(15);
 2319   format %{ %}
 2320   interface(CONST_INTER);
 2321 %}
 2322 
 2323 // Long Immediate zero
 2324 operand immL0()
 2325 %{
 2326   predicate(n->get_long() == 0L);
 2327   match(ConL);
 2328 
 2329   op_cost(10);
 2330   format %{ %}
 2331   interface(CONST_INTER);
 2332 %}
 2333 
 2334 // Constant for increment
 2335 operand immL1()
 2336 %{
 2337   predicate(n->get_long() == 1);
 2338   match(ConL);
 2339 
 2340   format %{ %}
 2341   interface(CONST_INTER);
 2342 %}
 2343 
 2344 // Constant for decrement
 2345 operand immL_M1()
 2346 %{
 2347   predicate(n->get_long() == -1);
 2348   match(ConL);
 2349 
 2350   format %{ %}
 2351   interface(CONST_INTER);
 2352 %}
 2353 
 2354 // Long Immediate: low 32-bit mask
 2355 operand immL_32bits()
 2356 %{
 2357   predicate(n->get_long() == 0xFFFFFFFFL);
 2358   match(ConL);
 2359   op_cost(20);
 2360 
 2361   format %{ %}
 2362   interface(CONST_INTER);
 2363 %}
 2364 
 2365 // Int Immediate: 2^n-1, positive
 2366 operand immI_Pow2M1()
 2367 %{
 2368   predicate((n->get_int() > 0)
 2369             && is_power_of_2((juint)n->get_int() + 1));
 2370   match(ConI);
 2371 
 2372   op_cost(20);
 2373   format %{ %}
 2374   interface(CONST_INTER);
 2375 %}
 2376 
 2377 // Float Immediate zero
 2378 operand immF0()
 2379 %{
 2380   predicate(jint_cast(n->getf()) == 0);
 2381   match(ConF);
 2382 
 2383   op_cost(5);
 2384   format %{ %}
 2385   interface(CONST_INTER);
 2386 %}
 2387 
 2388 // Float Immediate
 2389 operand immF()
 2390 %{
 2391   match(ConF);
 2392 
 2393   op_cost(15);
 2394   format %{ %}
 2395   interface(CONST_INTER);
 2396 %}
 2397 
 2398 // Half Float Immediate
 2399 operand immH()
 2400 %{
 2401   match(ConH);
 2402 
 2403   op_cost(15);
 2404   format %{ %}
 2405   interface(CONST_INTER);
 2406 %}
 2407 
 2408 // Double Immediate zero
 2409 operand immD0()
 2410 %{
 2411   predicate(jlong_cast(n->getd()) == 0);
 2412   match(ConD);
 2413 
 2414   op_cost(5);
 2415   format %{ %}
 2416   interface(CONST_INTER);
 2417 %}
 2418 
 2419 // Double Immediate
 2420 operand immD()
 2421 %{
 2422   match(ConD);
 2423 
 2424   op_cost(15);
 2425   format %{ %}
 2426   interface(CONST_INTER);
 2427 %}
 2428 
 2429 // Immediates for special shifts (sign extend)
 2430 
 2431 // Constants for increment
 2432 operand immI_16()
 2433 %{
 2434   predicate(n->get_int() == 16);
 2435   match(ConI);
 2436 
 2437   format %{ %}
 2438   interface(CONST_INTER);
 2439 %}
 2440 
 2441 operand immI_24()
 2442 %{
 2443   predicate(n->get_int() == 24);
 2444   match(ConI);
 2445 
 2446   format %{ %}
 2447   interface(CONST_INTER);
 2448 %}
 2449 
 2450 // Constant for byte-wide masking
 2451 operand immI_255()
 2452 %{
 2453   predicate(n->get_int() == 255);
 2454   match(ConI);
 2455 
 2456   format %{ %}
 2457   interface(CONST_INTER);
 2458 %}
 2459 
 2460 // Constant for short-wide masking
 2461 operand immI_65535()
 2462 %{
 2463   predicate(n->get_int() == 65535);
 2464   match(ConI);
 2465 
 2466   format %{ %}
 2467   interface(CONST_INTER);
 2468 %}
 2469 
 2470 // Constant for byte-wide masking
 2471 operand immL_255()
 2472 %{
 2473   predicate(n->get_long() == 255);
 2474   match(ConL);
 2475 
 2476   format %{ %}
 2477   interface(CONST_INTER);
 2478 %}
 2479 
 2480 // Constant for short-wide masking
 2481 operand immL_65535()
 2482 %{
 2483   predicate(n->get_long() == 65535);
 2484   match(ConL);
 2485 
 2486   format %{ %}
 2487   interface(CONST_INTER);
 2488 %}
 2489 
 2490 operand kReg()
 2491 %{
 2492   constraint(ALLOC_IN_RC(vectmask_reg));
 2493   match(RegVectMask);
 2494   format %{%}
 2495   interface(REG_INTER);
 2496 %}
 2497 
 2498 // Register Operands
 2499 // Integer Register
 2500 operand rRegI()
 2501 %{
 2502   constraint(ALLOC_IN_RC(int_reg));
 2503   match(RegI);
 2504 
 2505   match(rax_RegI);
 2506   match(rbx_RegI);
 2507   match(rcx_RegI);
 2508   match(rdx_RegI);
 2509   match(rdi_RegI);
 2510 
 2511   format %{ %}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 // Special Registers
 2516 operand rax_RegI()
 2517 %{
 2518   constraint(ALLOC_IN_RC(int_rax_reg));
 2519   match(RegI);
 2520   match(rRegI);
 2521 
 2522   format %{ "RAX" %}
 2523   interface(REG_INTER);
 2524 %}
 2525 
 2526 // Special Registers
 2527 operand rbx_RegI()
 2528 %{
 2529   constraint(ALLOC_IN_RC(int_rbx_reg));
 2530   match(RegI);
 2531   match(rRegI);
 2532 
 2533   format %{ "RBX" %}
 2534   interface(REG_INTER);
 2535 %}
 2536 
 2537 operand rcx_RegI()
 2538 %{
 2539   constraint(ALLOC_IN_RC(int_rcx_reg));
 2540   match(RegI);
 2541   match(rRegI);
 2542 
 2543   format %{ "RCX" %}
 2544   interface(REG_INTER);
 2545 %}
 2546 
 2547 operand rdx_RegI()
 2548 %{
 2549   constraint(ALLOC_IN_RC(int_rdx_reg));
 2550   match(RegI);
 2551   match(rRegI);
 2552 
 2553   format %{ "RDX" %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 operand rdi_RegI()
 2558 %{
 2559   constraint(ALLOC_IN_RC(int_rdi_reg));
 2560   match(RegI);
 2561   match(rRegI);
 2562 
 2563   format %{ "RDI" %}
 2564   interface(REG_INTER);
 2565 %}
 2566 
 2567 operand no_rax_rdx_RegI()
 2568 %{
 2569   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2570   match(RegI);
 2571   match(rbx_RegI);
 2572   match(rcx_RegI);
 2573   match(rdi_RegI);
 2574 
 2575   format %{ %}
 2576   interface(REG_INTER);
 2577 %}
 2578 
 2579 operand no_rbp_r13_RegI()
 2580 %{
 2581   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2582   match(RegI);
 2583   match(rRegI);
 2584   match(rax_RegI);
 2585   match(rbx_RegI);
 2586   match(rcx_RegI);
 2587   match(rdx_RegI);
 2588   match(rdi_RegI);
 2589 
 2590   format %{ %}
 2591   interface(REG_INTER);
 2592 %}
 2593 
 2594 // Pointer Register
 2595 operand any_RegP()
 2596 %{
 2597   constraint(ALLOC_IN_RC(any_reg));
 2598   match(RegP);
 2599   match(rax_RegP);
 2600   match(rbx_RegP);
 2601   match(rdi_RegP);
 2602   match(rsi_RegP);
 2603   match(rbp_RegP);
 2604   match(r15_RegP);
 2605   match(rRegP);
 2606 
 2607   format %{ %}
 2608   interface(REG_INTER);
 2609 %}
 2610 
 2611 operand rRegP()
 2612 %{
 2613   constraint(ALLOC_IN_RC(ptr_reg));
 2614   match(RegP);
 2615   match(rax_RegP);
 2616   match(rbx_RegP);
 2617   match(rdi_RegP);
 2618   match(rsi_RegP);
 2619   match(rbp_RegP);  // See Q&A below about
 2620   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2621 
 2622   format %{ %}
 2623   interface(REG_INTER);
 2624 %}
 2625 
 2626 operand rRegN() %{
 2627   constraint(ALLOC_IN_RC(int_reg));
 2628   match(RegN);
 2629 
 2630   format %{ %}
 2631   interface(REG_INTER);
 2632 %}
 2633 
 2634 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2635 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2636 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2637 // The output of an instruction is controlled by the allocator, which respects
 2638 // register class masks, not match rules.  Unless an instruction mentions
 2639 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2640 // by the allocator as an input.
 2641 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2642 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2643 // result, RBP is not included in the output of the instruction either.
 2644 
 2645 // This operand is not allowed to use RBP even if
 2646 // RBP is not used to hold the frame pointer.
 2647 operand no_rbp_RegP()
 2648 %{
 2649   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2650   match(RegP);
 2651   match(rbx_RegP);
 2652   match(rsi_RegP);
 2653   match(rdi_RegP);
 2654 
 2655   format %{ %}
 2656   interface(REG_INTER);
 2657 %}
 2658 
 2659 // Special Registers
 2660 // Return a pointer value
 2661 operand rax_RegP()
 2662 %{
 2663   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2664   match(RegP);
 2665   match(rRegP);
 2666 
 2667   format %{ %}
 2668   interface(REG_INTER);
 2669 %}
 2670 
 2671 // Special Registers
 2672 // Return a compressed pointer value
 2673 operand rax_RegN()
 2674 %{
 2675   constraint(ALLOC_IN_RC(int_rax_reg));
 2676   match(RegN);
 2677   match(rRegN);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 // Used in AtomicAdd
 2684 operand rbx_RegP()
 2685 %{
 2686   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2687   match(RegP);
 2688   match(rRegP);
 2689 
 2690   format %{ %}
 2691   interface(REG_INTER);
 2692 %}
 2693 
 2694 operand rsi_RegP()
 2695 %{
 2696   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2697   match(RegP);
 2698   match(rRegP);
 2699 
 2700   format %{ %}
 2701   interface(REG_INTER);
 2702 %}
 2703 
 2704 operand rbp_RegP()
 2705 %{
 2706   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2707   match(RegP);
 2708   match(rRegP);
 2709 
 2710   format %{ %}
 2711   interface(REG_INTER);
 2712 %}
 2713 
 2714 // Used in rep stosq
 2715 operand rdi_RegP()
 2716 %{
 2717   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2718   match(RegP);
 2719   match(rRegP);
 2720 
 2721   format %{ %}
 2722   interface(REG_INTER);
 2723 %}
 2724 
 2725 operand r15_RegP()
 2726 %{
 2727   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2728   match(RegP);
 2729   match(rRegP);
 2730 
 2731   format %{ %}
 2732   interface(REG_INTER);
 2733 %}
 2734 
 2735 operand rRegL()
 2736 %{
 2737   constraint(ALLOC_IN_RC(long_reg));
 2738   match(RegL);
 2739   match(rax_RegL);
 2740   match(rdx_RegL);
 2741 
 2742   format %{ %}
 2743   interface(REG_INTER);
 2744 %}
 2745 
 2746 // Special Registers
 2747 operand no_rax_rdx_RegL()
 2748 %{
 2749   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2750   match(RegL);
 2751   match(rRegL);
 2752 
 2753   format %{ %}
 2754   interface(REG_INTER);
 2755 %}
 2756 
 2757 operand rax_RegL()
 2758 %{
 2759   constraint(ALLOC_IN_RC(long_rax_reg));
 2760   match(RegL);
 2761   match(rRegL);
 2762 
 2763   format %{ "RAX" %}
 2764   interface(REG_INTER);
 2765 %}
 2766 
 2767 operand rcx_RegL()
 2768 %{
 2769   constraint(ALLOC_IN_RC(long_rcx_reg));
 2770   match(RegL);
 2771   match(rRegL);
 2772 
 2773   format %{ %}
 2774   interface(REG_INTER);
 2775 %}
 2776 
 2777 operand rdx_RegL()
 2778 %{
 2779   constraint(ALLOC_IN_RC(long_rdx_reg));
 2780   match(RegL);
 2781   match(rRegL);
 2782 
 2783   format %{ %}
 2784   interface(REG_INTER);
 2785 %}
 2786 
 2787 operand r11_RegL()
 2788 %{
 2789   constraint(ALLOC_IN_RC(long_r11_reg));
 2790   match(RegL);
 2791   match(rRegL);
 2792 
 2793   format %{ %}
 2794   interface(REG_INTER);
 2795 %}
 2796 
 2797 operand no_rbp_r13_RegL()
 2798 %{
 2799   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2800   match(RegL);
 2801   match(rRegL);
 2802   match(rax_RegL);
 2803   match(rcx_RegL);
 2804   match(rdx_RegL);
 2805 
 2806   format %{ %}
 2807   interface(REG_INTER);
 2808 %}
 2809 
 2810 // Flags register, used as output of compare instructions
 2811 operand rFlagsReg()
 2812 %{
 2813   constraint(ALLOC_IN_RC(int_flags));
 2814   match(RegFlags);
 2815 
 2816   format %{ "RFLAGS" %}
 2817   interface(REG_INTER);
 2818 %}
 2819 
 2820 // Flags register, used as output of FLOATING POINT compare instructions
 2821 operand rFlagsRegU()
 2822 %{
 2823   constraint(ALLOC_IN_RC(int_flags));
 2824   match(RegFlags);
 2825 
 2826   format %{ "RFLAGS_U" %}
 2827   interface(REG_INTER);
 2828 %}
 2829 
 2830 operand rFlagsRegUCF() %{
 2831   constraint(ALLOC_IN_RC(int_flags));
 2832   match(RegFlags);
 2833   predicate(false);
 2834 
 2835   format %{ "RFLAGS_U_CF" %}
 2836   interface(REG_INTER);
 2837 %}
 2838 
 2839 // Float register operands
 2840 operand regF() %{
 2841    constraint(ALLOC_IN_RC(float_reg));
 2842    match(RegF);
 2843 
 2844    format %{ %}
 2845    interface(REG_INTER);
 2846 %}
 2847 
 2848 // Float register operands
 2849 operand legRegF() %{
 2850    constraint(ALLOC_IN_RC(float_reg_legacy));
 2851    match(RegF);
 2852 
 2853    format %{ %}
 2854    interface(REG_INTER);
 2855 %}
 2856 
 2857 // Float register operands
 2858 operand vlRegF() %{
 2859    constraint(ALLOC_IN_RC(float_reg_vl));
 2860    match(RegF);
 2861 
 2862    format %{ %}
 2863    interface(REG_INTER);
 2864 %}
 2865 
 2866 // Double register operands
 2867 operand regD() %{
 2868    constraint(ALLOC_IN_RC(double_reg));
 2869    match(RegD);
 2870 
 2871    format %{ %}
 2872    interface(REG_INTER);
 2873 %}
 2874 
 2875 // Double register operands
 2876 operand legRegD() %{
 2877    constraint(ALLOC_IN_RC(double_reg_legacy));
 2878    match(RegD);
 2879 
 2880    format %{ %}
 2881    interface(REG_INTER);
 2882 %}
 2883 
 2884 // Double register operands
 2885 operand vlRegD() %{
 2886    constraint(ALLOC_IN_RC(double_reg_vl));
 2887    match(RegD);
 2888 
 2889    format %{ %}
 2890    interface(REG_INTER);
 2891 %}
 2892 
 2893 //----------Memory Operands----------------------------------------------------
 2894 // Direct Memory Operand
 2895 // operand direct(immP addr)
 2896 // %{
 2897 //   match(addr);
 2898 
 2899 //   format %{ "[$addr]" %}
 2900 //   interface(MEMORY_INTER) %{
 2901 //     base(0xFFFFFFFF);
 2902 //     index(0x4);
 2903 //     scale(0x0);
 2904 //     disp($addr);
 2905 //   %}
 2906 // %}
 2907 
 2908 // Indirect Memory Operand
 2909 operand indirect(any_RegP reg)
 2910 %{
 2911   constraint(ALLOC_IN_RC(ptr_reg));
 2912   match(reg);
 2913 
 2914   format %{ "[$reg]" %}
 2915   interface(MEMORY_INTER) %{
 2916     base($reg);
 2917     index(0x4);
 2918     scale(0x0);
 2919     disp(0x0);
 2920   %}
 2921 %}
 2922 
 2923 // Indirect Memory Plus Short Offset Operand
 2924 operand indOffset8(any_RegP reg, immL8 off)
 2925 %{
 2926   constraint(ALLOC_IN_RC(ptr_reg));
 2927   match(AddP reg off);
 2928 
 2929   format %{ "[$reg + $off (8-bit)]" %}
 2930   interface(MEMORY_INTER) %{
 2931     base($reg);
 2932     index(0x4);
 2933     scale(0x0);
 2934     disp($off);
 2935   %}
 2936 %}
 2937 
 2938 // Indirect Memory Plus Long Offset Operand
 2939 operand indOffset32(any_RegP reg, immL32 off)
 2940 %{
 2941   constraint(ALLOC_IN_RC(ptr_reg));
 2942   match(AddP reg off);
 2943 
 2944   format %{ "[$reg + $off (32-bit)]" %}
 2945   interface(MEMORY_INTER) %{
 2946     base($reg);
 2947     index(0x4);
 2948     scale(0x0);
 2949     disp($off);
 2950   %}
 2951 %}
 2952 
 2953 // Indirect Memory Plus Index Register Plus Offset Operand
 2954 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2955 %{
 2956   constraint(ALLOC_IN_RC(ptr_reg));
 2957   match(AddP (AddP reg lreg) off);
 2958 
 2959   op_cost(10);
 2960   format %{"[$reg + $off + $lreg]" %}
 2961   interface(MEMORY_INTER) %{
 2962     base($reg);
 2963     index($lreg);
 2964     scale(0x0);
 2965     disp($off);
 2966   %}
 2967 %}
 2968 
 2969 // Indirect Memory Plus Index Register Plus Offset Operand
 2970 operand indIndex(any_RegP reg, rRegL lreg)
 2971 %{
 2972   constraint(ALLOC_IN_RC(ptr_reg));
 2973   match(AddP reg lreg);
 2974 
 2975   op_cost(10);
 2976   format %{"[$reg + $lreg]" %}
 2977   interface(MEMORY_INTER) %{
 2978     base($reg);
 2979     index($lreg);
 2980     scale(0x0);
 2981     disp(0x0);
 2982   %}
 2983 %}
 2984 
 2985 // Indirect Memory Times Scale Plus Index Register
 2986 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2987 %{
 2988   constraint(ALLOC_IN_RC(ptr_reg));
 2989   match(AddP reg (LShiftL lreg scale));
 2990 
 2991   op_cost(10);
 2992   format %{"[$reg + $lreg << $scale]" %}
 2993   interface(MEMORY_INTER) %{
 2994     base($reg);
 2995     index($lreg);
 2996     scale($scale);
 2997     disp(0x0);
 2998   %}
 2999 %}
 3000 
 3001 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3002 %{
 3003   constraint(ALLOC_IN_RC(ptr_reg));
 3004   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3005   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3006 
 3007   op_cost(10);
 3008   format %{"[$reg + pos $idx << $scale]" %}
 3009   interface(MEMORY_INTER) %{
 3010     base($reg);
 3011     index($idx);
 3012     scale($scale);
 3013     disp(0x0);
 3014   %}
 3015 %}
 3016 
 3017 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3018 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3019 %{
 3020   constraint(ALLOC_IN_RC(ptr_reg));
 3021   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3022 
 3023   op_cost(10);
 3024   format %{"[$reg + $off + $lreg << $scale]" %}
 3025   interface(MEMORY_INTER) %{
 3026     base($reg);
 3027     index($lreg);
 3028     scale($scale);
 3029     disp($off);
 3030   %}
 3031 %}
 3032 
 3033 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3034 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3035 %{
 3036   constraint(ALLOC_IN_RC(ptr_reg));
 3037   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3038   match(AddP (AddP reg (ConvI2L idx)) off);
 3039 
 3040   op_cost(10);
 3041   format %{"[$reg + $off + $idx]" %}
 3042   interface(MEMORY_INTER) %{
 3043     base($reg);
 3044     index($idx);
 3045     scale(0x0);
 3046     disp($off);
 3047   %}
 3048 %}
 3049 
 3050 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3051 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3052 %{
 3053   constraint(ALLOC_IN_RC(ptr_reg));
 3054   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3055   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3056 
 3057   op_cost(10);
 3058   format %{"[$reg + $off + $idx << $scale]" %}
 3059   interface(MEMORY_INTER) %{
 3060     base($reg);
 3061     index($idx);
 3062     scale($scale);
 3063     disp($off);
 3064   %}
 3065 %}
 3066 
 3067 // Indirect Narrow Oop Operand
 3068 operand indCompressedOop(rRegN reg) %{
 3069   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3070   constraint(ALLOC_IN_RC(ptr_reg));
 3071   match(DecodeN reg);
 3072 
 3073   op_cost(10);
 3074   format %{"[R12 + $reg << 3] (compressed oop addressing)" %}
 3075   interface(MEMORY_INTER) %{
 3076     base(0xc); // R12
 3077     index($reg);
 3078     scale(0x3);
 3079     disp(0x0);
 3080   %}
 3081 %}
 3082 
 3083 // Indirect Narrow Oop Plus Offset Operand
 3084 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3085 // we can't free r12 even with CompressedOops::base() == nullptr.
 3086 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3087   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3088   constraint(ALLOC_IN_RC(ptr_reg));
 3089   match(AddP (DecodeN reg) off);
 3090 
 3091   op_cost(10);
 3092   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3093   interface(MEMORY_INTER) %{
 3094     base(0xc); // R12
 3095     index($reg);
 3096     scale(0x3);
 3097     disp($off);
 3098   %}
 3099 %}
 3100 
 3101 // Indirect Memory Operand
 3102 operand indirectNarrow(rRegN reg)
 3103 %{
 3104   predicate(CompressedOops::shift() == 0);
 3105   constraint(ALLOC_IN_RC(ptr_reg));
 3106   match(DecodeN reg);
 3107 
 3108   format %{ "[$reg]" %}
 3109   interface(MEMORY_INTER) %{
 3110     base($reg);
 3111     index(0x4);
 3112     scale(0x0);
 3113     disp(0x0);
 3114   %}
 3115 %}
 3116 
 3117 // Indirect Memory Plus Short Offset Operand
 3118 operand indOffset8Narrow(rRegN reg, immL8 off)
 3119 %{
 3120   predicate(CompressedOops::shift() == 0);
 3121   constraint(ALLOC_IN_RC(ptr_reg));
 3122   match(AddP (DecodeN reg) off);
 3123 
 3124   format %{ "[$reg + $off (8-bit)]" %}
 3125   interface(MEMORY_INTER) %{
 3126     base($reg);
 3127     index(0x4);
 3128     scale(0x0);
 3129     disp($off);
 3130   %}
 3131 %}
 3132 
 3133 // Indirect Memory Plus Long Offset Operand
 3134 operand indOffset32Narrow(rRegN reg, immL32 off)
 3135 %{
 3136   predicate(CompressedOops::shift() == 0);
 3137   constraint(ALLOC_IN_RC(ptr_reg));
 3138   match(AddP (DecodeN reg) off);
 3139 
 3140   format %{ "[$reg + $off (32-bit)]" %}
 3141   interface(MEMORY_INTER) %{
 3142     base($reg);
 3143     index(0x4);
 3144     scale(0x0);
 3145     disp($off);
 3146   %}
 3147 %}
 3148 
 3149 // Indirect Memory Plus Index Register Plus Offset Operand
 3150 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3151 %{
 3152   predicate(CompressedOops::shift() == 0);
 3153   constraint(ALLOC_IN_RC(ptr_reg));
 3154   match(AddP (AddP (DecodeN reg) lreg) off);
 3155 
 3156   op_cost(10);
 3157   format %{"[$reg + $off + $lreg]" %}
 3158   interface(MEMORY_INTER) %{
 3159     base($reg);
 3160     index($lreg);
 3161     scale(0x0);
 3162     disp($off);
 3163   %}
 3164 %}
 3165 
 3166 // Indirect Memory Plus Index Register Plus Offset Operand
 3167 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3168 %{
 3169   predicate(CompressedOops::shift() == 0);
 3170   constraint(ALLOC_IN_RC(ptr_reg));
 3171   match(AddP (DecodeN reg) lreg);
 3172 
 3173   op_cost(10);
 3174   format %{"[$reg + $lreg]" %}
 3175   interface(MEMORY_INTER) %{
 3176     base($reg);
 3177     index($lreg);
 3178     scale(0x0);
 3179     disp(0x0);
 3180   %}
 3181 %}
 3182 
 3183 // Indirect Memory Times Scale Plus Index Register
 3184 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3185 %{
 3186   predicate(CompressedOops::shift() == 0);
 3187   constraint(ALLOC_IN_RC(ptr_reg));
 3188   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3189 
 3190   op_cost(10);
 3191   format %{"[$reg + $lreg << $scale]" %}
 3192   interface(MEMORY_INTER) %{
 3193     base($reg);
 3194     index($lreg);
 3195     scale($scale);
 3196     disp(0x0);
 3197   %}
 3198 %}
 3199 
 3200 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3201 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3202 %{
 3203   predicate(CompressedOops::shift() == 0);
 3204   constraint(ALLOC_IN_RC(ptr_reg));
 3205   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3206 
 3207   op_cost(10);
 3208   format %{"[$reg + $off + $lreg << $scale]" %}
 3209   interface(MEMORY_INTER) %{
 3210     base($reg);
 3211     index($lreg);
 3212     scale($scale);
 3213     disp($off);
 3214   %}
 3215 %}
 3216 
 3217 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3218 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3219 %{
 3220   constraint(ALLOC_IN_RC(ptr_reg));
 3221   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3222   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3223 
 3224   op_cost(10);
 3225   format %{"[$reg + $off + $idx]" %}
 3226   interface(MEMORY_INTER) %{
 3227     base($reg);
 3228     index($idx);
 3229     scale(0x0);
 3230     disp($off);
 3231   %}
 3232 %}
 3233 
 3234 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3235 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3236 %{
 3237   constraint(ALLOC_IN_RC(ptr_reg));
 3238   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3239   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3240 
 3241   op_cost(10);
 3242   format %{"[$reg + $off + $idx << $scale]" %}
 3243   interface(MEMORY_INTER) %{
 3244     base($reg);
 3245     index($idx);
 3246     scale($scale);
 3247     disp($off);
 3248   %}
 3249 %}
 3250 
 3251 //----------Special Memory Operands--------------------------------------------
 3252 // Stack Slot Operand - This operand is used for loading and storing temporary
 3253 //                      values on the stack where a match requires a value to
 3254 //                      flow through memory.
 3255 operand stackSlotP(sRegP reg)
 3256 %{
 3257   constraint(ALLOC_IN_RC(stack_slots));
 3258   // No match rule because this operand is only generated in matching
 3259 
 3260   format %{ "[$reg]" %}
 3261   interface(MEMORY_INTER) %{
 3262     base(0x4);   // RSP
 3263     index(0x4);  // No Index
 3264     scale(0x0);  // No Scale
 3265     disp($reg);  // Stack Offset
 3266   %}
 3267 %}
 3268 
 3269 operand stackSlotI(sRegI reg)
 3270 %{
 3271   constraint(ALLOC_IN_RC(stack_slots));
 3272   // No match rule because this operand is only generated in matching
 3273 
 3274   format %{ "[$reg]" %}
 3275   interface(MEMORY_INTER) %{
 3276     base(0x4);   // RSP
 3277     index(0x4);  // No Index
 3278     scale(0x0);  // No Scale
 3279     disp($reg);  // Stack Offset
 3280   %}
 3281 %}
 3282 
 3283 operand stackSlotF(sRegF reg)
 3284 %{
 3285   constraint(ALLOC_IN_RC(stack_slots));
 3286   // No match rule because this operand is only generated in matching
 3287 
 3288   format %{ "[$reg]" %}
 3289   interface(MEMORY_INTER) %{
 3290     base(0x4);   // RSP
 3291     index(0x4);  // No Index
 3292     scale(0x0);  // No Scale
 3293     disp($reg);  // Stack Offset
 3294   %}
 3295 %}
 3296 
 3297 operand stackSlotD(sRegD reg)
 3298 %{
 3299   constraint(ALLOC_IN_RC(stack_slots));
 3300   // No match rule because this operand is only generated in matching
 3301 
 3302   format %{ "[$reg]" %}
 3303   interface(MEMORY_INTER) %{
 3304     base(0x4);   // RSP
 3305     index(0x4);  // No Index
 3306     scale(0x0);  // No Scale
 3307     disp($reg);  // Stack Offset
 3308   %}
 3309 %}
 3310 operand stackSlotL(sRegL reg)
 3311 %{
 3312   constraint(ALLOC_IN_RC(stack_slots));
 3313   // No match rule because this operand is only generated in matching
 3314 
 3315   format %{ "[$reg]" %}
 3316   interface(MEMORY_INTER) %{
 3317     base(0x4);   // RSP
 3318     index(0x4);  // No Index
 3319     scale(0x0);  // No Scale
 3320     disp($reg);  // Stack Offset
 3321   %}
 3322 %}
 3323 
 3324 //----------Conditional Branch Operands----------------------------------------
 3325 // Comparison Op  - This is the operation of the comparison, and is limited to
 3326 //                  the following set of codes:
 3327 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3328 //
 3329 // Other attributes of the comparison, such as unsignedness, are specified
 3330 // by the comparison instruction that sets a condition code flags register.
 3331 // That result is represented by a flags operand whose subtype is appropriate
 3332 // to the unsignedness (etc.) of the comparison.
 3333 //
 3334 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3335 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3336 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3337 
 3338 // Comparison Code
 3339 operand cmpOp()
 3340 %{
 3341   match(Bool);
 3342 
 3343   format %{ "" %}
 3344   interface(COND_INTER) %{
 3345     equal(0x4, "e");
 3346     not_equal(0x5, "ne");
 3347     less(0xC, "l");
 3348     greater_equal(0xD, "ge");
 3349     less_equal(0xE, "le");
 3350     greater(0xF, "g");
 3351     overflow(0x0, "o");
 3352     no_overflow(0x1, "no");
 3353   %}
 3354 %}
 3355 
 3356 // Comparison Code, unsigned compare.  Used by FP also, with
 3357 // C2 (unordered) turned into GT or LT already.  The other bits
 3358 // C0 and C3 are turned into Carry & Zero flags.
 3359 operand cmpOpU()
 3360 %{
 3361   match(Bool);
 3362 
 3363   format %{ "" %}
 3364   interface(COND_INTER) %{
 3365     equal(0x4, "e");
 3366     not_equal(0x5, "ne");
 3367     less(0x2, "b");
 3368     greater_equal(0x3, "ae");
 3369     less_equal(0x6, "be");
 3370     greater(0x7, "a");
 3371     overflow(0x0, "o");
 3372     no_overflow(0x1, "no");
 3373   %}
 3374 %}
 3375 
 3376 
 3377 // Floating comparisons that don't require any fixup for the unordered case,
 3378 // If both inputs of the comparison are the same, ZF is always set so we
 3379 // don't need to use cmpOpUCF2 for eq/ne
 3380 operand cmpOpUCF() %{
 3381   match(Bool);
 3382   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3383             n->as_Bool()->_test._test == BoolTest::ge ||
 3384             n->as_Bool()->_test._test == BoolTest::le ||
 3385             n->as_Bool()->_test._test == BoolTest::gt ||
 3386             n->in(1)->in(1) == n->in(1)->in(2));
 3387   format %{ "" %}
 3388   interface(COND_INTER) %{
 3389     equal(0xb, "np");
 3390     not_equal(0xa, "p");
 3391     less(0x2, "b");
 3392     greater_equal(0x3, "ae");
 3393     less_equal(0x6, "be");
 3394     greater(0x7, "a");
 3395     overflow(0x0, "o");
 3396     no_overflow(0x1, "no");
 3397   %}
 3398 %}
 3399 
 3400 
 3401 // Floating comparisons that can be fixed up with extra conditional jumps
 3402 operand cmpOpUCF2() %{
 3403   match(Bool);
 3404   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3405              n->as_Bool()->_test._test == BoolTest::eq) &&
 3406             n->in(1)->in(1) != n->in(1)->in(2));
 3407   format %{ "" %}
 3408   interface(COND_INTER) %{
 3409     equal(0x4, "e");
 3410     not_equal(0x5, "ne");
 3411     less(0x2, "b");
 3412     greater_equal(0x3, "ae");
 3413     less_equal(0x6, "be");
 3414     greater(0x7, "a");
 3415     overflow(0x0, "o");
 3416     no_overflow(0x1, "no");
 3417   %}
 3418 %}
 3419 
 3420 //----------OPERAND CLASSES----------------------------------------------------
 3421 // Operand Classes are groups of operands that are used as to simplify
 3422 // instruction definitions by not requiring the AD writer to specify separate
 3423 // instructions for every form of operand when the instruction accepts
 3424 // multiple operand types with the same basic encoding and format.  The classic
 3425 // case of this is memory operands.
 3426 
 3427 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3428                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3429                indCompressedOop, indCompressedOopOffset,
 3430                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3431                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3432                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3433 
 3434 //----------PIPELINE-----------------------------------------------------------
 3435 // Rules which define the behavior of the target architectures pipeline.
 3436 pipeline %{
 3437 
 3438 //----------ATTRIBUTES---------------------------------------------------------
 3439 attributes %{
 3440   variable_size_instructions;        // Fixed size instructions
 3441   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3442   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3443   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3444   instruction_fetch_units = 1;       // of 16 bytes
 3445 
 3446   // List of nop instructions
 3447   nops( MachNop );
 3448 %}
 3449 
 3450 //----------RESOURCES----------------------------------------------------------
 3451 // Resources are the functional units available to the machine
 3452 
 3453 // Generic P2/P3 pipeline
 3454 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3455 // 3 instructions decoded per cycle.
 3456 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3457 // 3 ALU op, only ALU0 handles mul instructions.
 3458 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3459            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3460            BR, FPU,
 3461            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3462 
 3463 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3464 // Pipeline Description specifies the stages in the machine's pipeline
 3465 
 3466 // Generic P2/P3 pipeline
 3467 pipe_desc(S0, S1, S2, S3, S4, S5);
 3468 
 3469 //----------PIPELINE CLASSES---------------------------------------------------
 3470 // Pipeline Classes describe the stages in which input and output are
 3471 // referenced by the hardware pipeline.
 3472 
 3473 // Naming convention: ialu or fpu
 3474 // Then: _reg
 3475 // Then: _reg if there is a 2nd register
 3476 // Then: _long if it's a pair of instructions implementing a long
 3477 // Then: _fat if it requires the big decoder
 3478 //   Or: _mem if it requires the big decoder and a memory unit.
 3479 
 3480 // Integer ALU reg operation
 3481 pipe_class ialu_reg(rRegI dst)
 3482 %{
 3483     single_instruction;
 3484     dst    : S4(write);
 3485     dst    : S3(read);
 3486     DECODE : S0;        // any decoder
 3487     ALU    : S3;        // any alu
 3488 %}
 3489 
 3490 // Long ALU reg operation
 3491 pipe_class ialu_reg_long(rRegL dst)
 3492 %{
 3493     instruction_count(2);
 3494     dst    : S4(write);
 3495     dst    : S3(read);
 3496     DECODE : S0(2);     // any 2 decoders
 3497     ALU    : S3(2);     // both alus
 3498 %}
 3499 
 3500 // Integer ALU reg operation using big decoder
 3501 pipe_class ialu_reg_fat(rRegI dst)
 3502 %{
 3503     single_instruction;
 3504     dst    : S4(write);
 3505     dst    : S3(read);
 3506     D0     : S0;        // big decoder only
 3507     ALU    : S3;        // any alu
 3508 %}
 3509 
 3510 // Integer ALU reg-reg operation
 3511 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3512 %{
 3513     single_instruction;
 3514     dst    : S4(write);
 3515     src    : S3(read);
 3516     DECODE : S0;        // any decoder
 3517     ALU    : S3;        // any alu
 3518 %}
 3519 
 3520 // Integer ALU reg-reg operation
 3521 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3522 %{
 3523     single_instruction;
 3524     dst    : S4(write);
 3525     src    : S3(read);
 3526     D0     : S0;        // big decoder only
 3527     ALU    : S3;        // any alu
 3528 %}
 3529 
 3530 // Integer ALU reg-mem operation
 3531 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3532 %{
 3533     single_instruction;
 3534     dst    : S5(write);
 3535     mem    : S3(read);
 3536     D0     : S0;        // big decoder only
 3537     ALU    : S4;        // any alu
 3538     MEM    : S3;        // any mem
 3539 %}
 3540 
 3541 // Integer mem operation (prefetch)
 3542 pipe_class ialu_mem(memory mem)
 3543 %{
 3544     single_instruction;
 3545     mem    : S3(read);
 3546     D0     : S0;        // big decoder only
 3547     MEM    : S3;        // any mem
 3548 %}
 3549 
 3550 // Integer Store to Memory
 3551 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3552 %{
 3553     single_instruction;
 3554     mem    : S3(read);
 3555     src    : S5(read);
 3556     D0     : S0;        // big decoder only
 3557     ALU    : S4;        // any alu
 3558     MEM    : S3;
 3559 %}
 3560 
 3561 // // Long Store to Memory
 3562 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3563 // %{
 3564 //     instruction_count(2);
 3565 //     mem    : S3(read);
 3566 //     src    : S5(read);
 3567 //     D0     : S0(2);          // big decoder only; twice
 3568 //     ALU    : S4(2);     // any 2 alus
 3569 //     MEM    : S3(2);  // Both mems
 3570 // %}
 3571 
 3572 // Integer Store to Memory
 3573 pipe_class ialu_mem_imm(memory mem)
 3574 %{
 3575     single_instruction;
 3576     mem    : S3(read);
 3577     D0     : S0;        // big decoder only
 3578     ALU    : S4;        // any alu
 3579     MEM    : S3;
 3580 %}
 3581 
 3582 // Integer ALU0 reg-reg operation
 3583 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3584 %{
 3585     single_instruction;
 3586     dst    : S4(write);
 3587     src    : S3(read);
 3588     D0     : S0;        // Big decoder only
 3589     ALU0   : S3;        // only alu0
 3590 %}
 3591 
 3592 // Integer ALU0 reg-mem operation
 3593 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3594 %{
 3595     single_instruction;
 3596     dst    : S5(write);
 3597     mem    : S3(read);
 3598     D0     : S0;        // big decoder only
 3599     ALU0   : S4;        // ALU0 only
 3600     MEM    : S3;        // any mem
 3601 %}
 3602 
 3603 // Integer ALU reg-reg operation
 3604 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3605 %{
 3606     single_instruction;
 3607     cr     : S4(write);
 3608     src1   : S3(read);
 3609     src2   : S3(read);
 3610     DECODE : S0;        // any decoder
 3611     ALU    : S3;        // any alu
 3612 %}
 3613 
 3614 // Integer ALU reg-imm operation
 3615 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3616 %{
 3617     single_instruction;
 3618     cr     : S4(write);
 3619     src1   : S3(read);
 3620     DECODE : S0;        // any decoder
 3621     ALU    : S3;        // any alu
 3622 %}
 3623 
 3624 // Integer ALU reg-mem operation
 3625 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3626 %{
 3627     single_instruction;
 3628     cr     : S4(write);
 3629     src1   : S3(read);
 3630     src2   : S3(read);
 3631     D0     : S0;        // big decoder only
 3632     ALU    : S4;        // any alu
 3633     MEM    : S3;
 3634 %}
 3635 
 3636 // Conditional move reg-reg
 3637 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3638 %{
 3639     instruction_count(4);
 3640     y      : S4(read);
 3641     q      : S3(read);
 3642     p      : S3(read);
 3643     DECODE : S0(4);     // any decoder
 3644 %}
 3645 
 3646 // Conditional move reg-reg
 3647 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3648 %{
 3649     single_instruction;
 3650     dst    : S4(write);
 3651     src    : S3(read);
 3652     cr     : S3(read);
 3653     DECODE : S0;        // any decoder
 3654 %}
 3655 
 3656 // Conditional move reg-mem
 3657 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3658 %{
 3659     single_instruction;
 3660     dst    : S4(write);
 3661     src    : S3(read);
 3662     cr     : S3(read);
 3663     DECODE : S0;        // any decoder
 3664     MEM    : S3;
 3665 %}
 3666 
 3667 // Conditional move reg-reg long
 3668 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3669 %{
 3670     single_instruction;
 3671     dst    : S4(write);
 3672     src    : S3(read);
 3673     cr     : S3(read);
 3674     DECODE : S0(2);     // any 2 decoders
 3675 %}
 3676 
 3677 // Float reg-reg operation
 3678 pipe_class fpu_reg(regD dst)
 3679 %{
 3680     instruction_count(2);
 3681     dst    : S3(read);
 3682     DECODE : S0(2);     // any 2 decoders
 3683     FPU    : S3;
 3684 %}
 3685 
 3686 // Float reg-reg operation
 3687 pipe_class fpu_reg_reg(regD dst, regD src)
 3688 %{
 3689     instruction_count(2);
 3690     dst    : S4(write);
 3691     src    : S3(read);
 3692     DECODE : S0(2);     // any 2 decoders
 3693     FPU    : S3;
 3694 %}
 3695 
 3696 // Float reg-reg operation
 3697 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3698 %{
 3699     instruction_count(3);
 3700     dst    : S4(write);
 3701     src1   : S3(read);
 3702     src2   : S3(read);
 3703     DECODE : S0(3);     // any 3 decoders
 3704     FPU    : S3(2);
 3705 %}
 3706 
 3707 // Float reg-reg operation
 3708 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3709 %{
 3710     instruction_count(4);
 3711     dst    : S4(write);
 3712     src1   : S3(read);
 3713     src2   : S3(read);
 3714     src3   : S3(read);
 3715     DECODE : S0(4);     // any 3 decoders
 3716     FPU    : S3(2);
 3717 %}
 3718 
 3719 // Float reg-reg operation
 3720 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3721 %{
 3722     instruction_count(4);
 3723     dst    : S4(write);
 3724     src1   : S3(read);
 3725     src2   : S3(read);
 3726     src3   : S3(read);
 3727     DECODE : S1(3);     // any 3 decoders
 3728     D0     : S0;        // Big decoder only
 3729     FPU    : S3(2);
 3730     MEM    : S3;
 3731 %}
 3732 
 3733 // Float reg-mem operation
 3734 pipe_class fpu_reg_mem(regD dst, memory mem)
 3735 %{
 3736     instruction_count(2);
 3737     dst    : S5(write);
 3738     mem    : S3(read);
 3739     D0     : S0;        // big decoder only
 3740     DECODE : S1;        // any decoder for FPU POP
 3741     FPU    : S4;
 3742     MEM    : S3;        // any mem
 3743 %}
 3744 
 3745 // Float reg-mem operation
 3746 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3747 %{
 3748     instruction_count(3);
 3749     dst    : S5(write);
 3750     src1   : S3(read);
 3751     mem    : S3(read);
 3752     D0     : S0;        // big decoder only
 3753     DECODE : S1(2);     // any decoder for FPU POP
 3754     FPU    : S4;
 3755     MEM    : S3;        // any mem
 3756 %}
 3757 
 3758 // Float mem-reg operation
 3759 pipe_class fpu_mem_reg(memory mem, regD src)
 3760 %{
 3761     instruction_count(2);
 3762     src    : S5(read);
 3763     mem    : S3(read);
 3764     DECODE : S0;        // any decoder for FPU PUSH
 3765     D0     : S1;        // big decoder only
 3766     FPU    : S4;
 3767     MEM    : S3;        // any mem
 3768 %}
 3769 
 3770 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3771 %{
 3772     instruction_count(3);
 3773     src1   : S3(read);
 3774     src2   : S3(read);
 3775     mem    : S3(read);
 3776     DECODE : S0(2);     // any decoder for FPU PUSH
 3777     D0     : S1;        // big decoder only
 3778     FPU    : S4;
 3779     MEM    : S3;        // any mem
 3780 %}
 3781 
 3782 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3783 %{
 3784     instruction_count(3);
 3785     src1   : S3(read);
 3786     src2   : S3(read);
 3787     mem    : S4(read);
 3788     DECODE : S0;        // any decoder for FPU PUSH
 3789     D0     : S0(2);     // big decoder only
 3790     FPU    : S4;
 3791     MEM    : S3(2);     // any mem
 3792 %}
 3793 
 3794 pipe_class fpu_mem_mem(memory dst, memory src1)
 3795 %{
 3796     instruction_count(2);
 3797     src1   : S3(read);
 3798     dst    : S4(read);
 3799     D0     : S0(2);     // big decoder only
 3800     MEM    : S3(2);     // any mem
 3801 %}
 3802 
 3803 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3804 %{
 3805     instruction_count(3);
 3806     src1   : S3(read);
 3807     src2   : S3(read);
 3808     dst    : S4(read);
 3809     D0     : S0(3);     // big decoder only
 3810     FPU    : S4;
 3811     MEM    : S3(3);     // any mem
 3812 %}
 3813 
 3814 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3815 %{
 3816     instruction_count(3);
 3817     src1   : S4(read);
 3818     mem    : S4(read);
 3819     DECODE : S0;        // any decoder for FPU PUSH
 3820     D0     : S0(2);     // big decoder only
 3821     FPU    : S4;
 3822     MEM    : S3(2);     // any mem
 3823 %}
 3824 
 3825 // Float load constant
 3826 pipe_class fpu_reg_con(regD dst)
 3827 %{
 3828     instruction_count(2);
 3829     dst    : S5(write);
 3830     D0     : S0;        // big decoder only for the load
 3831     DECODE : S1;        // any decoder for FPU POP
 3832     FPU    : S4;
 3833     MEM    : S3;        // any mem
 3834 %}
 3835 
 3836 // Float load constant
 3837 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3838 %{
 3839     instruction_count(3);
 3840     dst    : S5(write);
 3841     src    : S3(read);
 3842     D0     : S0;        // big decoder only for the load
 3843     DECODE : S1(2);     // any decoder for FPU POP
 3844     FPU    : S4;
 3845     MEM    : S3;        // any mem
 3846 %}
 3847 
 3848 // UnConditional branch
 3849 pipe_class pipe_jmp(label labl)
 3850 %{
 3851     single_instruction;
 3852     BR   : S3;
 3853 %}
 3854 
 3855 // Conditional branch
 3856 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3857 %{
 3858     single_instruction;
 3859     cr    : S1(read);
 3860     BR    : S3;
 3861 %}
 3862 
 3863 // Allocation idiom
 3864 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3865 %{
 3866     instruction_count(1); force_serialization;
 3867     fixed_latency(6);
 3868     heap_ptr : S3(read);
 3869     DECODE   : S0(3);
 3870     D0       : S2;
 3871     MEM      : S3;
 3872     ALU      : S3(2);
 3873     dst      : S5(write);
 3874     BR       : S5;
 3875 %}
 3876 
 3877 // Generic big/slow expanded idiom
 3878 pipe_class pipe_slow()
 3879 %{
 3880     instruction_count(10); multiple_bundles; force_serialization;
 3881     fixed_latency(100);
 3882     D0  : S0(2);
 3883     MEM : S3(2);
 3884 %}
 3885 
 3886 // The real do-nothing guy
 3887 pipe_class empty()
 3888 %{
 3889     instruction_count(0);
 3890 %}
 3891 
 3892 // Define the class for the Nop node
 3893 define
 3894 %{
 3895    MachNop = empty;
 3896 %}
 3897 
 3898 %}
 3899 
 3900 //----------INSTRUCTIONS-------------------------------------------------------
 3901 //
 3902 // match      -- States which machine-independent subtree may be replaced
 3903 //               by this instruction.
 3904 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3905 //               selection to identify a minimum cost tree of machine
 3906 //               instructions that matches a tree of machine-independent
 3907 //               instructions.
 3908 // format     -- A string providing the disassembly for this instruction.
 3909 //               The value of an instruction's operand may be inserted
 3910 //               by referring to it with a '$' prefix.
 3911 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3912 //               to within an encode class as $primary, $secondary, and $tertiary
 3913 //               rrspectively.  The primary opcode is commonly used to
 3914 //               indicate the type of machine instruction, while secondary
 3915 //               and tertiary are often used for prefix options or addressing
 3916 //               modes.
 3917 // ins_encode -- A list of encode classes with parameters. The encode class
 3918 //               name must have been defined in an 'enc_class' specification
 3919 //               in the encode section of the architecture description.
 3920 
 3921 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3922 // Load Float
 3923 instruct MoveF2VL(vlRegF dst, regF src) %{
 3924   match(Set dst src);
 3925   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3926   ins_encode %{
 3927     ShouldNotReachHere();
 3928   %}
 3929   ins_pipe( fpu_reg_reg );
 3930 %}
 3931 
 3932 // Load Float
 3933 instruct MoveF2LEG(legRegF dst, regF src) %{
 3934   match(Set dst src);
 3935   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3936   ins_encode %{
 3937     ShouldNotReachHere();
 3938   %}
 3939   ins_pipe( fpu_reg_reg );
 3940 %}
 3941 
 3942 // Load Float
 3943 instruct MoveVL2F(regF dst, vlRegF src) %{
 3944   match(Set dst src);
 3945   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3946   ins_encode %{
 3947     ShouldNotReachHere();
 3948   %}
 3949   ins_pipe( fpu_reg_reg );
 3950 %}
 3951 
 3952 // Load Float
 3953 instruct MoveLEG2F(regF dst, legRegF src) %{
 3954   match(Set dst src);
 3955   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3956   ins_encode %{
 3957     ShouldNotReachHere();
 3958   %}
 3959   ins_pipe( fpu_reg_reg );
 3960 %}
 3961 
 3962 // Load Double
 3963 instruct MoveD2VL(vlRegD dst, regD src) %{
 3964   match(Set dst src);
 3965   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3966   ins_encode %{
 3967     ShouldNotReachHere();
 3968   %}
 3969   ins_pipe( fpu_reg_reg );
 3970 %}
 3971 
 3972 // Load Double
 3973 instruct MoveD2LEG(legRegD dst, regD src) %{
 3974   match(Set dst src);
 3975   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3976   ins_encode %{
 3977     ShouldNotReachHere();
 3978   %}
 3979   ins_pipe( fpu_reg_reg );
 3980 %}
 3981 
 3982 // Load Double
 3983 instruct MoveVL2D(regD dst, vlRegD src) %{
 3984   match(Set dst src);
 3985   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3986   ins_encode %{
 3987     ShouldNotReachHere();
 3988   %}
 3989   ins_pipe( fpu_reg_reg );
 3990 %}
 3991 
 3992 // Load Double
 3993 instruct MoveLEG2D(regD dst, legRegD src) %{
 3994   match(Set dst src);
 3995   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3996   ins_encode %{
 3997     ShouldNotReachHere();
 3998   %}
 3999   ins_pipe( fpu_reg_reg );
 4000 %}
 4001 
 4002 //----------Load/Store/Move Instructions---------------------------------------
 4003 //----------Load Instructions--------------------------------------------------
 4004 
 4005 // Load Byte (8 bit signed)
 4006 instruct loadB(rRegI dst, memory mem)
 4007 %{
 4008   match(Set dst (LoadB mem));
 4009 
 4010   ins_cost(125);
 4011   format %{ "movsbl  $dst, $mem\t# byte" %}
 4012 
 4013   ins_encode %{
 4014     __ movsbl($dst$$Register, $mem$$Address);
 4015   %}
 4016 
 4017   ins_pipe(ialu_reg_mem);
 4018 %}
 4019 
 4020 // Load Byte (8 bit signed) into Long Register
 4021 instruct loadB2L(rRegL dst, memory mem)
 4022 %{
 4023   match(Set dst (ConvI2L (LoadB mem)));
 4024 
 4025   ins_cost(125);
 4026   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4027 
 4028   ins_encode %{
 4029     __ movsbq($dst$$Register, $mem$$Address);
 4030   %}
 4031 
 4032   ins_pipe(ialu_reg_mem);
 4033 %}
 4034 
 4035 // Load Unsigned Byte (8 bit UNsigned)
 4036 instruct loadUB(rRegI dst, memory mem)
 4037 %{
 4038   match(Set dst (LoadUB mem));
 4039 
 4040   ins_cost(125);
 4041   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4042 
 4043   ins_encode %{
 4044     __ movzbl($dst$$Register, $mem$$Address);
 4045   %}
 4046 
 4047   ins_pipe(ialu_reg_mem);
 4048 %}
 4049 
 4050 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4051 instruct loadUB2L(rRegL dst, memory mem)
 4052 %{
 4053   match(Set dst (ConvI2L (LoadUB mem)));
 4054 
 4055   ins_cost(125);
 4056   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4057 
 4058   ins_encode %{
 4059     __ movzbq($dst$$Register, $mem$$Address);
 4060   %}
 4061 
 4062   ins_pipe(ialu_reg_mem);
 4063 %}
 4064 
 4065 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4066 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4067   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4068   effect(KILL cr);
 4069 
 4070   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4071             "andl    $dst, right_n_bits($mask, 8)" %}
 4072   ins_encode %{
 4073     Register Rdst = $dst$$Register;
 4074     __ movzbq(Rdst, $mem$$Address);
 4075     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4076   %}
 4077   ins_pipe(ialu_reg_mem);
 4078 %}
 4079 
 4080 // Load Short (16 bit signed)
 4081 instruct loadS(rRegI dst, memory mem)
 4082 %{
 4083   match(Set dst (LoadS mem));
 4084 
 4085   ins_cost(125);
 4086   format %{ "movswl $dst, $mem\t# short" %}
 4087 
 4088   ins_encode %{
 4089     __ movswl($dst$$Register, $mem$$Address);
 4090   %}
 4091 
 4092   ins_pipe(ialu_reg_mem);
 4093 %}
 4094 
 4095 // Load Short (16 bit signed) to Byte (8 bit signed)
 4096 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4097   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4098 
 4099   ins_cost(125);
 4100   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4101   ins_encode %{
 4102     __ movsbl($dst$$Register, $mem$$Address);
 4103   %}
 4104   ins_pipe(ialu_reg_mem);
 4105 %}
 4106 
 4107 // Load Short (16 bit signed) into Long Register
 4108 instruct loadS2L(rRegL dst, memory mem)
 4109 %{
 4110   match(Set dst (ConvI2L (LoadS mem)));
 4111 
 4112   ins_cost(125);
 4113   format %{ "movswq $dst, $mem\t# short -> long" %}
 4114 
 4115   ins_encode %{
 4116     __ movswq($dst$$Register, $mem$$Address);
 4117   %}
 4118 
 4119   ins_pipe(ialu_reg_mem);
 4120 %}
 4121 
 4122 // Load Unsigned Short/Char (16 bit UNsigned)
 4123 instruct loadUS(rRegI dst, memory mem)
 4124 %{
 4125   match(Set dst (LoadUS mem));
 4126 
 4127   ins_cost(125);
 4128   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4129 
 4130   ins_encode %{
 4131     __ movzwl($dst$$Register, $mem$$Address);
 4132   %}
 4133 
 4134   ins_pipe(ialu_reg_mem);
 4135 %}
 4136 
 4137 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4138 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4139   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4140 
 4141   ins_cost(125);
 4142   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4143   ins_encode %{
 4144     __ movsbl($dst$$Register, $mem$$Address);
 4145   %}
 4146   ins_pipe(ialu_reg_mem);
 4147 %}
 4148 
 4149 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4150 instruct loadUS2L(rRegL dst, memory mem)
 4151 %{
 4152   match(Set dst (ConvI2L (LoadUS mem)));
 4153 
 4154   ins_cost(125);
 4155   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4156 
 4157   ins_encode %{
 4158     __ movzwq($dst$$Register, $mem$$Address);
 4159   %}
 4160 
 4161   ins_pipe(ialu_reg_mem);
 4162 %}
 4163 
 4164 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4165 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4166   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4167 
 4168   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4169   ins_encode %{
 4170     __ movzbq($dst$$Register, $mem$$Address);
 4171   %}
 4172   ins_pipe(ialu_reg_mem);
 4173 %}
 4174 
 4175 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4176 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4177   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4178   effect(KILL cr);
 4179 
 4180   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4181             "andl    $dst, right_n_bits($mask, 16)" %}
 4182   ins_encode %{
 4183     Register Rdst = $dst$$Register;
 4184     __ movzwq(Rdst, $mem$$Address);
 4185     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4186   %}
 4187   ins_pipe(ialu_reg_mem);
 4188 %}
 4189 
 4190 // Load Integer
 4191 instruct loadI(rRegI dst, memory mem)
 4192 %{
 4193   match(Set dst (LoadI mem));
 4194 
 4195   ins_cost(125);
 4196   format %{ "movl    $dst, $mem\t# int" %}
 4197 
 4198   ins_encode %{
 4199     __ movl($dst$$Register, $mem$$Address);
 4200   %}
 4201 
 4202   ins_pipe(ialu_reg_mem);
 4203 %}
 4204 
 4205 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4206 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4207   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4208 
 4209   ins_cost(125);
 4210   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4211   ins_encode %{
 4212     __ movsbl($dst$$Register, $mem$$Address);
 4213   %}
 4214   ins_pipe(ialu_reg_mem);
 4215 %}
 4216 
 4217 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4218 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4219   match(Set dst (AndI (LoadI mem) mask));
 4220 
 4221   ins_cost(125);
 4222   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4223   ins_encode %{
 4224     __ movzbl($dst$$Register, $mem$$Address);
 4225   %}
 4226   ins_pipe(ialu_reg_mem);
 4227 %}
 4228 
 4229 // Load Integer (32 bit signed) to Short (16 bit signed)
 4230 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4231   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4232 
 4233   ins_cost(125);
 4234   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4235   ins_encode %{
 4236     __ movswl($dst$$Register, $mem$$Address);
 4237   %}
 4238   ins_pipe(ialu_reg_mem);
 4239 %}
 4240 
 4241 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4242 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4243   match(Set dst (AndI (LoadI mem) mask));
 4244 
 4245   ins_cost(125);
 4246   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4247   ins_encode %{
 4248     __ movzwl($dst$$Register, $mem$$Address);
 4249   %}
 4250   ins_pipe(ialu_reg_mem);
 4251 %}
 4252 
 4253 // Load Integer into Long Register
 4254 instruct loadI2L(rRegL dst, memory mem)
 4255 %{
 4256   match(Set dst (ConvI2L (LoadI mem)));
 4257 
 4258   ins_cost(125);
 4259   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4260 
 4261   ins_encode %{
 4262     __ movslq($dst$$Register, $mem$$Address);
 4263   %}
 4264 
 4265   ins_pipe(ialu_reg_mem);
 4266 %}
 4267 
 4268 // Load Integer with mask 0xFF into Long Register
 4269 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4270   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4271 
 4272   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4273   ins_encode %{
 4274     __ movzbq($dst$$Register, $mem$$Address);
 4275   %}
 4276   ins_pipe(ialu_reg_mem);
 4277 %}
 4278 
 4279 // Load Integer with mask 0xFFFF into Long Register
 4280 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4281   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4282 
 4283   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4284   ins_encode %{
 4285     __ movzwq($dst$$Register, $mem$$Address);
 4286   %}
 4287   ins_pipe(ialu_reg_mem);
 4288 %}
 4289 
 4290 // Load Integer with a 31-bit mask into Long Register
 4291 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4292   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4293   effect(KILL cr);
 4294 
 4295   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4296             "andl    $dst, $mask" %}
 4297   ins_encode %{
 4298     Register Rdst = $dst$$Register;
 4299     __ movl(Rdst, $mem$$Address);
 4300     __ andl(Rdst, $mask$$constant);
 4301   %}
 4302   ins_pipe(ialu_reg_mem);
 4303 %}
 4304 
 4305 // Load Unsigned Integer into Long Register
 4306 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4307 %{
 4308   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4309 
 4310   ins_cost(125);
 4311   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4312 
 4313   ins_encode %{
 4314     __ movl($dst$$Register, $mem$$Address);
 4315   %}
 4316 
 4317   ins_pipe(ialu_reg_mem);
 4318 %}
 4319 
 4320 // Load Long
 4321 instruct loadL(rRegL dst, memory mem)
 4322 %{
 4323   match(Set dst (LoadL mem));
 4324 
 4325   ins_cost(125);
 4326   format %{ "movq    $dst, $mem\t# long" %}
 4327 
 4328   ins_encode %{
 4329     __ movq($dst$$Register, $mem$$Address);
 4330   %}
 4331 
 4332   ins_pipe(ialu_reg_mem); // XXX
 4333 %}
 4334 
 4335 // Load Range
 4336 instruct loadRange(rRegI dst, memory mem)
 4337 %{
 4338   match(Set dst (LoadRange mem));
 4339 
 4340   ins_cost(125); // XXX
 4341   format %{ "movl    $dst, $mem\t# range" %}
 4342   ins_encode %{
 4343     __ movl($dst$$Register, $mem$$Address);
 4344   %}
 4345   ins_pipe(ialu_reg_mem);
 4346 %}
 4347 
 4348 // Load Pointer
 4349 instruct loadP(rRegP dst, memory mem)
 4350 %{
 4351   match(Set dst (LoadP mem));
 4352   predicate(n->as_Load()->barrier_data() == 0);
 4353 
 4354   ins_cost(125); // XXX
 4355   format %{ "movq    $dst, $mem\t# ptr" %}
 4356   ins_encode %{
 4357     __ movq($dst$$Register, $mem$$Address);
 4358   %}
 4359   ins_pipe(ialu_reg_mem); // XXX
 4360 %}
 4361 
 4362 // Load Compressed Pointer
 4363 instruct loadN(rRegN dst, memory mem)
 4364 %{
 4365    predicate(n->as_Load()->barrier_data() == 0);
 4366    match(Set dst (LoadN mem));
 4367 
 4368    ins_cost(125); // XXX
 4369    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4370    ins_encode %{
 4371      __ movl($dst$$Register, $mem$$Address);
 4372    %}
 4373    ins_pipe(ialu_reg_mem); // XXX
 4374 %}
 4375 
 4376 
 4377 // Load Klass Pointer
 4378 instruct loadKlass(rRegP dst, memory mem)
 4379 %{
 4380   match(Set dst (LoadKlass mem));
 4381 
 4382   ins_cost(125); // XXX
 4383   format %{ "movq    $dst, $mem\t# class" %}
 4384   ins_encode %{
 4385     __ movq($dst$$Register, $mem$$Address);
 4386   %}
 4387   ins_pipe(ialu_reg_mem); // XXX
 4388 %}
 4389 
 4390 // Load narrow Klass Pointer
 4391 instruct loadNKlass(rRegN dst, memory mem)
 4392 %{
 4393   predicate(!UseCompactObjectHeaders);
 4394   match(Set dst (LoadNKlass mem));
 4395 
 4396   ins_cost(125); // XXX
 4397   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4398   ins_encode %{
 4399     __ movl($dst$$Register, $mem$$Address);
 4400   %}
 4401   ins_pipe(ialu_reg_mem); // XXX
 4402 %}
 4403 
 4404 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4405 %{
 4406   predicate(UseCompactObjectHeaders);
 4407   match(Set dst (LoadNKlass mem));
 4408   effect(KILL cr);
 4409   ins_cost(125);
 4410   format %{
 4411     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4412     "shrl    $dst, markWord::klass_shift_at_offset"
 4413   %}
 4414   ins_encode %{
 4415     if (UseAPX) {
 4416       __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
 4417     }
 4418     else {
 4419       __ movl($dst$$Register, $mem$$Address);
 4420       __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4421     }
 4422   %}
 4423   ins_pipe(ialu_reg_mem);
 4424 %}
 4425 
 4426 // Load Float
 4427 instruct loadF(regF dst, memory mem)
 4428 %{
 4429   match(Set dst (LoadF mem));
 4430 
 4431   ins_cost(145); // XXX
 4432   format %{ "movss   $dst, $mem\t# float" %}
 4433   ins_encode %{
 4434     __ movflt($dst$$XMMRegister, $mem$$Address);
 4435   %}
 4436   ins_pipe(pipe_slow); // XXX
 4437 %}
 4438 
 4439 // Load Double
 4440 instruct loadD_partial(regD dst, memory mem)
 4441 %{
 4442   predicate(!UseXmmLoadAndClearUpper);
 4443   match(Set dst (LoadD mem));
 4444 
 4445   ins_cost(145); // XXX
 4446   format %{ "movlpd  $dst, $mem\t# double" %}
 4447   ins_encode %{
 4448     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4449   %}
 4450   ins_pipe(pipe_slow); // XXX
 4451 %}
 4452 
 4453 instruct loadD(regD dst, memory mem)
 4454 %{
 4455   predicate(UseXmmLoadAndClearUpper);
 4456   match(Set dst (LoadD mem));
 4457 
 4458   ins_cost(145); // XXX
 4459   format %{ "movsd   $dst, $mem\t# double" %}
 4460   ins_encode %{
 4461     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4462   %}
 4463   ins_pipe(pipe_slow); // XXX
 4464 %}
 4465 
 4466 // max = java.lang.Math.max(float a, float b)
 4467 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4468   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4469   match(Set dst (MaxF a b));
 4470   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4471   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4472   ins_encode %{
 4473     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4474   %}
 4475   ins_pipe( pipe_slow );
 4476 %}
 4477 
 4478 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4479   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4480   match(Set dst (MaxF a b));
 4481   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4482 
 4483   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4484   ins_encode %{
 4485     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4486                     false /*min*/, true /*single*/);
 4487   %}
 4488   ins_pipe( pipe_slow );
 4489 %}
 4490 
 4491 // max = java.lang.Math.max(double a, double b)
 4492 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4493   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4494   match(Set dst (MaxD a b));
 4495   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4496   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4497   ins_encode %{
 4498     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4499   %}
 4500   ins_pipe( pipe_slow );
 4501 %}
 4502 
 4503 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4504   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4505   match(Set dst (MaxD a b));
 4506   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4507 
 4508   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4509   ins_encode %{
 4510     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4511                     false /*min*/, false /*single*/);
 4512   %}
 4513   ins_pipe( pipe_slow );
 4514 %}
 4515 
 4516 // min = java.lang.Math.min(float a, float b)
 4517 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4518   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4519   match(Set dst (MinF a b));
 4520   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4521   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4522   ins_encode %{
 4523     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4524   %}
 4525   ins_pipe( pipe_slow );
 4526 %}
 4527 
 4528 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4529   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4530   match(Set dst (MinF a b));
 4531   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4532 
 4533   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4534   ins_encode %{
 4535     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4536                     true /*min*/, true /*single*/);
 4537   %}
 4538   ins_pipe( pipe_slow );
 4539 %}
 4540 
 4541 // min = java.lang.Math.min(double a, double b)
 4542 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4543   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4544   match(Set dst (MinD a b));
 4545   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4546     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4547   ins_encode %{
 4548     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4549   %}
 4550   ins_pipe( pipe_slow );
 4551 %}
 4552 
 4553 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4554   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4555   match(Set dst (MinD a b));
 4556   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4557 
 4558   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4559   ins_encode %{
 4560     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4561                     true /*min*/, false /*single*/);
 4562   %}
 4563   ins_pipe( pipe_slow );
 4564 %}
 4565 
 4566 // Load Effective Address
 4567 instruct leaP8(rRegP dst, indOffset8 mem)
 4568 %{
 4569   match(Set dst mem);
 4570 
 4571   ins_cost(110); // XXX
 4572   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4573   ins_encode %{
 4574     __ leaq($dst$$Register, $mem$$Address);
 4575   %}
 4576   ins_pipe(ialu_reg_reg_fat);
 4577 %}
 4578 
 4579 instruct leaP32(rRegP dst, indOffset32 mem)
 4580 %{
 4581   match(Set dst mem);
 4582 
 4583   ins_cost(110);
 4584   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4585   ins_encode %{
 4586     __ leaq($dst$$Register, $mem$$Address);
 4587   %}
 4588   ins_pipe(ialu_reg_reg_fat);
 4589 %}
 4590 
 4591 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4592 %{
 4593   match(Set dst mem);
 4594 
 4595   ins_cost(110);
 4596   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4597   ins_encode %{
 4598     __ leaq($dst$$Register, $mem$$Address);
 4599   %}
 4600   ins_pipe(ialu_reg_reg_fat);
 4601 %}
 4602 
 4603 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4604 %{
 4605   match(Set dst mem);
 4606 
 4607   ins_cost(110);
 4608   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4609   ins_encode %{
 4610     __ leaq($dst$$Register, $mem$$Address);
 4611   %}
 4612   ins_pipe(ialu_reg_reg_fat);
 4613 %}
 4614 
 4615 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4616 %{
 4617   match(Set dst mem);
 4618 
 4619   ins_cost(110);
 4620   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4621   ins_encode %{
 4622     __ leaq($dst$$Register, $mem$$Address);
 4623   %}
 4624   ins_pipe(ialu_reg_reg_fat);
 4625 %}
 4626 
 4627 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4628 %{
 4629   match(Set dst mem);
 4630 
 4631   ins_cost(110);
 4632   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4633   ins_encode %{
 4634     __ leaq($dst$$Register, $mem$$Address);
 4635   %}
 4636   ins_pipe(ialu_reg_reg_fat);
 4637 %}
 4638 
 4639 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4640 %{
 4641   match(Set dst mem);
 4642 
 4643   ins_cost(110);
 4644   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4645   ins_encode %{
 4646     __ leaq($dst$$Register, $mem$$Address);
 4647   %}
 4648   ins_pipe(ialu_reg_reg_fat);
 4649 %}
 4650 
 4651 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4652 %{
 4653   match(Set dst mem);
 4654 
 4655   ins_cost(110);
 4656   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4657   ins_encode %{
 4658     __ leaq($dst$$Register, $mem$$Address);
 4659   %}
 4660   ins_pipe(ialu_reg_reg_fat);
 4661 %}
 4662 
 4663 // Load Effective Address which uses Narrow (32-bits) oop
 4664 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4665 %{
 4666   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4667   match(Set dst mem);
 4668 
 4669   ins_cost(110);
 4670   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4671   ins_encode %{
 4672     __ leaq($dst$$Register, $mem$$Address);
 4673   %}
 4674   ins_pipe(ialu_reg_reg_fat);
 4675 %}
 4676 
 4677 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4678 %{
 4679   predicate(CompressedOops::shift() == 0);
 4680   match(Set dst mem);
 4681 
 4682   ins_cost(110); // XXX
 4683   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4684   ins_encode %{
 4685     __ leaq($dst$$Register, $mem$$Address);
 4686   %}
 4687   ins_pipe(ialu_reg_reg_fat);
 4688 %}
 4689 
 4690 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4691 %{
 4692   predicate(CompressedOops::shift() == 0);
 4693   match(Set dst mem);
 4694 
 4695   ins_cost(110);
 4696   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4697   ins_encode %{
 4698     __ leaq($dst$$Register, $mem$$Address);
 4699   %}
 4700   ins_pipe(ialu_reg_reg_fat);
 4701 %}
 4702 
 4703 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4704 %{
 4705   predicate(CompressedOops::shift() == 0);
 4706   match(Set dst mem);
 4707 
 4708   ins_cost(110);
 4709   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4710   ins_encode %{
 4711     __ leaq($dst$$Register, $mem$$Address);
 4712   %}
 4713   ins_pipe(ialu_reg_reg_fat);
 4714 %}
 4715 
 4716 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4717 %{
 4718   predicate(CompressedOops::shift() == 0);
 4719   match(Set dst mem);
 4720 
 4721   ins_cost(110);
 4722   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4723   ins_encode %{
 4724     __ leaq($dst$$Register, $mem$$Address);
 4725   %}
 4726   ins_pipe(ialu_reg_reg_fat);
 4727 %}
 4728 
 4729 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4730 %{
 4731   predicate(CompressedOops::shift() == 0);
 4732   match(Set dst mem);
 4733 
 4734   ins_cost(110);
 4735   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4736   ins_encode %{
 4737     __ leaq($dst$$Register, $mem$$Address);
 4738   %}
 4739   ins_pipe(ialu_reg_reg_fat);
 4740 %}
 4741 
 4742 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4743 %{
 4744   predicate(CompressedOops::shift() == 0);
 4745   match(Set dst mem);
 4746 
 4747   ins_cost(110);
 4748   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4749   ins_encode %{
 4750     __ leaq($dst$$Register, $mem$$Address);
 4751   %}
 4752   ins_pipe(ialu_reg_reg_fat);
 4753 %}
 4754 
 4755 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4756 %{
 4757   predicate(CompressedOops::shift() == 0);
 4758   match(Set dst mem);
 4759 
 4760   ins_cost(110);
 4761   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4762   ins_encode %{
 4763     __ leaq($dst$$Register, $mem$$Address);
 4764   %}
 4765   ins_pipe(ialu_reg_reg_fat);
 4766 %}
 4767 
 4768 instruct loadConI(rRegI dst, immI src)
 4769 %{
 4770   match(Set dst src);
 4771 
 4772   format %{ "movl    $dst, $src\t# int" %}
 4773   ins_encode %{
 4774     __ movl($dst$$Register, $src$$constant);
 4775   %}
 4776   ins_pipe(ialu_reg_fat); // XXX
 4777 %}
 4778 
 4779 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4780 %{
 4781   match(Set dst src);
 4782   effect(KILL cr);
 4783 
 4784   ins_cost(50);
 4785   format %{ "xorl    $dst, $dst\t# int" %}
 4786   ins_encode %{
 4787     __ xorl($dst$$Register, $dst$$Register);
 4788   %}
 4789   ins_pipe(ialu_reg);
 4790 %}
 4791 
 4792 instruct loadConL(rRegL dst, immL src)
 4793 %{
 4794   match(Set dst src);
 4795 
 4796   ins_cost(150);
 4797   format %{ "movq    $dst, $src\t# long" %}
 4798   ins_encode %{
 4799     __ mov64($dst$$Register, $src$$constant);
 4800   %}
 4801   ins_pipe(ialu_reg);
 4802 %}
 4803 
 4804 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4805 %{
 4806   match(Set dst src);
 4807   effect(KILL cr);
 4808 
 4809   ins_cost(50);
 4810   format %{ "xorl    $dst, $dst\t# long" %}
 4811   ins_encode %{
 4812     __ xorl($dst$$Register, $dst$$Register);
 4813   %}
 4814   ins_pipe(ialu_reg); // XXX
 4815 %}
 4816 
 4817 instruct loadConUL32(rRegL dst, immUL32 src)
 4818 %{
 4819   match(Set dst src);
 4820 
 4821   ins_cost(60);
 4822   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4823   ins_encode %{
 4824     __ movl($dst$$Register, $src$$constant);
 4825   %}
 4826   ins_pipe(ialu_reg);
 4827 %}
 4828 
 4829 instruct loadConL32(rRegL dst, immL32 src)
 4830 %{
 4831   match(Set dst src);
 4832 
 4833   ins_cost(70);
 4834   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4835   ins_encode %{
 4836     __ movq($dst$$Register, $src$$constant);
 4837   %}
 4838   ins_pipe(ialu_reg);
 4839 %}
 4840 
 4841 instruct loadConP(rRegP dst, immP con) %{
 4842   match(Set dst con);
 4843 
 4844   format %{ "movq    $dst, $con\t# ptr" %}
 4845   ins_encode %{
 4846     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4847   %}
 4848   ins_pipe(ialu_reg_fat); // XXX
 4849 %}
 4850 
 4851 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4852 %{
 4853   match(Set dst src);
 4854   effect(KILL cr);
 4855 
 4856   ins_cost(50);
 4857   format %{ "xorl    $dst, $dst\t# ptr" %}
 4858   ins_encode %{
 4859     __ xorl($dst$$Register, $dst$$Register);
 4860   %}
 4861   ins_pipe(ialu_reg);
 4862 %}
 4863 
 4864 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4865 %{
 4866   match(Set dst src);
 4867   effect(KILL cr);
 4868 
 4869   ins_cost(60);
 4870   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4871   ins_encode %{
 4872     __ movl($dst$$Register, $src$$constant);
 4873   %}
 4874   ins_pipe(ialu_reg);
 4875 %}
 4876 
 4877 instruct loadConF(regF dst, immF con) %{
 4878   match(Set dst con);
 4879   ins_cost(125);
 4880   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4881   ins_encode %{
 4882     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4883   %}
 4884   ins_pipe(pipe_slow);
 4885 %}
 4886 
 4887 instruct loadConH(regF dst, immH con) %{
 4888   match(Set dst con);
 4889   ins_cost(125);
 4890   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4891   ins_encode %{
 4892     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4893   %}
 4894   ins_pipe(pipe_slow);
 4895 %}
 4896 
 4897 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4898   match(Set dst src);
 4899   effect(KILL cr);
 4900   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4901   ins_encode %{
 4902     __ xorq($dst$$Register, $dst$$Register);
 4903   %}
 4904   ins_pipe(ialu_reg);
 4905 %}
 4906 
 4907 instruct loadConN(rRegN dst, immN src) %{
 4908   match(Set dst src);
 4909 
 4910   ins_cost(125);
 4911   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4912   ins_encode %{
 4913     address con = (address)$src$$constant;
 4914     if (con == nullptr) {
 4915       ShouldNotReachHere();
 4916     } else {
 4917       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4918     }
 4919   %}
 4920   ins_pipe(ialu_reg_fat); // XXX
 4921 %}
 4922 
 4923 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4924   match(Set dst src);
 4925 
 4926   ins_cost(125);
 4927   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4928   ins_encode %{
 4929     address con = (address)$src$$constant;
 4930     if (con == nullptr) {
 4931       ShouldNotReachHere();
 4932     } else {
 4933       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4934     }
 4935   %}
 4936   ins_pipe(ialu_reg_fat); // XXX
 4937 %}
 4938 
 4939 instruct loadConF0(regF dst, immF0 src)
 4940 %{
 4941   match(Set dst src);
 4942   ins_cost(100);
 4943 
 4944   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4945   ins_encode %{
 4946     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4947   %}
 4948   ins_pipe(pipe_slow);
 4949 %}
 4950 
 4951 // Use the same format since predicate() can not be used here.
 4952 instruct loadConD(regD dst, immD con) %{
 4953   match(Set dst con);
 4954   ins_cost(125);
 4955   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4956   ins_encode %{
 4957     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4958   %}
 4959   ins_pipe(pipe_slow);
 4960 %}
 4961 
 4962 instruct loadConD0(regD dst, immD0 src)
 4963 %{
 4964   match(Set dst src);
 4965   ins_cost(100);
 4966 
 4967   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4968   ins_encode %{
 4969     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4970   %}
 4971   ins_pipe(pipe_slow);
 4972 %}
 4973 
 4974 instruct loadSSI(rRegI dst, stackSlotI src)
 4975 %{
 4976   match(Set dst src);
 4977 
 4978   ins_cost(125);
 4979   format %{ "movl    $dst, $src\t# int stk" %}
 4980   ins_encode %{
 4981     __ movl($dst$$Register, $src$$Address);
 4982   %}
 4983   ins_pipe(ialu_reg_mem);
 4984 %}
 4985 
 4986 instruct loadSSL(rRegL dst, stackSlotL src)
 4987 %{
 4988   match(Set dst src);
 4989 
 4990   ins_cost(125);
 4991   format %{ "movq    $dst, $src\t# long stk" %}
 4992   ins_encode %{
 4993     __ movq($dst$$Register, $src$$Address);
 4994   %}
 4995   ins_pipe(ialu_reg_mem);
 4996 %}
 4997 
 4998 instruct loadSSP(rRegP dst, stackSlotP src)
 4999 %{
 5000   match(Set dst src);
 5001 
 5002   ins_cost(125);
 5003   format %{ "movq    $dst, $src\t# ptr stk" %}
 5004   ins_encode %{
 5005     __ movq($dst$$Register, $src$$Address);
 5006   %}
 5007   ins_pipe(ialu_reg_mem);
 5008 %}
 5009 
 5010 instruct loadSSF(regF dst, stackSlotF src)
 5011 %{
 5012   match(Set dst src);
 5013 
 5014   ins_cost(125);
 5015   format %{ "movss   $dst, $src\t# float stk" %}
 5016   ins_encode %{
 5017     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5018   %}
 5019   ins_pipe(pipe_slow); // XXX
 5020 %}
 5021 
 5022 // Use the same format since predicate() can not be used here.
 5023 instruct loadSSD(regD dst, stackSlotD src)
 5024 %{
 5025   match(Set dst src);
 5026 
 5027   ins_cost(125);
 5028   format %{ "movsd   $dst, $src\t# double stk" %}
 5029   ins_encode  %{
 5030     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5031   %}
 5032   ins_pipe(pipe_slow); // XXX
 5033 %}
 5034 
 5035 // Prefetch instructions for allocation.
 5036 // Must be safe to execute with invalid address (cannot fault).
 5037 
 5038 instruct prefetchAlloc( memory mem ) %{
 5039   predicate(AllocatePrefetchInstr==3);
 5040   match(PrefetchAllocation mem);
 5041   ins_cost(125);
 5042 
 5043   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5044   ins_encode %{
 5045     __ prefetchw($mem$$Address);
 5046   %}
 5047   ins_pipe(ialu_mem);
 5048 %}
 5049 
 5050 instruct prefetchAllocNTA( memory mem ) %{
 5051   predicate(AllocatePrefetchInstr==0);
 5052   match(PrefetchAllocation mem);
 5053   ins_cost(125);
 5054 
 5055   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5056   ins_encode %{
 5057     __ prefetchnta($mem$$Address);
 5058   %}
 5059   ins_pipe(ialu_mem);
 5060 %}
 5061 
 5062 instruct prefetchAllocT0( memory mem ) %{
 5063   predicate(AllocatePrefetchInstr==1);
 5064   match(PrefetchAllocation mem);
 5065   ins_cost(125);
 5066 
 5067   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5068   ins_encode %{
 5069     __ prefetcht0($mem$$Address);
 5070   %}
 5071   ins_pipe(ialu_mem);
 5072 %}
 5073 
 5074 instruct prefetchAllocT2( memory mem ) %{
 5075   predicate(AllocatePrefetchInstr==2);
 5076   match(PrefetchAllocation mem);
 5077   ins_cost(125);
 5078 
 5079   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5080   ins_encode %{
 5081     __ prefetcht2($mem$$Address);
 5082   %}
 5083   ins_pipe(ialu_mem);
 5084 %}
 5085 
 5086 //----------Store Instructions-------------------------------------------------
 5087 
 5088 // Store Byte
 5089 instruct storeB(memory mem, rRegI src)
 5090 %{
 5091   match(Set mem (StoreB mem src));
 5092 
 5093   ins_cost(125); // XXX
 5094   format %{ "movb    $mem, $src\t# byte" %}
 5095   ins_encode %{
 5096     __ movb($mem$$Address, $src$$Register);
 5097   %}
 5098   ins_pipe(ialu_mem_reg);
 5099 %}
 5100 
 5101 // Store Char/Short
 5102 instruct storeC(memory mem, rRegI src)
 5103 %{
 5104   match(Set mem (StoreC mem src));
 5105 
 5106   ins_cost(125); // XXX
 5107   format %{ "movw    $mem, $src\t# char/short" %}
 5108   ins_encode %{
 5109     __ movw($mem$$Address, $src$$Register);
 5110   %}
 5111   ins_pipe(ialu_mem_reg);
 5112 %}
 5113 
 5114 // Store Integer
 5115 instruct storeI(memory mem, rRegI src)
 5116 %{
 5117   match(Set mem (StoreI mem src));
 5118 
 5119   ins_cost(125); // XXX
 5120   format %{ "movl    $mem, $src\t# int" %}
 5121   ins_encode %{
 5122     __ movl($mem$$Address, $src$$Register);
 5123   %}
 5124   ins_pipe(ialu_mem_reg);
 5125 %}
 5126 
 5127 // Store Long
 5128 instruct storeL(memory mem, rRegL src)
 5129 %{
 5130   match(Set mem (StoreL mem src));
 5131 
 5132   ins_cost(125); // XXX
 5133   format %{ "movq    $mem, $src\t# long" %}
 5134   ins_encode %{
 5135     __ movq($mem$$Address, $src$$Register);
 5136   %}
 5137   ins_pipe(ialu_mem_reg); // XXX
 5138 %}
 5139 
 5140 // Store Pointer
 5141 instruct storeP(memory mem, any_RegP src)
 5142 %{
 5143   predicate(n->as_Store()->barrier_data() == 0);
 5144   match(Set mem (StoreP mem src));
 5145 
 5146   ins_cost(125); // XXX
 5147   format %{ "movq    $mem, $src\t# ptr" %}
 5148   ins_encode %{
 5149     __ movq($mem$$Address, $src$$Register);
 5150   %}
 5151   ins_pipe(ialu_mem_reg);
 5152 %}
 5153 
 5154 instruct storeImmP0(memory mem, immP0 zero)
 5155 %{
 5156   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5157   match(Set mem (StoreP mem zero));
 5158 
 5159   ins_cost(125); // XXX
 5160   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5161   ins_encode %{
 5162     __ movq($mem$$Address, r12);
 5163   %}
 5164   ins_pipe(ialu_mem_reg);
 5165 %}
 5166 
 5167 // Store Null Pointer, mark word, or other simple pointer constant.
 5168 instruct storeImmP(memory mem, immP31 src)
 5169 %{
 5170   predicate(n->as_Store()->barrier_data() == 0);
 5171   match(Set mem (StoreP mem src));
 5172 
 5173   ins_cost(150); // XXX
 5174   format %{ "movq    $mem, $src\t# ptr" %}
 5175   ins_encode %{
 5176     __ movq($mem$$Address, $src$$constant);
 5177   %}
 5178   ins_pipe(ialu_mem_imm);
 5179 %}
 5180 
 5181 // Store Compressed Pointer
 5182 instruct storeN(memory mem, rRegN src)
 5183 %{
 5184   predicate(n->as_Store()->barrier_data() == 0);
 5185   match(Set mem (StoreN mem src));
 5186 
 5187   ins_cost(125); // XXX
 5188   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5189   ins_encode %{
 5190     __ movl($mem$$Address, $src$$Register);
 5191   %}
 5192   ins_pipe(ialu_mem_reg);
 5193 %}
 5194 
 5195 instruct storeNKlass(memory mem, rRegN src)
 5196 %{
 5197   match(Set mem (StoreNKlass mem src));
 5198 
 5199   ins_cost(125); // XXX
 5200   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5201   ins_encode %{
 5202     __ movl($mem$$Address, $src$$Register);
 5203   %}
 5204   ins_pipe(ialu_mem_reg);
 5205 %}
 5206 
 5207 instruct storeImmN0(memory mem, immN0 zero)
 5208 %{
 5209   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5210   match(Set mem (StoreN mem zero));
 5211 
 5212   ins_cost(125); // XXX
 5213   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5214   ins_encode %{
 5215     __ movl($mem$$Address, r12);
 5216   %}
 5217   ins_pipe(ialu_mem_reg);
 5218 %}
 5219 
 5220 instruct storeImmN(memory mem, immN src)
 5221 %{
 5222   predicate(n->as_Store()->barrier_data() == 0);
 5223   match(Set mem (StoreN mem src));
 5224 
 5225   ins_cost(150); // XXX
 5226   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5227   ins_encode %{
 5228     address con = (address)$src$$constant;
 5229     if (con == nullptr) {
 5230       __ movl($mem$$Address, 0);
 5231     } else {
 5232       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5233     }
 5234   %}
 5235   ins_pipe(ialu_mem_imm);
 5236 %}
 5237 
 5238 instruct storeImmNKlass(memory mem, immNKlass src)
 5239 %{
 5240   match(Set mem (StoreNKlass mem src));
 5241 
 5242   ins_cost(150); // XXX
 5243   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5244   ins_encode %{
 5245     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5246   %}
 5247   ins_pipe(ialu_mem_imm);
 5248 %}
 5249 
 5250 // Store Integer Immediate
 5251 instruct storeImmI0(memory mem, immI_0 zero)
 5252 %{
 5253   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5254   match(Set mem (StoreI mem zero));
 5255 
 5256   ins_cost(125); // XXX
 5257   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5258   ins_encode %{
 5259     __ movl($mem$$Address, r12);
 5260   %}
 5261   ins_pipe(ialu_mem_reg);
 5262 %}
 5263 
 5264 instruct storeImmI(memory mem, immI src)
 5265 %{
 5266   match(Set mem (StoreI mem src));
 5267 
 5268   ins_cost(150);
 5269   format %{ "movl    $mem, $src\t# int" %}
 5270   ins_encode %{
 5271     __ movl($mem$$Address, $src$$constant);
 5272   %}
 5273   ins_pipe(ialu_mem_imm);
 5274 %}
 5275 
 5276 // Store Long Immediate
 5277 instruct storeImmL0(memory mem, immL0 zero)
 5278 %{
 5279   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5280   match(Set mem (StoreL mem zero));
 5281 
 5282   ins_cost(125); // XXX
 5283   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5284   ins_encode %{
 5285     __ movq($mem$$Address, r12);
 5286   %}
 5287   ins_pipe(ialu_mem_reg);
 5288 %}
 5289 
 5290 instruct storeImmL(memory mem, immL32 src)
 5291 %{
 5292   match(Set mem (StoreL mem src));
 5293 
 5294   ins_cost(150);
 5295   format %{ "movq    $mem, $src\t# long" %}
 5296   ins_encode %{
 5297     __ movq($mem$$Address, $src$$constant);
 5298   %}
 5299   ins_pipe(ialu_mem_imm);
 5300 %}
 5301 
 5302 // Store Short/Char Immediate
 5303 instruct storeImmC0(memory mem, immI_0 zero)
 5304 %{
 5305   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5306   match(Set mem (StoreC mem zero));
 5307 
 5308   ins_cost(125); // XXX
 5309   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5310   ins_encode %{
 5311     __ movw($mem$$Address, r12);
 5312   %}
 5313   ins_pipe(ialu_mem_reg);
 5314 %}
 5315 
 5316 instruct storeImmI16(memory mem, immI16 src)
 5317 %{
 5318   predicate(UseStoreImmI16);
 5319   match(Set mem (StoreC mem src));
 5320 
 5321   ins_cost(150);
 5322   format %{ "movw    $mem, $src\t# short/char" %}
 5323   ins_encode %{
 5324     __ movw($mem$$Address, $src$$constant);
 5325   %}
 5326   ins_pipe(ialu_mem_imm);
 5327 %}
 5328 
 5329 // Store Byte Immediate
 5330 instruct storeImmB0(memory mem, immI_0 zero)
 5331 %{
 5332   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5333   match(Set mem (StoreB mem zero));
 5334 
 5335   ins_cost(125); // XXX
 5336   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5337   ins_encode %{
 5338     __ movb($mem$$Address, r12);
 5339   %}
 5340   ins_pipe(ialu_mem_reg);
 5341 %}
 5342 
 5343 instruct storeImmB(memory mem, immI8 src)
 5344 %{
 5345   match(Set mem (StoreB mem src));
 5346 
 5347   ins_cost(150); // XXX
 5348   format %{ "movb    $mem, $src\t# byte" %}
 5349   ins_encode %{
 5350     __ movb($mem$$Address, $src$$constant);
 5351   %}
 5352   ins_pipe(ialu_mem_imm);
 5353 %}
 5354 
 5355 // Store Float
 5356 instruct storeF(memory mem, regF src)
 5357 %{
 5358   match(Set mem (StoreF mem src));
 5359 
 5360   ins_cost(95); // XXX
 5361   format %{ "movss   $mem, $src\t# float" %}
 5362   ins_encode %{
 5363     __ movflt($mem$$Address, $src$$XMMRegister);
 5364   %}
 5365   ins_pipe(pipe_slow); // XXX
 5366 %}
 5367 
 5368 // Store immediate Float value (it is faster than store from XMM register)
 5369 instruct storeF0(memory mem, immF0 zero)
 5370 %{
 5371   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5372   match(Set mem (StoreF mem zero));
 5373 
 5374   ins_cost(25); // XXX
 5375   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5376   ins_encode %{
 5377     __ movl($mem$$Address, r12);
 5378   %}
 5379   ins_pipe(ialu_mem_reg);
 5380 %}
 5381 
 5382 instruct storeF_imm(memory mem, immF src)
 5383 %{
 5384   match(Set mem (StoreF mem src));
 5385 
 5386   ins_cost(50);
 5387   format %{ "movl    $mem, $src\t# float" %}
 5388   ins_encode %{
 5389     __ movl($mem$$Address, jint_cast($src$$constant));
 5390   %}
 5391   ins_pipe(ialu_mem_imm);
 5392 %}
 5393 
 5394 // Store Double
 5395 instruct storeD(memory mem, regD src)
 5396 %{
 5397   match(Set mem (StoreD mem src));
 5398 
 5399   ins_cost(95); // XXX
 5400   format %{ "movsd   $mem, $src\t# double" %}
 5401   ins_encode %{
 5402     __ movdbl($mem$$Address, $src$$XMMRegister);
 5403   %}
 5404   ins_pipe(pipe_slow); // XXX
 5405 %}
 5406 
 5407 // Store immediate double 0.0 (it is faster than store from XMM register)
 5408 instruct storeD0_imm(memory mem, immD0 src)
 5409 %{
 5410   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5411   match(Set mem (StoreD mem src));
 5412 
 5413   ins_cost(50);
 5414   format %{ "movq    $mem, $src\t# double 0." %}
 5415   ins_encode %{
 5416     __ movq($mem$$Address, $src$$constant);
 5417   %}
 5418   ins_pipe(ialu_mem_imm);
 5419 %}
 5420 
 5421 instruct storeD0(memory mem, immD0 zero)
 5422 %{
 5423   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5424   match(Set mem (StoreD mem zero));
 5425 
 5426   ins_cost(25); // XXX
 5427   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5428   ins_encode %{
 5429     __ movq($mem$$Address, r12);
 5430   %}
 5431   ins_pipe(ialu_mem_reg);
 5432 %}
 5433 
 5434 instruct storeSSI(stackSlotI dst, rRegI src)
 5435 %{
 5436   match(Set dst src);
 5437 
 5438   ins_cost(100);
 5439   format %{ "movl    $dst, $src\t# int stk" %}
 5440   ins_encode %{
 5441     __ movl($dst$$Address, $src$$Register);
 5442   %}
 5443   ins_pipe( ialu_mem_reg );
 5444 %}
 5445 
 5446 instruct storeSSL(stackSlotL dst, rRegL src)
 5447 %{
 5448   match(Set dst src);
 5449 
 5450   ins_cost(100);
 5451   format %{ "movq    $dst, $src\t# long stk" %}
 5452   ins_encode %{
 5453     __ movq($dst$$Address, $src$$Register);
 5454   %}
 5455   ins_pipe(ialu_mem_reg);
 5456 %}
 5457 
 5458 instruct storeSSP(stackSlotP dst, rRegP src)
 5459 %{
 5460   match(Set dst src);
 5461 
 5462   ins_cost(100);
 5463   format %{ "movq    $dst, $src\t# ptr stk" %}
 5464   ins_encode %{
 5465     __ movq($dst$$Address, $src$$Register);
 5466   %}
 5467   ins_pipe(ialu_mem_reg);
 5468 %}
 5469 
 5470 instruct storeSSF(stackSlotF dst, regF src)
 5471 %{
 5472   match(Set dst src);
 5473 
 5474   ins_cost(95); // XXX
 5475   format %{ "movss   $dst, $src\t# float stk" %}
 5476   ins_encode %{
 5477     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5478   %}
 5479   ins_pipe(pipe_slow); // XXX
 5480 %}
 5481 
 5482 instruct storeSSD(stackSlotD dst, regD src)
 5483 %{
 5484   match(Set dst src);
 5485 
 5486   ins_cost(95); // XXX
 5487   format %{ "movsd   $dst, $src\t# double stk" %}
 5488   ins_encode %{
 5489     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5490   %}
 5491   ins_pipe(pipe_slow); // XXX
 5492 %}
 5493 
 5494 instruct cacheWB(indirect addr)
 5495 %{
 5496   predicate(VM_Version::supports_data_cache_line_flush());
 5497   match(CacheWB addr);
 5498 
 5499   ins_cost(100);
 5500   format %{"cache wb $addr" %}
 5501   ins_encode %{
 5502     assert($addr->index_position() < 0, "should be");
 5503     assert($addr$$disp == 0, "should be");
 5504     __ cache_wb(Address($addr$$base$$Register, 0));
 5505   %}
 5506   ins_pipe(pipe_slow); // XXX
 5507 %}
 5508 
 5509 instruct cacheWBPreSync()
 5510 %{
 5511   predicate(VM_Version::supports_data_cache_line_flush());
 5512   match(CacheWBPreSync);
 5513 
 5514   ins_cost(100);
 5515   format %{"cache wb presync" %}
 5516   ins_encode %{
 5517     __ cache_wbsync(true);
 5518   %}
 5519   ins_pipe(pipe_slow); // XXX
 5520 %}
 5521 
 5522 instruct cacheWBPostSync()
 5523 %{
 5524   predicate(VM_Version::supports_data_cache_line_flush());
 5525   match(CacheWBPostSync);
 5526 
 5527   ins_cost(100);
 5528   format %{"cache wb postsync" %}
 5529   ins_encode %{
 5530     __ cache_wbsync(false);
 5531   %}
 5532   ins_pipe(pipe_slow); // XXX
 5533 %}
 5534 
 5535 //----------BSWAP Instructions-------------------------------------------------
 5536 instruct bytes_reverse_int(rRegI dst) %{
 5537   match(Set dst (ReverseBytesI dst));
 5538 
 5539   format %{ "bswapl  $dst" %}
 5540   ins_encode %{
 5541     __ bswapl($dst$$Register);
 5542   %}
 5543   ins_pipe( ialu_reg );
 5544 %}
 5545 
 5546 instruct bytes_reverse_long(rRegL dst) %{
 5547   match(Set dst (ReverseBytesL dst));
 5548 
 5549   format %{ "bswapq  $dst" %}
 5550   ins_encode %{
 5551     __ bswapq($dst$$Register);
 5552   %}
 5553   ins_pipe( ialu_reg);
 5554 %}
 5555 
 5556 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5557   match(Set dst (ReverseBytesUS dst));
 5558   effect(KILL cr);
 5559 
 5560   format %{ "bswapl  $dst\n\t"
 5561             "shrl    $dst,16\n\t" %}
 5562   ins_encode %{
 5563     __ bswapl($dst$$Register);
 5564     __ shrl($dst$$Register, 16);
 5565   %}
 5566   ins_pipe( ialu_reg );
 5567 %}
 5568 
 5569 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5570   match(Set dst (ReverseBytesS dst));
 5571   effect(KILL cr);
 5572 
 5573   format %{ "bswapl  $dst\n\t"
 5574             "sar     $dst,16\n\t" %}
 5575   ins_encode %{
 5576     __ bswapl($dst$$Register);
 5577     __ sarl($dst$$Register, 16);
 5578   %}
 5579   ins_pipe( ialu_reg );
 5580 %}
 5581 
 5582 //---------- Zeros Count Instructions ------------------------------------------
 5583 
 5584 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5585   predicate(UseCountLeadingZerosInstruction);
 5586   match(Set dst (CountLeadingZerosI src));
 5587   effect(KILL cr);
 5588 
 5589   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5590   ins_encode %{
 5591     __ lzcntl($dst$$Register, $src$$Register);
 5592   %}
 5593   ins_pipe(ialu_reg);
 5594 %}
 5595 
 5596 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5597   predicate(UseCountLeadingZerosInstruction);
 5598   match(Set dst (CountLeadingZerosI (LoadI src)));
 5599   effect(KILL cr);
 5600   ins_cost(175);
 5601   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5602   ins_encode %{
 5603     __ lzcntl($dst$$Register, $src$$Address);
 5604   %}
 5605   ins_pipe(ialu_reg_mem);
 5606 %}
 5607 
 5608 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5609   predicate(!UseCountLeadingZerosInstruction);
 5610   match(Set dst (CountLeadingZerosI src));
 5611   effect(KILL cr);
 5612 
 5613   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5614             "jnz     skip\n\t"
 5615             "movl    $dst, -1\n"
 5616       "skip:\n\t"
 5617             "negl    $dst\n\t"
 5618             "addl    $dst, 31" %}
 5619   ins_encode %{
 5620     Register Rdst = $dst$$Register;
 5621     Register Rsrc = $src$$Register;
 5622     Label skip;
 5623     __ bsrl(Rdst, Rsrc);
 5624     __ jccb(Assembler::notZero, skip);
 5625     __ movl(Rdst, -1);
 5626     __ bind(skip);
 5627     __ negl(Rdst);
 5628     __ addl(Rdst, BitsPerInt - 1);
 5629   %}
 5630   ins_pipe(ialu_reg);
 5631 %}
 5632 
 5633 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5634   predicate(UseCountLeadingZerosInstruction);
 5635   match(Set dst (CountLeadingZerosL src));
 5636   effect(KILL cr);
 5637 
 5638   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5639   ins_encode %{
 5640     __ lzcntq($dst$$Register, $src$$Register);
 5641   %}
 5642   ins_pipe(ialu_reg);
 5643 %}
 5644 
 5645 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5646   predicate(UseCountLeadingZerosInstruction);
 5647   match(Set dst (CountLeadingZerosL (LoadL src)));
 5648   effect(KILL cr);
 5649   ins_cost(175);
 5650   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5651   ins_encode %{
 5652     __ lzcntq($dst$$Register, $src$$Address);
 5653   %}
 5654   ins_pipe(ialu_reg_mem);
 5655 %}
 5656 
 5657 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5658   predicate(!UseCountLeadingZerosInstruction);
 5659   match(Set dst (CountLeadingZerosL src));
 5660   effect(KILL cr);
 5661 
 5662   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5663             "jnz     skip\n\t"
 5664             "movl    $dst, -1\n"
 5665       "skip:\n\t"
 5666             "negl    $dst\n\t"
 5667             "addl    $dst, 63" %}
 5668   ins_encode %{
 5669     Register Rdst = $dst$$Register;
 5670     Register Rsrc = $src$$Register;
 5671     Label skip;
 5672     __ bsrq(Rdst, Rsrc);
 5673     __ jccb(Assembler::notZero, skip);
 5674     __ movl(Rdst, -1);
 5675     __ bind(skip);
 5676     __ negl(Rdst);
 5677     __ addl(Rdst, BitsPerLong - 1);
 5678   %}
 5679   ins_pipe(ialu_reg);
 5680 %}
 5681 
 5682 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5683   predicate(UseCountTrailingZerosInstruction);
 5684   match(Set dst (CountTrailingZerosI src));
 5685   effect(KILL cr);
 5686 
 5687   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5688   ins_encode %{
 5689     __ tzcntl($dst$$Register, $src$$Register);
 5690   %}
 5691   ins_pipe(ialu_reg);
 5692 %}
 5693 
 5694 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5695   predicate(UseCountTrailingZerosInstruction);
 5696   match(Set dst (CountTrailingZerosI (LoadI src)));
 5697   effect(KILL cr);
 5698   ins_cost(175);
 5699   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5700   ins_encode %{
 5701     __ tzcntl($dst$$Register, $src$$Address);
 5702   %}
 5703   ins_pipe(ialu_reg_mem);
 5704 %}
 5705 
 5706 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5707   predicate(!UseCountTrailingZerosInstruction);
 5708   match(Set dst (CountTrailingZerosI src));
 5709   effect(KILL cr);
 5710 
 5711   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5712             "jnz     done\n\t"
 5713             "movl    $dst, 32\n"
 5714       "done:" %}
 5715   ins_encode %{
 5716     Register Rdst = $dst$$Register;
 5717     Label done;
 5718     __ bsfl(Rdst, $src$$Register);
 5719     __ jccb(Assembler::notZero, done);
 5720     __ movl(Rdst, BitsPerInt);
 5721     __ bind(done);
 5722   %}
 5723   ins_pipe(ialu_reg);
 5724 %}
 5725 
 5726 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5727   predicate(UseCountTrailingZerosInstruction);
 5728   match(Set dst (CountTrailingZerosL src));
 5729   effect(KILL cr);
 5730 
 5731   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5732   ins_encode %{
 5733     __ tzcntq($dst$$Register, $src$$Register);
 5734   %}
 5735   ins_pipe(ialu_reg);
 5736 %}
 5737 
 5738 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5739   predicate(UseCountTrailingZerosInstruction);
 5740   match(Set dst (CountTrailingZerosL (LoadL src)));
 5741   effect(KILL cr);
 5742   ins_cost(175);
 5743   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5744   ins_encode %{
 5745     __ tzcntq($dst$$Register, $src$$Address);
 5746   %}
 5747   ins_pipe(ialu_reg_mem);
 5748 %}
 5749 
 5750 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5751   predicate(!UseCountTrailingZerosInstruction);
 5752   match(Set dst (CountTrailingZerosL src));
 5753   effect(KILL cr);
 5754 
 5755   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5756             "jnz     done\n\t"
 5757             "movl    $dst, 64\n"
 5758       "done:" %}
 5759   ins_encode %{
 5760     Register Rdst = $dst$$Register;
 5761     Label done;
 5762     __ bsfq(Rdst, $src$$Register);
 5763     __ jccb(Assembler::notZero, done);
 5764     __ movl(Rdst, BitsPerLong);
 5765     __ bind(done);
 5766   %}
 5767   ins_pipe(ialu_reg);
 5768 %}
 5769 
 5770 //--------------- Reverse Operation Instructions ----------------
 5771 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5772   predicate(!VM_Version::supports_gfni());
 5773   match(Set dst (ReverseI src));
 5774   effect(TEMP dst, TEMP rtmp, KILL cr);
 5775   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5776   ins_encode %{
 5777     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5778   %}
 5779   ins_pipe( ialu_reg );
 5780 %}
 5781 
 5782 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5783   predicate(VM_Version::supports_gfni());
 5784   match(Set dst (ReverseI src));
 5785   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5786   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5787   ins_encode %{
 5788     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5789   %}
 5790   ins_pipe( ialu_reg );
 5791 %}
 5792 
 5793 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5794   predicate(!VM_Version::supports_gfni());
 5795   match(Set dst (ReverseL src));
 5796   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5797   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5798   ins_encode %{
 5799     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5800   %}
 5801   ins_pipe( ialu_reg );
 5802 %}
 5803 
 5804 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5805   predicate(VM_Version::supports_gfni());
 5806   match(Set dst (ReverseL src));
 5807   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5808   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5809   ins_encode %{
 5810     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5811   %}
 5812   ins_pipe( ialu_reg );
 5813 %}
 5814 
 5815 //---------- Population Count Instructions -------------------------------------
 5816 
 5817 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5818   predicate(UsePopCountInstruction);
 5819   match(Set dst (PopCountI src));
 5820   effect(KILL cr);
 5821 
 5822   format %{ "popcnt  $dst, $src" %}
 5823   ins_encode %{
 5824     __ popcntl($dst$$Register, $src$$Register);
 5825   %}
 5826   ins_pipe(ialu_reg);
 5827 %}
 5828 
 5829 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5830   predicate(UsePopCountInstruction);
 5831   match(Set dst (PopCountI (LoadI mem)));
 5832   effect(KILL cr);
 5833 
 5834   format %{ "popcnt  $dst, $mem" %}
 5835   ins_encode %{
 5836     __ popcntl($dst$$Register, $mem$$Address);
 5837   %}
 5838   ins_pipe(ialu_reg);
 5839 %}
 5840 
 5841 // Note: Long.bitCount(long) returns an int.
 5842 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5843   predicate(UsePopCountInstruction);
 5844   match(Set dst (PopCountL src));
 5845   effect(KILL cr);
 5846 
 5847   format %{ "popcnt  $dst, $src" %}
 5848   ins_encode %{
 5849     __ popcntq($dst$$Register, $src$$Register);
 5850   %}
 5851   ins_pipe(ialu_reg);
 5852 %}
 5853 
 5854 // Note: Long.bitCount(long) returns an int.
 5855 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5856   predicate(UsePopCountInstruction);
 5857   match(Set dst (PopCountL (LoadL mem)));
 5858   effect(KILL cr);
 5859 
 5860   format %{ "popcnt  $dst, $mem" %}
 5861   ins_encode %{
 5862     __ popcntq($dst$$Register, $mem$$Address);
 5863   %}
 5864   ins_pipe(ialu_reg);
 5865 %}
 5866 
 5867 
 5868 //----------MemBar Instructions-----------------------------------------------
 5869 // Memory barrier flavors
 5870 
 5871 instruct membar_acquire()
 5872 %{
 5873   match(MemBarAcquire);
 5874   match(LoadFence);
 5875   ins_cost(0);
 5876 
 5877   size(0);
 5878   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5879   ins_encode();
 5880   ins_pipe(empty);
 5881 %}
 5882 
 5883 instruct membar_acquire_lock()
 5884 %{
 5885   match(MemBarAcquireLock);
 5886   ins_cost(0);
 5887 
 5888   size(0);
 5889   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5890   ins_encode();
 5891   ins_pipe(empty);
 5892 %}
 5893 
 5894 instruct membar_release()
 5895 %{
 5896   match(MemBarRelease);
 5897   match(StoreFence);
 5898   ins_cost(0);
 5899 
 5900   size(0);
 5901   format %{ "MEMBAR-release ! (empty encoding)" %}
 5902   ins_encode();
 5903   ins_pipe(empty);
 5904 %}
 5905 
 5906 instruct membar_release_lock()
 5907 %{
 5908   match(MemBarReleaseLock);
 5909   ins_cost(0);
 5910 
 5911   size(0);
 5912   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5913   ins_encode();
 5914   ins_pipe(empty);
 5915 %}
 5916 
 5917 instruct membar_volatile(rFlagsReg cr) %{
 5918   match(MemBarVolatile);
 5919   effect(KILL cr);
 5920   ins_cost(400);
 5921 
 5922   format %{
 5923     $$template
 5924     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5925   %}
 5926   ins_encode %{
 5927     __ membar(Assembler::StoreLoad);
 5928   %}
 5929   ins_pipe(pipe_slow);
 5930 %}
 5931 
 5932 instruct unnecessary_membar_volatile()
 5933 %{
 5934   match(MemBarVolatile);
 5935   predicate(Matcher::post_store_load_barrier(n));
 5936   ins_cost(0);
 5937 
 5938   size(0);
 5939   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5940   ins_encode();
 5941   ins_pipe(empty);
 5942 %}
 5943 
 5944 instruct membar_storestore() %{
 5945   match(MemBarStoreStore);
 5946   match(StoreStoreFence);
 5947   ins_cost(0);
 5948 
 5949   size(0);
 5950   format %{ "MEMBAR-storestore (empty encoding)" %}
 5951   ins_encode( );
 5952   ins_pipe(empty);
 5953 %}
 5954 
 5955 //----------Move Instructions--------------------------------------------------
 5956 
 5957 instruct castX2P(rRegP dst, rRegL src)
 5958 %{
 5959   match(Set dst (CastX2P src));
 5960 
 5961   format %{ "movq    $dst, $src\t# long->ptr" %}
 5962   ins_encode %{
 5963     if ($dst$$reg != $src$$reg) {
 5964       __ movptr($dst$$Register, $src$$Register);
 5965     }
 5966   %}
 5967   ins_pipe(ialu_reg_reg); // XXX
 5968 %}
 5969 
 5970 instruct castI2N(rRegN dst, rRegI src)
 5971 %{
 5972   match(Set dst (CastI2N src));
 5973 
 5974   format %{ "movq    $dst, $src\t# int -> narrow ptr" %}
 5975   ins_encode %{
 5976     if ($dst$$reg != $src$$reg) {
 5977       __ movl($dst$$Register, $src$$Register);
 5978     }
 5979   %}
 5980   ins_pipe(ialu_reg_reg); // XXX
 5981 %}
 5982 
 5983 instruct castN2X(rRegL dst, rRegN src)
 5984 %{
 5985   match(Set dst (CastP2X src));
 5986 
 5987   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5988   ins_encode %{
 5989     if ($dst$$reg != $src$$reg) {
 5990       __ movptr($dst$$Register, $src$$Register);
 5991     }
 5992   %}
 5993   ins_pipe(ialu_reg_reg); // XXX
 5994 %}
 5995 
 5996 instruct castP2X(rRegL dst, rRegP src)
 5997 %{
 5998   match(Set dst (CastP2X src));
 5999 
 6000   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6001   ins_encode %{
 6002     if ($dst$$reg != $src$$reg) {
 6003       __ movptr($dst$$Register, $src$$Register);
 6004     }
 6005   %}
 6006   ins_pipe(ialu_reg_reg); // XXX
 6007 %}
 6008 
 6009 // Convert oop into int for vectors alignment masking
 6010 instruct convP2I(rRegI dst, rRegP src)
 6011 %{
 6012   match(Set dst (ConvL2I (CastP2X src)));
 6013 
 6014   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6015   ins_encode %{
 6016     __ movl($dst$$Register, $src$$Register);
 6017   %}
 6018   ins_pipe(ialu_reg_reg); // XXX
 6019 %}
 6020 
 6021 // Convert compressed oop into int for vectors alignment masking
 6022 // in case of 32bit oops (heap < 4Gb).
 6023 instruct convN2I(rRegI dst, rRegN src)
 6024 %{
 6025   predicate(CompressedOops::shift() == 0);
 6026   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6027 
 6028   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6029   ins_encode %{
 6030     __ movl($dst$$Register, $src$$Register);
 6031   %}
 6032   ins_pipe(ialu_reg_reg); // XXX
 6033 %}
 6034 
 6035 // Convert oop pointer into compressed form
 6036 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6037   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6038   match(Set dst (EncodeP src));
 6039   effect(KILL cr);
 6040   format %{ "encode_heap_oop $dst,$src" %}
 6041   ins_encode %{
 6042     Register s = $src$$Register;
 6043     Register d = $dst$$Register;
 6044     if (s != d) {
 6045       __ movq(d, s);
 6046     }
 6047     __ encode_heap_oop(d);
 6048   %}
 6049   ins_pipe(ialu_reg_long);
 6050 %}
 6051 
 6052 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6053   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6054   match(Set dst (EncodeP src));
 6055   effect(KILL cr);
 6056   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6057   ins_encode %{
 6058     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6059   %}
 6060   ins_pipe(ialu_reg_long);
 6061 %}
 6062 
 6063 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6064   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6065             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6066   match(Set dst (DecodeN src));
 6067   effect(KILL cr);
 6068   format %{ "decode_heap_oop $dst,$src" %}
 6069   ins_encode %{
 6070     Register s = $src$$Register;
 6071     Register d = $dst$$Register;
 6072     if (s != d) {
 6073       __ movq(d, s);
 6074     }
 6075     __ decode_heap_oop(d);
 6076   %}
 6077   ins_pipe(ialu_reg_long);
 6078 %}
 6079 
 6080 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6081   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6082             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6083   match(Set dst (DecodeN src));
 6084   effect(KILL cr);
 6085   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6086   ins_encode %{
 6087     Register s = $src$$Register;
 6088     Register d = $dst$$Register;
 6089     if (s != d) {
 6090       __ decode_heap_oop_not_null(d, s);
 6091     } else {
 6092       __ decode_heap_oop_not_null(d);
 6093     }
 6094   %}
 6095   ins_pipe(ialu_reg_long);
 6096 %}
 6097 
 6098 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6099   match(Set dst (EncodePKlass src));
 6100   effect(TEMP dst, KILL cr);
 6101   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6102   ins_encode %{
 6103     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6104   %}
 6105   ins_pipe(ialu_reg_long);
 6106 %}
 6107 
 6108 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6109   match(Set dst (DecodeNKlass src));
 6110   effect(TEMP dst, KILL cr);
 6111   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6112   ins_encode %{
 6113     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6114   %}
 6115   ins_pipe(ialu_reg_long);
 6116 %}
 6117 
 6118 //----------Conditional Move---------------------------------------------------
 6119 // Jump
 6120 // dummy instruction for generating temp registers
 6121 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6122   match(Jump (LShiftL switch_val shift));
 6123   ins_cost(350);
 6124   predicate(false);
 6125   effect(TEMP dest);
 6126 
 6127   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6128             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6129   ins_encode %{
 6130     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6131     // to do that and the compiler is using that register as one it can allocate.
 6132     // So we build it all by hand.
 6133     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6134     // ArrayAddress dispatch(table, index);
 6135     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6136     __ lea($dest$$Register, $constantaddress);
 6137     __ jmp(dispatch);
 6138   %}
 6139   ins_pipe(pipe_jmp);
 6140 %}
 6141 
 6142 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6143   match(Jump (AddL (LShiftL switch_val shift) offset));
 6144   ins_cost(350);
 6145   effect(TEMP dest);
 6146 
 6147   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6148             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6149   ins_encode %{
 6150     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6151     // to do that and the compiler is using that register as one it can allocate.
 6152     // So we build it all by hand.
 6153     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6154     // ArrayAddress dispatch(table, index);
 6155     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6156     __ lea($dest$$Register, $constantaddress);
 6157     __ jmp(dispatch);
 6158   %}
 6159   ins_pipe(pipe_jmp);
 6160 %}
 6161 
 6162 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6163   match(Jump switch_val);
 6164   ins_cost(350);
 6165   effect(TEMP dest);
 6166 
 6167   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6168             "jmp     [$dest + $switch_val]\n\t" %}
 6169   ins_encode %{
 6170     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6171     // to do that and the compiler is using that register as one it can allocate.
 6172     // So we build it all by hand.
 6173     // Address index(noreg, switch_reg, Address::times_1);
 6174     // ArrayAddress dispatch(table, index);
 6175     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6176     __ lea($dest$$Register, $constantaddress);
 6177     __ jmp(dispatch);
 6178   %}
 6179   ins_pipe(pipe_jmp);
 6180 %}
 6181 
 6182 // Conditional move
 6183 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6184 %{
 6185   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6186   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6187 
 6188   ins_cost(100); // XXX
 6189   format %{ "setbn$cop $dst\t# signed, int" %}
 6190   ins_encode %{
 6191     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6192     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6193   %}
 6194   ins_pipe(ialu_reg);
 6195 %}
 6196 
 6197 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6198 %{
 6199   predicate(!UseAPX);
 6200   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6201 
 6202   ins_cost(200); // XXX
 6203   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6204   ins_encode %{
 6205     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6206   %}
 6207   ins_pipe(pipe_cmov_reg);
 6208 %}
 6209 
 6210 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6211 %{
 6212   predicate(UseAPX);
 6213   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6214 
 6215   ins_cost(200);
 6216   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6217   ins_encode %{
 6218     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6219   %}
 6220   ins_pipe(pipe_cmov_reg);
 6221 %}
 6222 
 6223 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6224 %{
 6225   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6226   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6227 
 6228   ins_cost(100); // XXX
 6229   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6230   ins_encode %{
 6231     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6232     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6233   %}
 6234   ins_pipe(ialu_reg);
 6235 %}
 6236 
 6237 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6238   predicate(!UseAPX);
 6239   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6240 
 6241   ins_cost(200); // XXX
 6242   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6243   ins_encode %{
 6244     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6245   %}
 6246   ins_pipe(pipe_cmov_reg);
 6247 %}
 6248 
 6249 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6250   predicate(UseAPX);
 6251   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6252 
 6253   ins_cost(200);
 6254   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6255   ins_encode %{
 6256     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6257   %}
 6258   ins_pipe(pipe_cmov_reg);
 6259 %}
 6260 
 6261 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6262 %{
 6263   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6264   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6265 
 6266   ins_cost(100); // XXX
 6267   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6268   ins_encode %{
 6269     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6270     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6271   %}
 6272   ins_pipe(ialu_reg);
 6273 %}
 6274 
 6275 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6276   predicate(!UseAPX);
 6277   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6278   ins_cost(200);
 6279   expand %{
 6280     cmovI_regU(cop, cr, dst, src);
 6281   %}
 6282 %}
 6283 
 6284 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6285   predicate(UseAPX);
 6286   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6287   ins_cost(200);
 6288   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6289   ins_encode %{
 6290     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6291   %}
 6292   ins_pipe(pipe_cmov_reg);
 6293 %}
 6294 
 6295 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6296   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6297   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6298 
 6299   ins_cost(200); // XXX
 6300   format %{ "cmovpl  $dst, $src\n\t"
 6301             "cmovnel $dst, $src" %}
 6302   ins_encode %{
 6303     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6304     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6305   %}
 6306   ins_pipe(pipe_cmov_reg);
 6307 %}
 6308 
 6309 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6310   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6311   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6312 
 6313   ins_cost(200);
 6314   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6315             "ecmovnel $dst, $src1, $src2" %}
 6316   ins_encode %{
 6317     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6318     __ ecmovl(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6319   %}
 6320   ins_pipe(pipe_cmov_reg);
 6321 %}
 6322 
 6323 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6324 // inputs of the CMove
 6325 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6326   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6327   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6328 
 6329   ins_cost(200); // XXX
 6330   format %{ "cmovpl  $dst, $src\n\t"
 6331             "cmovnel $dst, $src" %}
 6332   ins_encode %{
 6333     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6334     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6335   %}
 6336   ins_pipe(pipe_cmov_reg);
 6337 %}
 6338 
 6339 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6340 // and parity flag bit is set if any of the operand is a NaN.
 6341 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6342   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6343   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6344 
 6345   ins_cost(200);
 6346   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6347             "ecmovnel $dst, $src1, $src2" %}
 6348   ins_encode %{
 6349     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6350     __ ecmovl(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6351   %}
 6352   ins_pipe(pipe_cmov_reg);
 6353 %}
 6354 
 6355 // Conditional move
 6356 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6357   predicate(!UseAPX);
 6358   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6359 
 6360   ins_cost(250); // XXX
 6361   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6362   ins_encode %{
 6363     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6364   %}
 6365   ins_pipe(pipe_cmov_mem);
 6366 %}
 6367 
 6368 // Conditional move
 6369 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6370 %{
 6371   predicate(UseAPX);
 6372   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6373 
 6374   ins_cost(250);
 6375   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6376   ins_encode %{
 6377     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6378   %}
 6379   ins_pipe(pipe_cmov_mem);
 6380 %}
 6381 
 6382 // Conditional move
 6383 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6384 %{
 6385   predicate(!UseAPX);
 6386   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6387 
 6388   ins_cost(250); // XXX
 6389   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6390   ins_encode %{
 6391     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6392   %}
 6393   ins_pipe(pipe_cmov_mem);
 6394 %}
 6395 
 6396 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6397   predicate(!UseAPX);
 6398   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6399   ins_cost(250);
 6400   expand %{
 6401     cmovI_memU(cop, cr, dst, src);
 6402   %}
 6403 %}
 6404 
 6405 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6406 %{
 6407   predicate(UseAPX);
 6408   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6409 
 6410   ins_cost(250);
 6411   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6412   ins_encode %{
 6413     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6414   %}
 6415   ins_pipe(pipe_cmov_mem);
 6416 %}
 6417 
 6418 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2) 
 6419 %{
 6420   predicate(UseAPX);
 6421   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6422   ins_cost(250);
 6423   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6424   ins_encode %{
 6425     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6426   %}
 6427   ins_pipe(pipe_cmov_mem);
 6428 %}
 6429 
 6430 // Conditional move
 6431 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6432 %{
 6433   predicate(!UseAPX);
 6434   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6435 
 6436   ins_cost(200); // XXX
 6437   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6438   ins_encode %{
 6439     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6440   %}
 6441   ins_pipe(pipe_cmov_reg);
 6442 %}
 6443 
 6444 // Conditional move ndd
 6445 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6446 %{
 6447   predicate(UseAPX);
 6448   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6449 
 6450   ins_cost(200);
 6451   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6452   ins_encode %{
 6453     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6454   %}
 6455   ins_pipe(pipe_cmov_reg);
 6456 %}
 6457 
 6458 // Conditional move
 6459 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6460 %{
 6461   predicate(!UseAPX);
 6462   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6463 
 6464   ins_cost(200); // XXX
 6465   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6466   ins_encode %{
 6467     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6468   %}
 6469   ins_pipe(pipe_cmov_reg);
 6470 %}
 6471 
 6472 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6473   predicate(!UseAPX);
 6474   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6475   ins_cost(200);
 6476   expand %{
 6477     cmovN_regU(cop, cr, dst, src);
 6478   %}
 6479 %}
 6480 
 6481 // Conditional move ndd
 6482 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6483 %{
 6484   predicate(UseAPX);
 6485   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6486 
 6487   ins_cost(200);
 6488   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6489   ins_encode %{
 6490     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6491   %}
 6492   ins_pipe(pipe_cmov_reg);
 6493 %}
 6494 
 6495 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6496   predicate(UseAPX);
 6497   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6498   ins_cost(200);
 6499   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6500   ins_encode %{
 6501     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6502   %}
 6503   ins_pipe(pipe_cmov_reg);
 6504 %}
 6505 
 6506 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6507   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6508   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6509 
 6510   ins_cost(200); // XXX
 6511   format %{ "cmovpl  $dst, $src\n\t"
 6512             "cmovnel $dst, $src" %}
 6513   ins_encode %{
 6514     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6515     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6516   %}
 6517   ins_pipe(pipe_cmov_reg);
 6518 %}
 6519 
 6520 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6521 // inputs of the CMove
 6522 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6523   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6524   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6525 
 6526   ins_cost(200); // XXX
 6527   format %{ "cmovpl  $dst, $src\n\t"
 6528             "cmovnel $dst, $src" %}
 6529   ins_encode %{
 6530     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6531     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6532   %}
 6533   ins_pipe(pipe_cmov_reg);
 6534 %}
 6535 
 6536 // Conditional move
 6537 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6538 %{
 6539   predicate(!UseAPX);
 6540   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6541 
 6542   ins_cost(200); // XXX
 6543   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6544   ins_encode %{
 6545     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6546   %}
 6547   ins_pipe(pipe_cmov_reg);  // XXX
 6548 %}
 6549 
 6550 // Conditional move ndd
 6551 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6552 %{
 6553   predicate(UseAPX);
 6554   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6555 
 6556   ins_cost(200);
 6557   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6558   ins_encode %{
 6559     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6560   %}
 6561   ins_pipe(pipe_cmov_reg);
 6562 %}
 6563 
 6564 // Conditional move
 6565 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6566 %{
 6567   predicate(!UseAPX);
 6568   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6569 
 6570   ins_cost(200); // XXX
 6571   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6572   ins_encode %{
 6573     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6574   %}
 6575   ins_pipe(pipe_cmov_reg); // XXX
 6576 %}
 6577 
 6578 // Conditional move ndd
 6579 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6580 %{
 6581   predicate(UseAPX);
 6582   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6583 
 6584   ins_cost(200);
 6585   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6586   ins_encode %{
 6587     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6588   %}
 6589   ins_pipe(pipe_cmov_reg);
 6590 %}
 6591 
 6592 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6593   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6594   ins_cost(200);
 6595   expand %{
 6596     cmovP_regU(cop, cr, dst, src);
 6597   %}
 6598 %}
 6599 
 6600 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6601   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6602   ins_cost(200);
 6603   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6604   ins_encode %{
 6605     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6606   %}
 6607   ins_pipe(pipe_cmov_reg);
 6608 %}
 6609 
 6610 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6611   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6612   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6613 
 6614   ins_cost(200); // XXX
 6615   format %{ "cmovpq  $dst, $src\n\t"
 6616             "cmovneq $dst, $src" %}
 6617   ins_encode %{
 6618     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6619     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6620   %}
 6621   ins_pipe(pipe_cmov_reg);
 6622 %}
 6623 
 6624 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6625   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6626   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6627 
 6628   ins_cost(200);
 6629   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6630             "ecmovneq $dst, $src1, $src2" %}
 6631   ins_encode %{
 6632     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6633     __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6634   %}
 6635   ins_pipe(pipe_cmov_reg);
 6636 %}
 6637 
 6638 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6639 // inputs of the CMove
 6640 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6641   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6642   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6643 
 6644   ins_cost(200); // XXX
 6645   format %{ "cmovpq  $dst, $src\n\t"
 6646             "cmovneq $dst, $src" %}
 6647   ins_encode %{
 6648     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6649     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6650   %}
 6651   ins_pipe(pipe_cmov_reg);
 6652 %}
 6653 
 6654 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6655   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6656   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6657 
 6658   ins_cost(200);
 6659   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6660             "ecmovneq $dst, $src1, $src2" %}
 6661   ins_encode %{
 6662     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6663     __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6664   %}
 6665   ins_pipe(pipe_cmov_reg);
 6666 %}
 6667 
 6668 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6669 %{
 6670   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6671   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6672 
 6673   ins_cost(100); // XXX
 6674   format %{ "setbn$cop $dst\t# signed, long" %}
 6675   ins_encode %{
 6676     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6677     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6678   %}
 6679   ins_pipe(ialu_reg);
 6680 %}
 6681 
 6682 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6683 %{
 6684   predicate(!UseAPX);
 6685   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6686 
 6687   ins_cost(200); // XXX
 6688   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6689   ins_encode %{
 6690     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6691   %}
 6692   ins_pipe(pipe_cmov_reg);  // XXX
 6693 %}
 6694 
 6695 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6696 %{
 6697   predicate(UseAPX);
 6698   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6699 
 6700   ins_cost(200);
 6701   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6702   ins_encode %{
 6703     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6704   %}
 6705   ins_pipe(pipe_cmov_reg);
 6706 %}
 6707 
 6708 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6709 %{
 6710   predicate(!UseAPX);
 6711   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6712 
 6713   ins_cost(200); // XXX
 6714   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6715   ins_encode %{
 6716     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6717   %}
 6718   ins_pipe(pipe_cmov_mem);  // XXX
 6719 %}
 6720 
 6721 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6722 %{
 6723   predicate(UseAPX);
 6724   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6725 
 6726   ins_cost(200);
 6727   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6728   ins_encode %{
 6729     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6730   %}
 6731   ins_pipe(pipe_cmov_mem);
 6732 %}
 6733 
 6734 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6735 %{
 6736   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6737   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6738 
 6739   ins_cost(100); // XXX
 6740   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6741   ins_encode %{
 6742     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6743     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6744   %}
 6745   ins_pipe(ialu_reg);
 6746 %}
 6747 
 6748 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6749 %{
 6750   predicate(!UseAPX);
 6751   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6752 
 6753   ins_cost(200); // XXX
 6754   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6755   ins_encode %{
 6756     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6757   %}
 6758   ins_pipe(pipe_cmov_reg); // XXX
 6759 %}
 6760 
 6761 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6762 %{
 6763   predicate(UseAPX);
 6764   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6765 
 6766   ins_cost(200);
 6767   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6768   ins_encode %{
 6769     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6770   %}
 6771   ins_pipe(pipe_cmov_reg);
 6772 %}
 6773 
 6774 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6775 %{
 6776   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6777   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6778 
 6779   ins_cost(100); // XXX
 6780   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6781   ins_encode %{
 6782     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6783     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6784   %}
 6785   ins_pipe(ialu_reg);
 6786 %}
 6787 
 6788 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6789   predicate(!UseAPX);
 6790   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6791   ins_cost(200);
 6792   expand %{
 6793     cmovL_regU(cop, cr, dst, src);
 6794   %}
 6795 %}
 6796 
 6797 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2) 
 6798 %{
 6799   predicate(UseAPX);
 6800   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6801   ins_cost(200);
 6802   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6803   ins_encode %{
 6804     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6805   %}
 6806   ins_pipe(pipe_cmov_reg);
 6807 %}
 6808 
 6809 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6810   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6811   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6812 
 6813   ins_cost(200); // XXX
 6814   format %{ "cmovpq  $dst, $src\n\t"
 6815             "cmovneq $dst, $src" %}
 6816   ins_encode %{
 6817     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6818     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6819   %}
 6820   ins_pipe(pipe_cmov_reg);
 6821 %}
 6822 
 6823 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6824   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6825   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6826 
 6827   ins_cost(200);
 6828   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6829             "ecmovneq $dst, $src1, $src2" %}
 6830   ins_encode %{
 6831     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6832     __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6833   %}
 6834   ins_pipe(pipe_cmov_reg);
 6835 %}
 6836 
 6837 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6838 // inputs of the CMove
 6839 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6840   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6841   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6842 
 6843   ins_cost(200); // XXX
 6844   format %{ "cmovpq  $dst, $src\n\t"
 6845             "cmovneq $dst, $src" %}
 6846   ins_encode %{
 6847     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6848     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6849   %}
 6850   ins_pipe(pipe_cmov_reg);
 6851 %}
 6852 
 6853 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6854   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6855   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6856 
 6857   ins_cost(200);
 6858   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6859             "ecmovneq $dst, $src1, $src2" %}
 6860   ins_encode %{
 6861     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6862     __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
 6863   %}
 6864   ins_pipe(pipe_cmov_reg);
 6865 %}
 6866 
 6867 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6868 %{
 6869   predicate(!UseAPX);
 6870   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6871 
 6872   ins_cost(200); // XXX
 6873   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6874   ins_encode %{
 6875     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6876   %}
 6877   ins_pipe(pipe_cmov_mem); // XXX
 6878 %}
 6879 
 6880 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6881   predicate(!UseAPX);
 6882   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6883   ins_cost(200);
 6884   expand %{
 6885     cmovL_memU(cop, cr, dst, src);
 6886   %}
 6887 %}
 6888 
 6889 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6890 %{
 6891   predicate(UseAPX);
 6892   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6893 
 6894   ins_cost(200);
 6895   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6896   ins_encode %{
 6897     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6898   %}
 6899   ins_pipe(pipe_cmov_mem);
 6900 %}
 6901 
 6902 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2) 
 6903 %{
 6904   predicate(UseAPX);
 6905   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6906   ins_cost(200);
 6907   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6908   ins_encode %{
 6909     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6910   %}
 6911   ins_pipe(pipe_cmov_mem);
 6912 %}
 6913 
 6914 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6915 %{
 6916   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6917 
 6918   ins_cost(200); // XXX
 6919   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6920             "movss     $dst, $src\n"
 6921     "skip:" %}
 6922   ins_encode %{
 6923     Label Lskip;
 6924     // Invert sense of branch from sense of CMOV
 6925     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6926     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6927     __ bind(Lskip);
 6928   %}
 6929   ins_pipe(pipe_slow);
 6930 %}
 6931 
 6932 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6933 %{
 6934   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6935 
 6936   ins_cost(200); // XXX
 6937   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6938             "movss     $dst, $src\n"
 6939     "skip:" %}
 6940   ins_encode %{
 6941     Label Lskip;
 6942     // Invert sense of branch from sense of CMOV
 6943     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6944     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6945     __ bind(Lskip);
 6946   %}
 6947   ins_pipe(pipe_slow);
 6948 %}
 6949 
 6950 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6951   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6952   ins_cost(200);
 6953   expand %{
 6954     cmovF_regU(cop, cr, dst, src);
 6955   %}
 6956 %}
 6957 
 6958 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6959 %{
 6960   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6961 
 6962   ins_cost(200); // XXX
 6963   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6964             "movsd     $dst, $src\n"
 6965     "skip:" %}
 6966   ins_encode %{
 6967     Label Lskip;
 6968     // Invert sense of branch from sense of CMOV
 6969     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6970     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6971     __ bind(Lskip);
 6972   %}
 6973   ins_pipe(pipe_slow);
 6974 %}
 6975 
 6976 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6977 %{
 6978   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6979 
 6980   ins_cost(200); // XXX
 6981   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6982             "movsd     $dst, $src\n"
 6983     "skip:" %}
 6984   ins_encode %{
 6985     Label Lskip;
 6986     // Invert sense of branch from sense of CMOV
 6987     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6988     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6989     __ bind(Lskip);
 6990   %}
 6991   ins_pipe(pipe_slow);
 6992 %}
 6993 
 6994 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6995   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6996   ins_cost(200);
 6997   expand %{
 6998     cmovD_regU(cop, cr, dst, src);
 6999   %}
 7000 %}
 7001 
 7002 //----------Arithmetic Instructions--------------------------------------------
 7003 //----------Addition Instructions----------------------------------------------
 7004 
 7005 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7006 %{
 7007   predicate(!UseAPX);
 7008   match(Set dst (AddI dst src));
 7009   effect(KILL cr);
 7010   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7011   format %{ "addl    $dst, $src\t# int" %}
 7012   ins_encode %{
 7013     __ addl($dst$$Register, $src$$Register);
 7014   %}
 7015   ins_pipe(ialu_reg_reg);
 7016 %}
 7017 
 7018 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 7019 %{
 7020   predicate(UseAPX);
 7021   match(Set dst (AddI src1 src2));
 7022   effect(KILL cr);
 7023   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7024 
 7025   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7026   ins_encode %{
 7027     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 7028   %}
 7029   ins_pipe(ialu_reg_reg);
 7030 %}
 7031 
 7032 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7033 %{
 7034   predicate(!UseAPX);
 7035   match(Set dst (AddI dst src));
 7036   effect(KILL cr);
 7037   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7038 
 7039   format %{ "addl    $dst, $src\t# int" %}
 7040   ins_encode %{
 7041     __ addl($dst$$Register, $src$$constant);
 7042   %}
 7043   ins_pipe( ialu_reg );
 7044 %}
 7045 
 7046 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7047 %{
 7048   predicate(UseAPX);
 7049   match(Set dst (AddI src1 src2));
 7050   effect(KILL cr);
 7051   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7052 
 7053   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7054   ins_encode %{
 7055     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7056   %}
 7057   ins_pipe( ialu_reg );
 7058 %}
 7059 
 7060 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7061 %{
 7062   predicate(UseAPX);
 7063   match(Set dst (AddI (LoadI src1) src2));
 7064   effect(KILL cr);
 7065   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7066 
 7067   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7068   ins_encode %{
 7069     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7070   %}
 7071   ins_pipe( ialu_reg );
 7072 %}
 7073 
 7074 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7075 %{
 7076   predicate(!UseAPX);
 7077   match(Set dst (AddI dst (LoadI src)));
 7078   effect(KILL cr);
 7079   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7080 
 7081   ins_cost(150); // XXX
 7082   format %{ "addl    $dst, $src\t# int" %}
 7083   ins_encode %{
 7084     __ addl($dst$$Register, $src$$Address);
 7085   %}
 7086   ins_pipe(ialu_reg_mem);
 7087 %}
 7088 
 7089 instruct addI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 7090 %{
 7091   predicate(UseAPX);
 7092   match(Set dst (AddI (LoadI src1) src2));
 7093   effect(KILL cr);
 7094   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7095 
 7096   ins_cost(150);
 7097   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7098   ins_encode %{
 7099     __ eaddl($dst$$Register, $src1$$Address, $src2$$Register, false);
 7100   %}
 7101   ins_pipe(ialu_reg_mem);
 7102 %}
 7103 
 7104 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7105 %{
 7106   predicate(UseAPX);
 7107   match(Set dst (AddI src1 (LoadI src2)));
 7108   effect(KILL cr);
 7109   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7110 
 7111   ins_cost(150);
 7112   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7113   ins_encode %{
 7114     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7115   %}
 7116   ins_pipe(ialu_reg_mem);
 7117 %}
 7118 
 7119 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7120 %{
 7121   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7122   effect(KILL cr);
 7123   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7124 
 7125   ins_cost(150); // XXX
 7126   format %{ "addl    $dst, $src\t# int" %}
 7127   ins_encode %{
 7128     __ addl($dst$$Address, $src$$Register);
 7129   %}
 7130   ins_pipe(ialu_mem_reg);
 7131 %}
 7132 
 7133 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7134 %{
 7135   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7136   effect(KILL cr);
 7137   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7138 
 7139 
 7140   ins_cost(125); // XXX
 7141   format %{ "addl    $dst, $src\t# int" %}
 7142   ins_encode %{
 7143     __ addl($dst$$Address, $src$$constant);
 7144   %}
 7145   ins_pipe(ialu_mem_imm);
 7146 %}
 7147 
 7148 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7149 %{
 7150   predicate(!UseAPX && UseIncDec);
 7151   match(Set dst (AddI dst src));
 7152   effect(KILL cr);
 7153 
 7154   format %{ "incl    $dst\t# int" %}
 7155   ins_encode %{
 7156     __ incrementl($dst$$Register);
 7157   %}
 7158   ins_pipe(ialu_reg);
 7159 %}
 7160 
 7161 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7162 %{
 7163   predicate(UseAPX && UseIncDec);
 7164   match(Set dst (AddI src val));
 7165   effect(KILL cr);
 7166 
 7167   format %{ "eincl    $dst, $src\t# int ndd" %}
 7168   ins_encode %{
 7169     __ eincl($dst$$Register, $src$$Register, false);
 7170   %}
 7171   ins_pipe(ialu_reg);
 7172 %}
 7173 
 7174 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7175 %{
 7176   predicate(UseAPX && UseIncDec);
 7177   match(Set dst (AddI (LoadI src) val));
 7178   effect(KILL cr);
 7179 
 7180   format %{ "eincl    $dst, $src\t# int ndd" %}
 7181   ins_encode %{
 7182     __ eincl($dst$$Register, $src$$Address, false);
 7183   %}
 7184   ins_pipe(ialu_reg);
 7185 %}
 7186 
 7187 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7188 %{
 7189   predicate(UseIncDec);
 7190   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7191   effect(KILL cr);
 7192 
 7193   ins_cost(125); // XXX
 7194   format %{ "incl    $dst\t# int" %}
 7195   ins_encode %{
 7196     __ incrementl($dst$$Address);
 7197   %}
 7198   ins_pipe(ialu_mem_imm);
 7199 %}
 7200 
 7201 // XXX why does that use AddI
 7202 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7203 %{
 7204   predicate(!UseAPX && UseIncDec);
 7205   match(Set dst (AddI dst src));
 7206   effect(KILL cr);
 7207 
 7208   format %{ "decl    $dst\t# int" %}
 7209   ins_encode %{
 7210     __ decrementl($dst$$Register);
 7211   %}
 7212   ins_pipe(ialu_reg);
 7213 %}
 7214 
 7215 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7216 %{
 7217   predicate(UseAPX && UseIncDec);
 7218   match(Set dst (AddI src val));
 7219   effect(KILL cr);
 7220 
 7221   format %{ "edecl    $dst, $src\t# int ndd" %}
 7222   ins_encode %{
 7223     __ edecl($dst$$Register, $src$$Register, false);
 7224   %}
 7225   ins_pipe(ialu_reg);
 7226 %}
 7227 
 7228 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7229 %{
 7230   predicate(UseAPX && UseIncDec);
 7231   match(Set dst (AddI (LoadI src) val));
 7232   effect(KILL cr);
 7233 
 7234   format %{ "edecl    $dst, $src\t# int ndd" %}
 7235   ins_encode %{
 7236     __ edecl($dst$$Register, $src$$Address, false);
 7237   %}
 7238   ins_pipe(ialu_reg);
 7239 %}
 7240 
 7241 // XXX why does that use AddI
 7242 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7243 %{
 7244   predicate(UseIncDec);
 7245   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7246   effect(KILL cr);
 7247 
 7248   ins_cost(125); // XXX
 7249   format %{ "decl    $dst\t# int" %}
 7250   ins_encode %{
 7251     __ decrementl($dst$$Address);
 7252   %}
 7253   ins_pipe(ialu_mem_imm);
 7254 %}
 7255 
 7256 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7257 %{
 7258   predicate(VM_Version::supports_fast_2op_lea());
 7259   match(Set dst (AddI (LShiftI index scale) disp));
 7260 
 7261   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7262   ins_encode %{
 7263     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7264     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7265   %}
 7266   ins_pipe(ialu_reg_reg);
 7267 %}
 7268 
 7269 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7270 %{
 7271   predicate(VM_Version::supports_fast_3op_lea());
 7272   match(Set dst (AddI (AddI base index) disp));
 7273 
 7274   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7275   ins_encode %{
 7276     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7277   %}
 7278   ins_pipe(ialu_reg_reg);
 7279 %}
 7280 
 7281 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7282 %{
 7283   predicate(VM_Version::supports_fast_2op_lea());
 7284   match(Set dst (AddI base (LShiftI index scale)));
 7285 
 7286   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7287   ins_encode %{
 7288     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7289     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7290   %}
 7291   ins_pipe(ialu_reg_reg);
 7292 %}
 7293 
 7294 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7295 %{
 7296   predicate(VM_Version::supports_fast_3op_lea());
 7297   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7298 
 7299   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7300   ins_encode %{
 7301     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7302     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7303   %}
 7304   ins_pipe(ialu_reg_reg);
 7305 %}
 7306 
 7307 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7308 %{
 7309   predicate(!UseAPX);
 7310   match(Set dst (AddL dst src));
 7311   effect(KILL cr);
 7312   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7313 
 7314   format %{ "addq    $dst, $src\t# long" %}
 7315   ins_encode %{
 7316     __ addq($dst$$Register, $src$$Register);
 7317   %}
 7318   ins_pipe(ialu_reg_reg);
 7319 %}
 7320 
 7321 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7322 %{
 7323   predicate(UseAPX);
 7324   match(Set dst (AddL src1 src2));
 7325   effect(KILL cr);
 7326   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7327 
 7328   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7329   ins_encode %{
 7330     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7331   %}
 7332   ins_pipe(ialu_reg_reg);
 7333 %}
 7334 
 7335 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7336 %{
 7337   predicate(!UseAPX);
 7338   match(Set dst (AddL dst src));
 7339   effect(KILL cr);
 7340   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7341 
 7342   format %{ "addq    $dst, $src\t# long" %}
 7343   ins_encode %{
 7344     __ addq($dst$$Register, $src$$constant);
 7345   %}
 7346   ins_pipe( ialu_reg );
 7347 %}
 7348 
 7349 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7350 %{
 7351   predicate(UseAPX);
 7352   match(Set dst (AddL src1 src2));
 7353   effect(KILL cr);
 7354   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7355 
 7356   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7357   ins_encode %{
 7358     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7359   %}
 7360   ins_pipe( ialu_reg );
 7361 %}
 7362 
 7363 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7364 %{
 7365   predicate(UseAPX);
 7366   match(Set dst (AddL (LoadL src1) src2));
 7367   effect(KILL cr);
 7368   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7369 
 7370   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7371   ins_encode %{
 7372     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7373   %}
 7374   ins_pipe( ialu_reg );
 7375 %}
 7376 
 7377 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7378 %{
 7379   predicate(!UseAPX);
 7380   match(Set dst (AddL dst (LoadL src)));
 7381   effect(KILL cr);
 7382   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7383 
 7384   ins_cost(150); // XXX
 7385   format %{ "addq    $dst, $src\t# long" %}
 7386   ins_encode %{
 7387     __ addq($dst$$Register, $src$$Address);
 7388   %}
 7389   ins_pipe(ialu_reg_mem);
 7390 %}
 7391 
 7392 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7393 %{
 7394   predicate(UseAPX);
 7395   match(Set dst (AddL src1 (LoadL src2)));
 7396   effect(KILL cr);
 7397   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7398 
 7399   ins_cost(150);
 7400   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7401   ins_encode %{
 7402     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7403   %}
 7404   ins_pipe(ialu_reg_mem);
 7405 %}
 7406 
 7407 instruct addL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 7408 %{
 7409   predicate(UseAPX);
 7410   match(Set dst (AddL (LoadL src1) src2));
 7411   effect(KILL cr);
 7412   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7413 
 7414   ins_cost(150);
 7415   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7416   ins_encode %{
 7417     __ eaddq($dst$$Register, $src1$$Address, $src2$$Register, false);
 7418   %}
 7419   ins_pipe(ialu_reg_mem);
 7420 %}
 7421 
 7422 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7423 %{
 7424   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7425   effect(KILL cr);
 7426   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7427 
 7428   ins_cost(150); // XXX
 7429   format %{ "addq    $dst, $src\t# long" %}
 7430   ins_encode %{
 7431     __ addq($dst$$Address, $src$$Register);
 7432   %}
 7433   ins_pipe(ialu_mem_reg);
 7434 %}
 7435 
 7436 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7437 %{
 7438   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7439   effect(KILL cr);
 7440   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7441 
 7442   ins_cost(125); // XXX
 7443   format %{ "addq    $dst, $src\t# long" %}
 7444   ins_encode %{
 7445     __ addq($dst$$Address, $src$$constant);
 7446   %}
 7447   ins_pipe(ialu_mem_imm);
 7448 %}
 7449 
 7450 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
 7451 %{
 7452   predicate(!UseAPX && UseIncDec);
 7453   match(Set dst (AddL dst src));
 7454   effect(KILL cr);
 7455 
 7456   format %{ "incq    $dst\t# long" %}
 7457   ins_encode %{
 7458     __ incrementq($dst$$Register);
 7459   %}
 7460   ins_pipe(ialu_reg);
 7461 %}
 7462 
 7463 instruct incL_rReg_ndd(rRegI dst, rRegI src, immL1 val, rFlagsReg cr)
 7464 %{
 7465   predicate(UseAPX && UseIncDec);
 7466   match(Set dst (AddL src val));
 7467   effect(KILL cr);
 7468 
 7469   format %{ "eincq    $dst, $src\t# long ndd" %}
 7470   ins_encode %{
 7471     __ eincq($dst$$Register, $src$$Register, false);
 7472   %}
 7473   ins_pipe(ialu_reg);
 7474 %}
 7475 
 7476 instruct incL_rReg_mem_ndd(rRegI dst, memory src, immL1 val, rFlagsReg cr)
 7477 %{
 7478   predicate(UseAPX && UseIncDec);
 7479   match(Set dst (AddL (LoadL src) val));
 7480   effect(KILL cr);
 7481 
 7482   format %{ "eincq    $dst, $src\t# long ndd" %}
 7483   ins_encode %{
 7484     __ eincq($dst$$Register, $src$$Address, false);
 7485   %}
 7486   ins_pipe(ialu_reg);
 7487 %}
 7488 
 7489 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7490 %{
 7491   predicate(UseIncDec);
 7492   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7493   effect(KILL cr);
 7494 
 7495   ins_cost(125); // XXX
 7496   format %{ "incq    $dst\t# long" %}
 7497   ins_encode %{
 7498     __ incrementq($dst$$Address);
 7499   %}
 7500   ins_pipe(ialu_mem_imm);
 7501 %}
 7502 
 7503 // XXX why does that use AddL
 7504 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7505 %{
 7506   predicate(!UseAPX && UseIncDec);
 7507   match(Set dst (AddL dst src));
 7508   effect(KILL cr);
 7509 
 7510   format %{ "decq    $dst\t# long" %}
 7511   ins_encode %{
 7512     __ decrementq($dst$$Register);
 7513   %}
 7514   ins_pipe(ialu_reg);
 7515 %}
 7516 
 7517 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7518 %{
 7519   predicate(UseAPX && UseIncDec);
 7520   match(Set dst (AddL src val));
 7521   effect(KILL cr);
 7522 
 7523   format %{ "edecq    $dst, $src\t# long ndd" %}
 7524   ins_encode %{
 7525     __ edecq($dst$$Register, $src$$Register, false);
 7526   %}
 7527   ins_pipe(ialu_reg);
 7528 %}
 7529 
 7530 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7531 %{
 7532   predicate(UseAPX && UseIncDec);
 7533   match(Set dst (AddL (LoadL src) val));
 7534   effect(KILL cr);
 7535 
 7536   format %{ "edecq    $dst, $src\t# long ndd" %}
 7537   ins_encode %{
 7538     __ edecq($dst$$Register, $src$$Address, false);
 7539   %}
 7540   ins_pipe(ialu_reg);
 7541 %}
 7542 
 7543 // XXX why does that use AddL
 7544 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7545 %{
 7546   predicate(UseIncDec);
 7547   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7548   effect(KILL cr);
 7549 
 7550   ins_cost(125); // XXX
 7551   format %{ "decq    $dst\t# long" %}
 7552   ins_encode %{
 7553     __ decrementq($dst$$Address);
 7554   %}
 7555   ins_pipe(ialu_mem_imm);
 7556 %}
 7557 
 7558 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7559 %{
 7560   predicate(VM_Version::supports_fast_2op_lea());
 7561   match(Set dst (AddL (LShiftL index scale) disp));
 7562 
 7563   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7564   ins_encode %{
 7565     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7566     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7567   %}
 7568   ins_pipe(ialu_reg_reg);
 7569 %}
 7570 
 7571 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7572 %{
 7573   predicate(VM_Version::supports_fast_3op_lea());
 7574   match(Set dst (AddL (AddL base index) disp));
 7575 
 7576   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7577   ins_encode %{
 7578     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7579   %}
 7580   ins_pipe(ialu_reg_reg);
 7581 %}
 7582 
 7583 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7584 %{
 7585   predicate(VM_Version::supports_fast_2op_lea());
 7586   match(Set dst (AddL base (LShiftL index scale)));
 7587 
 7588   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7589   ins_encode %{
 7590     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7591     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7592   %}
 7593   ins_pipe(ialu_reg_reg);
 7594 %}
 7595 
 7596 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7597 %{
 7598   predicate(VM_Version::supports_fast_3op_lea());
 7599   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7600 
 7601   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7602   ins_encode %{
 7603     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7604     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7605   %}
 7606   ins_pipe(ialu_reg_reg);
 7607 %}
 7608 
 7609 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7610 %{
 7611   match(Set dst (AddP dst src));
 7612   effect(KILL cr);
 7613   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7614 
 7615   format %{ "addq    $dst, $src\t# ptr" %}
 7616   ins_encode %{
 7617     __ addq($dst$$Register, $src$$Register);
 7618   %}
 7619   ins_pipe(ialu_reg_reg);
 7620 %}
 7621 
 7622 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7623 %{
 7624   match(Set dst (AddP dst src));
 7625   effect(KILL cr);
 7626   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7627 
 7628   format %{ "addq    $dst, $src\t# ptr" %}
 7629   ins_encode %{
 7630     __ addq($dst$$Register, $src$$constant);
 7631   %}
 7632   ins_pipe( ialu_reg );
 7633 %}
 7634 
 7635 // XXX addP mem ops ????
 7636 
 7637 instruct checkCastPP(rRegP dst)
 7638 %{
 7639   match(Set dst (CheckCastPP dst));
 7640 
 7641   size(0);
 7642   format %{ "# checkcastPP of $dst" %}
 7643   ins_encode(/* empty encoding */);
 7644   ins_pipe(empty);
 7645 %}
 7646 
 7647 instruct castPP(rRegP dst)
 7648 %{
 7649   match(Set dst (CastPP dst));
 7650 
 7651   size(0);
 7652   format %{ "# castPP of $dst" %}
 7653   ins_encode(/* empty encoding */);
 7654   ins_pipe(empty);
 7655 %}
 7656 
 7657 instruct castII(rRegI dst)
 7658 %{
 7659   match(Set dst (CastII dst));
 7660 
 7661   size(0);
 7662   format %{ "# castII of $dst" %}
 7663   ins_encode(/* empty encoding */);
 7664   ins_cost(0);
 7665   ins_pipe(empty);
 7666 %}
 7667 
 7668 instruct castLL(rRegL dst)
 7669 %{
 7670   match(Set dst (CastLL dst));
 7671 
 7672   size(0);
 7673   format %{ "# castLL of $dst" %}
 7674   ins_encode(/* empty encoding */);
 7675   ins_cost(0);
 7676   ins_pipe(empty);
 7677 %}
 7678 
 7679 instruct castFF(regF dst)
 7680 %{
 7681   match(Set dst (CastFF dst));
 7682 
 7683   size(0);
 7684   format %{ "# castFF of $dst" %}
 7685   ins_encode(/* empty encoding */);
 7686   ins_cost(0);
 7687   ins_pipe(empty);
 7688 %}
 7689 
 7690 instruct castHH(regF dst)
 7691 %{
 7692   match(Set dst (CastHH dst));
 7693 
 7694   size(0);
 7695   format %{ "# castHH of $dst" %}
 7696   ins_encode(/* empty encoding */);
 7697   ins_cost(0);
 7698   ins_pipe(empty);
 7699 %}
 7700 
 7701 instruct castDD(regD dst)
 7702 %{
 7703   match(Set dst (CastDD dst));
 7704 
 7705   size(0);
 7706   format %{ "# castDD of $dst" %}
 7707   ins_encode(/* empty encoding */);
 7708   ins_cost(0);
 7709   ins_pipe(empty);
 7710 %}
 7711 
 7712 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7713 instruct compareAndSwapP(rRegI res,
 7714                          memory mem_ptr,
 7715                          rax_RegP oldval, rRegP newval,
 7716                          rFlagsReg cr)
 7717 %{
 7718   predicate(n->as_LoadStore()->barrier_data() == 0);
 7719   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7720   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7721   effect(KILL cr, KILL oldval);
 7722 
 7723   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7724             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7725             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7726   ins_encode %{
 7727     __ lock();
 7728     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7729     __ setcc(Assembler::equal, $res$$Register);
 7730   %}
 7731   ins_pipe( pipe_cmpxchg );
 7732 %}
 7733 
 7734 instruct compareAndSwapL(rRegI res,
 7735                          memory mem_ptr,
 7736                          rax_RegL oldval, rRegL newval,
 7737                          rFlagsReg cr)
 7738 %{
 7739   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7740   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7741   effect(KILL cr, KILL oldval);
 7742 
 7743   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7744             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7745             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7746   ins_encode %{
 7747     __ lock();
 7748     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7749     __ setcc(Assembler::equal, $res$$Register);
 7750   %}
 7751   ins_pipe( pipe_cmpxchg );
 7752 %}
 7753 
 7754 instruct compareAndSwapI(rRegI res,
 7755                          memory mem_ptr,
 7756                          rax_RegI oldval, rRegI newval,
 7757                          rFlagsReg cr)
 7758 %{
 7759   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7760   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7761   effect(KILL cr, KILL oldval);
 7762 
 7763   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7764             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7765             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7766   ins_encode %{
 7767     __ lock();
 7768     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7769     __ setcc(Assembler::equal, $res$$Register);
 7770   %}
 7771   ins_pipe( pipe_cmpxchg );
 7772 %}
 7773 
 7774 instruct compareAndSwapB(rRegI res,
 7775                          memory mem_ptr,
 7776                          rax_RegI oldval, rRegI newval,
 7777                          rFlagsReg cr)
 7778 %{
 7779   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7780   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7781   effect(KILL cr, KILL oldval);
 7782 
 7783   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7784             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7785             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7786   ins_encode %{
 7787     __ lock();
 7788     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7789     __ setcc(Assembler::equal, $res$$Register);
 7790   %}
 7791   ins_pipe( pipe_cmpxchg );
 7792 %}
 7793 
 7794 instruct compareAndSwapS(rRegI res,
 7795                          memory mem_ptr,
 7796                          rax_RegI oldval, rRegI newval,
 7797                          rFlagsReg cr)
 7798 %{
 7799   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7800   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7801   effect(KILL cr, KILL oldval);
 7802 
 7803   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7804             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7805             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7806   ins_encode %{
 7807     __ lock();
 7808     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7809     __ setcc(Assembler::equal, $res$$Register);
 7810   %}
 7811   ins_pipe( pipe_cmpxchg );
 7812 %}
 7813 
 7814 instruct compareAndSwapN(rRegI res,
 7815                           memory mem_ptr,
 7816                           rax_RegN oldval, rRegN newval,
 7817                           rFlagsReg cr) %{
 7818   predicate(n->as_LoadStore()->barrier_data() == 0);
 7819   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7820   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7821   effect(KILL cr, KILL oldval);
 7822 
 7823   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7824             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7825             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7826   ins_encode %{
 7827     __ lock();
 7828     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7829     __ setcc(Assembler::equal, $res$$Register);
 7830   %}
 7831   ins_pipe( pipe_cmpxchg );
 7832 %}
 7833 
 7834 instruct compareAndExchangeB(
 7835                          memory mem_ptr,
 7836                          rax_RegI oldval, rRegI newval,
 7837                          rFlagsReg cr)
 7838 %{
 7839   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7840   effect(KILL cr);
 7841 
 7842   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7843             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7844   ins_encode %{
 7845     __ lock();
 7846     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7847   %}
 7848   ins_pipe( pipe_cmpxchg );
 7849 %}
 7850 
 7851 instruct compareAndExchangeS(
 7852                          memory mem_ptr,
 7853                          rax_RegI oldval, rRegI newval,
 7854                          rFlagsReg cr)
 7855 %{
 7856   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7857   effect(KILL cr);
 7858 
 7859   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7860             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7861   ins_encode %{
 7862     __ lock();
 7863     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7864   %}
 7865   ins_pipe( pipe_cmpxchg );
 7866 %}
 7867 
 7868 instruct compareAndExchangeI(
 7869                          memory mem_ptr,
 7870                          rax_RegI oldval, rRegI newval,
 7871                          rFlagsReg cr)
 7872 %{
 7873   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7874   effect(KILL cr);
 7875 
 7876   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7877             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7878   ins_encode %{
 7879     __ lock();
 7880     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7881   %}
 7882   ins_pipe( pipe_cmpxchg );
 7883 %}
 7884 
 7885 instruct compareAndExchangeL(
 7886                          memory mem_ptr,
 7887                          rax_RegL oldval, rRegL newval,
 7888                          rFlagsReg cr)
 7889 %{
 7890   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7891   effect(KILL cr);
 7892 
 7893   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7894             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7895   ins_encode %{
 7896     __ lock();
 7897     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7898   %}
 7899   ins_pipe( pipe_cmpxchg );
 7900 %}
 7901 
 7902 instruct compareAndExchangeN(
 7903                           memory mem_ptr,
 7904                           rax_RegN oldval, rRegN newval,
 7905                           rFlagsReg cr) %{
 7906   predicate(n->as_LoadStore()->barrier_data() == 0);
 7907   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7908   effect(KILL cr);
 7909 
 7910   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7911             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7912   ins_encode %{
 7913     __ lock();
 7914     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7915   %}
 7916   ins_pipe( pipe_cmpxchg );
 7917 %}
 7918 
 7919 instruct compareAndExchangeP(
 7920                          memory mem_ptr,
 7921                          rax_RegP oldval, rRegP newval,
 7922                          rFlagsReg cr)
 7923 %{
 7924   predicate(n->as_LoadStore()->barrier_data() == 0);
 7925   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7926   effect(KILL cr);
 7927 
 7928   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7929             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7930   ins_encode %{
 7931     __ lock();
 7932     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7933   %}
 7934   ins_pipe( pipe_cmpxchg );
 7935 %}
 7936 
 7937 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7938   predicate(n->as_LoadStore()->result_not_used());
 7939   match(Set dummy (GetAndAddB mem add));
 7940   effect(KILL cr);
 7941   format %{ "addb_lock   $mem, $add" %}
 7942   ins_encode %{
 7943     __ lock();
 7944     __ addb($mem$$Address, $add$$Register);
 7945   %}
 7946   ins_pipe(pipe_cmpxchg);
 7947 %}
 7948 
 7949 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7950   predicate(n->as_LoadStore()->result_not_used());
 7951   match(Set dummy (GetAndAddB mem add));
 7952   effect(KILL cr);
 7953   format %{ "addb_lock   $mem, $add" %}
 7954   ins_encode %{
 7955     __ lock();
 7956     __ addb($mem$$Address, $add$$constant);
 7957   %}
 7958   ins_pipe(pipe_cmpxchg);
 7959 %}
 7960 
 7961 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7962   predicate(!n->as_LoadStore()->result_not_used());
 7963   match(Set newval (GetAndAddB mem newval));
 7964   effect(KILL cr);
 7965   format %{ "xaddb_lock  $mem, $newval" %}
 7966   ins_encode %{
 7967     __ lock();
 7968     __ xaddb($mem$$Address, $newval$$Register);
 7969   %}
 7970   ins_pipe(pipe_cmpxchg);
 7971 %}
 7972 
 7973 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7974   predicate(n->as_LoadStore()->result_not_used());
 7975   match(Set dummy (GetAndAddS mem add));
 7976   effect(KILL cr);
 7977   format %{ "addw_lock   $mem, $add" %}
 7978   ins_encode %{
 7979     __ lock();
 7980     __ addw($mem$$Address, $add$$Register);
 7981   %}
 7982   ins_pipe(pipe_cmpxchg);
 7983 %}
 7984 
 7985 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7986   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7987   match(Set dummy (GetAndAddS mem add));
 7988   effect(KILL cr);
 7989   format %{ "addw_lock   $mem, $add" %}
 7990   ins_encode %{
 7991     __ lock();
 7992     __ addw($mem$$Address, $add$$constant);
 7993   %}
 7994   ins_pipe(pipe_cmpxchg);
 7995 %}
 7996 
 7997 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 7998   predicate(!n->as_LoadStore()->result_not_used());
 7999   match(Set newval (GetAndAddS mem newval));
 8000   effect(KILL cr);
 8001   format %{ "xaddw_lock  $mem, $newval" %}
 8002   ins_encode %{
 8003     __ lock();
 8004     __ xaddw($mem$$Address, $newval$$Register);
 8005   %}
 8006   ins_pipe(pipe_cmpxchg);
 8007 %}
 8008 
 8009 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8010   predicate(n->as_LoadStore()->result_not_used());
 8011   match(Set dummy (GetAndAddI mem add));
 8012   effect(KILL cr);
 8013   format %{ "addl_lock   $mem, $add" %}
 8014   ins_encode %{
 8015     __ lock();
 8016     __ addl($mem$$Address, $add$$Register);
 8017   %}
 8018   ins_pipe(pipe_cmpxchg);
 8019 %}
 8020 
 8021 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8022   predicate(n->as_LoadStore()->result_not_used());
 8023   match(Set dummy (GetAndAddI mem add));
 8024   effect(KILL cr);
 8025   format %{ "addl_lock   $mem, $add" %}
 8026   ins_encode %{
 8027     __ lock();
 8028     __ addl($mem$$Address, $add$$constant);
 8029   %}
 8030   ins_pipe(pipe_cmpxchg);
 8031 %}
 8032 
 8033 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8034   predicate(!n->as_LoadStore()->result_not_used());
 8035   match(Set newval (GetAndAddI mem newval));
 8036   effect(KILL cr);
 8037   format %{ "xaddl_lock  $mem, $newval" %}
 8038   ins_encode %{
 8039     __ lock();
 8040     __ xaddl($mem$$Address, $newval$$Register);
 8041   %}
 8042   ins_pipe(pipe_cmpxchg);
 8043 %}
 8044 
 8045 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8046   predicate(n->as_LoadStore()->result_not_used());
 8047   match(Set dummy (GetAndAddL mem add));
 8048   effect(KILL cr);
 8049   format %{ "addq_lock   $mem, $add" %}
 8050   ins_encode %{
 8051     __ lock();
 8052     __ addq($mem$$Address, $add$$Register);
 8053   %}
 8054   ins_pipe(pipe_cmpxchg);
 8055 %}
 8056 
 8057 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8058   predicate(n->as_LoadStore()->result_not_used());
 8059   match(Set dummy (GetAndAddL mem add));
 8060   effect(KILL cr);
 8061   format %{ "addq_lock   $mem, $add" %}
 8062   ins_encode %{
 8063     __ lock();
 8064     __ addq($mem$$Address, $add$$constant);
 8065   %}
 8066   ins_pipe(pipe_cmpxchg);
 8067 %}
 8068 
 8069 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8070   predicate(!n->as_LoadStore()->result_not_used());
 8071   match(Set newval (GetAndAddL mem newval));
 8072   effect(KILL cr);
 8073   format %{ "xaddq_lock  $mem, $newval" %}
 8074   ins_encode %{
 8075     __ lock();
 8076     __ xaddq($mem$$Address, $newval$$Register);
 8077   %}
 8078   ins_pipe(pipe_cmpxchg);
 8079 %}
 8080 
 8081 instruct xchgB( memory mem, rRegI newval) %{
 8082   match(Set newval (GetAndSetB mem newval));
 8083   format %{ "XCHGB  $newval,[$mem]" %}
 8084   ins_encode %{
 8085     __ xchgb($newval$$Register, $mem$$Address);
 8086   %}
 8087   ins_pipe( pipe_cmpxchg );
 8088 %}
 8089 
 8090 instruct xchgS( memory mem, rRegI newval) %{
 8091   match(Set newval (GetAndSetS mem newval));
 8092   format %{ "XCHGW  $newval,[$mem]" %}
 8093   ins_encode %{
 8094     __ xchgw($newval$$Register, $mem$$Address);
 8095   %}
 8096   ins_pipe( pipe_cmpxchg );
 8097 %}
 8098 
 8099 instruct xchgI( memory mem, rRegI newval) %{
 8100   match(Set newval (GetAndSetI mem newval));
 8101   format %{ "XCHGL  $newval,[$mem]" %}
 8102   ins_encode %{
 8103     __ xchgl($newval$$Register, $mem$$Address);
 8104   %}
 8105   ins_pipe( pipe_cmpxchg );
 8106 %}
 8107 
 8108 instruct xchgL( memory mem, rRegL newval) %{
 8109   match(Set newval (GetAndSetL mem newval));
 8110   format %{ "XCHGL  $newval,[$mem]" %}
 8111   ins_encode %{
 8112     __ xchgq($newval$$Register, $mem$$Address);
 8113   %}
 8114   ins_pipe( pipe_cmpxchg );
 8115 %}
 8116 
 8117 instruct xchgP( memory mem, rRegP newval) %{
 8118   match(Set newval (GetAndSetP mem newval));
 8119   predicate(n->as_LoadStore()->barrier_data() == 0);
 8120   format %{ "XCHGQ  $newval,[$mem]" %}
 8121   ins_encode %{
 8122     __ xchgq($newval$$Register, $mem$$Address);
 8123   %}
 8124   ins_pipe( pipe_cmpxchg );
 8125 %}
 8126 
 8127 instruct xchgN( memory mem, rRegN newval) %{
 8128   predicate(n->as_LoadStore()->barrier_data() == 0);
 8129   match(Set newval (GetAndSetN mem newval));
 8130   format %{ "XCHGL  $newval,$mem]" %}
 8131   ins_encode %{
 8132     __ xchgl($newval$$Register, $mem$$Address);
 8133   %}
 8134   ins_pipe( pipe_cmpxchg );
 8135 %}
 8136 
 8137 //----------Abs Instructions-------------------------------------------
 8138 
 8139 // Integer Absolute Instructions
 8140 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8141 %{
 8142   match(Set dst (AbsI src));
 8143   effect(TEMP dst, KILL cr);
 8144   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8145             "subl    $dst, $src\n\t"
 8146             "cmovll  $dst, $src" %}
 8147   ins_encode %{
 8148     __ xorl($dst$$Register, $dst$$Register);
 8149     __ subl($dst$$Register, $src$$Register);
 8150     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8151   %}
 8152 
 8153   ins_pipe(ialu_reg_reg);
 8154 %}
 8155 
 8156 // Long Absolute Instructions
 8157 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8158 %{
 8159   match(Set dst (AbsL src));
 8160   effect(TEMP dst, KILL cr);
 8161   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8162             "subq    $dst, $src\n\t"
 8163             "cmovlq  $dst, $src" %}
 8164   ins_encode %{
 8165     __ xorl($dst$$Register, $dst$$Register);
 8166     __ subq($dst$$Register, $src$$Register);
 8167     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8168   %}
 8169 
 8170   ins_pipe(ialu_reg_reg);
 8171 %}
 8172 
 8173 //----------Subtraction Instructions-------------------------------------------
 8174 
 8175 // Integer Subtraction Instructions
 8176 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8177 %{
 8178   predicate(!UseAPX);
 8179   match(Set dst (SubI dst src));
 8180   effect(KILL cr);
 8181   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8182 
 8183   format %{ "subl    $dst, $src\t# int" %}
 8184   ins_encode %{
 8185     __ subl($dst$$Register, $src$$Register);
 8186   %}
 8187   ins_pipe(ialu_reg_reg);
 8188 %}
 8189 
 8190 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8191 %{
 8192   predicate(UseAPX);
 8193   match(Set dst (SubI src1 src2));
 8194   effect(KILL cr);
 8195   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8196 
 8197   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8198   ins_encode %{
 8199     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8200   %}
 8201   ins_pipe(ialu_reg_reg);
 8202 %}
 8203 
 8204 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8205 %{
 8206   predicate(UseAPX);
 8207   match(Set dst (SubI src1 src2));
 8208   effect(KILL cr);
 8209   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8210 
 8211   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8212   ins_encode %{
 8213     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8214   %}
 8215   ins_pipe(ialu_reg_reg);
 8216 %}
 8217 
 8218 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8219 %{
 8220   predicate(UseAPX);
 8221   match(Set dst (SubI (LoadI src1) src2));
 8222   effect(KILL cr);
 8223   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8224 
 8225   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8226   ins_encode %{
 8227     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8228   %}
 8229   ins_pipe(ialu_reg_reg);
 8230 %}
 8231 
 8232 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8233 %{
 8234   predicate(!UseAPX);
 8235   match(Set dst (SubI dst (LoadI src)));
 8236   effect(KILL cr);
 8237   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8238 
 8239   ins_cost(150);
 8240   format %{ "subl    $dst, $src\t# int" %}
 8241   ins_encode %{
 8242     __ subl($dst$$Register, $src$$Address);
 8243   %}
 8244   ins_pipe(ialu_reg_mem);
 8245 %}
 8246 
 8247 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8248 %{
 8249   predicate(UseAPX);
 8250   match(Set dst (SubI src1 (LoadI src2)));
 8251   effect(KILL cr);
 8252   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8253 
 8254   ins_cost(150);
 8255   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8256   ins_encode %{
 8257     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8258   %}
 8259   ins_pipe(ialu_reg_mem);
 8260 %}
 8261 
 8262 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8263 %{
 8264   predicate(UseAPX);
 8265   match(Set dst (SubI (LoadI src1) src2));
 8266   effect(KILL cr);
 8267   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8268 
 8269   ins_cost(150);
 8270   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8271   ins_encode %{
 8272     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8273   %}
 8274   ins_pipe(ialu_reg_mem);
 8275 %}
 8276 
 8277 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8278 %{
 8279   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8280   effect(KILL cr);
 8281   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8282 
 8283   ins_cost(150);
 8284   format %{ "subl    $dst, $src\t# int" %}
 8285   ins_encode %{
 8286     __ subl($dst$$Address, $src$$Register);
 8287   %}
 8288   ins_pipe(ialu_mem_reg);
 8289 %}
 8290 
 8291 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8292 %{
 8293   predicate(!UseAPX);
 8294   match(Set dst (SubL dst src));
 8295   effect(KILL cr);
 8296   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8297 
 8298   format %{ "subq    $dst, $src\t# long" %}
 8299   ins_encode %{
 8300     __ subq($dst$$Register, $src$$Register);
 8301   %}
 8302   ins_pipe(ialu_reg_reg);
 8303 %}
 8304 
 8305 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8306 %{
 8307   predicate(UseAPX);
 8308   match(Set dst (SubL src1 src2));
 8309   effect(KILL cr);
 8310   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8311 
 8312   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8313   ins_encode %{
 8314     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8315   %}
 8316   ins_pipe(ialu_reg_reg);
 8317 %}
 8318 
 8319 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8320 %{
 8321   predicate(UseAPX);
 8322   match(Set dst (SubL src1 src2));
 8323   effect(KILL cr);
 8324   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8325 
 8326   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8327   ins_encode %{
 8328     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8329   %}
 8330   ins_pipe(ialu_reg_reg);
 8331 %}
 8332 
 8333 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8334 %{
 8335   predicate(UseAPX);
 8336   match(Set dst (SubL (LoadL src1) src2));
 8337   effect(KILL cr);
 8338   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8339 
 8340   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8341   ins_encode %{
 8342     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8343   %}
 8344   ins_pipe(ialu_reg_reg);
 8345 %}
 8346 
 8347 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8348 %{
 8349   predicate(!UseAPX);
 8350   match(Set dst (SubL dst (LoadL src)));
 8351   effect(KILL cr);
 8352   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8353 
 8354   ins_cost(150);
 8355   format %{ "subq    $dst, $src\t# long" %}
 8356   ins_encode %{
 8357     __ subq($dst$$Register, $src$$Address);
 8358   %}
 8359   ins_pipe(ialu_reg_mem);
 8360 %}
 8361 
 8362 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8363 %{
 8364   predicate(UseAPX);
 8365   match(Set dst (SubL src1 (LoadL src2)));
 8366   effect(KILL cr);
 8367   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8368 
 8369   ins_cost(150);
 8370   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8371   ins_encode %{
 8372     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8373   %}
 8374   ins_pipe(ialu_reg_mem);
 8375 %}
 8376 
 8377 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8378 %{
 8379   predicate(UseAPX);
 8380   match(Set dst (SubL (LoadL src1) src2));
 8381   effect(KILL cr);
 8382   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8383 
 8384   ins_cost(150);
 8385   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8386   ins_encode %{
 8387     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8388   %}
 8389   ins_pipe(ialu_reg_mem);
 8390 %}
 8391 
 8392 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8393 %{
 8394   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8395   effect(KILL cr);
 8396   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8397 
 8398   ins_cost(150);
 8399   format %{ "subq    $dst, $src\t# long" %}
 8400   ins_encode %{
 8401     __ subq($dst$$Address, $src$$Register);
 8402   %}
 8403   ins_pipe(ialu_mem_reg);
 8404 %}
 8405 
 8406 // Subtract from a pointer
 8407 // XXX hmpf???
 8408 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8409 %{
 8410   match(Set dst (AddP dst (SubI zero src)));
 8411   effect(KILL cr);
 8412 
 8413   format %{ "subq    $dst, $src\t# ptr - int" %}
 8414   ins_encode %{
 8415     __ subq($dst$$Register, $src$$Register);
 8416   %}
 8417   ins_pipe(ialu_reg_reg);
 8418 %}
 8419 
 8420 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8421 %{
 8422   predicate(!UseAPX);
 8423   match(Set dst (SubI zero dst));
 8424   effect(KILL cr);
 8425   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8426 
 8427   format %{ "negl    $dst\t# int" %}
 8428   ins_encode %{
 8429     __ negl($dst$$Register);
 8430   %}
 8431   ins_pipe(ialu_reg);
 8432 %}
 8433 
 8434 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8435 %{
 8436   predicate(UseAPX);
 8437   match(Set dst (SubI zero src));
 8438   effect(KILL cr);
 8439   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8440 
 8441   format %{ "enegl    $dst, $src\t# int ndd" %}
 8442   ins_encode %{
 8443     __ enegl($dst$$Register, $src$$Register, false);
 8444   %}
 8445   ins_pipe(ialu_reg);
 8446 %}
 8447 
 8448 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8449 %{
 8450   predicate(!UseAPX);
 8451   match(Set dst (NegI dst));
 8452   effect(KILL cr);
 8453   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8454 
 8455   format %{ "negl    $dst\t# int" %}
 8456   ins_encode %{
 8457     __ negl($dst$$Register);
 8458   %}
 8459   ins_pipe(ialu_reg);
 8460 %}
 8461 
 8462 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8463 %{
 8464   predicate(UseAPX);
 8465   match(Set dst (NegI src));
 8466   effect(KILL cr);
 8467   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8468 
 8469   format %{ "enegl    $dst, $src\t# int ndd" %}
 8470   ins_encode %{
 8471     __ enegl($dst$$Register, $src$$Register, false);
 8472   %}
 8473   ins_pipe(ialu_reg);
 8474 %}
 8475 
 8476 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8477 %{
 8478   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8479   effect(KILL cr);
 8480   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8481 
 8482   format %{ "negl    $dst\t# int" %}
 8483   ins_encode %{
 8484     __ negl($dst$$Address);
 8485   %}
 8486   ins_pipe(ialu_reg);
 8487 %}
 8488 
 8489 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8490 %{
 8491   predicate(!UseAPX);
 8492   match(Set dst (SubL zero dst));
 8493   effect(KILL cr);
 8494   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8495 
 8496   format %{ "negq    $dst\t# long" %}
 8497   ins_encode %{
 8498     __ negq($dst$$Register);
 8499   %}
 8500   ins_pipe(ialu_reg);
 8501 %}
 8502 
 8503 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8504 %{
 8505   predicate(UseAPX);
 8506   match(Set dst (SubL zero src));
 8507   effect(KILL cr);
 8508   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8509 
 8510   format %{ "enegq    $dst, $src\t# long ndd" %}
 8511   ins_encode %{
 8512     __ enegq($dst$$Register, $src$$Register, false);
 8513   %}
 8514   ins_pipe(ialu_reg);
 8515 %}
 8516 
 8517 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8518 %{
 8519   predicate(!UseAPX);
 8520   match(Set dst (NegL dst));
 8521   effect(KILL cr);
 8522   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8523 
 8524   format %{ "negq    $dst\t# int" %}
 8525   ins_encode %{
 8526     __ negq($dst$$Register);
 8527   %}
 8528   ins_pipe(ialu_reg);
 8529 %}
 8530 
 8531 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8532 %{
 8533   predicate(UseAPX);
 8534   match(Set dst (NegL src));
 8535   effect(KILL cr);
 8536   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8537 
 8538   format %{ "enegq    $dst, $src\t# long ndd" %}
 8539   ins_encode %{
 8540     __ enegq($dst$$Register, $src$$Register, false);
 8541   %}
 8542   ins_pipe(ialu_reg);
 8543 %}
 8544 
 8545 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8546 %{
 8547   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8548   effect(KILL cr);
 8549   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8550 
 8551   format %{ "negq    $dst\t# long" %}
 8552   ins_encode %{
 8553     __ negq($dst$$Address);
 8554   %}
 8555   ins_pipe(ialu_reg);
 8556 %}
 8557 
 8558 //----------Multiplication/Division Instructions-------------------------------
 8559 // Integer Multiplication Instructions
 8560 // Multiply Register
 8561 
 8562 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8563 %{
 8564   predicate(!UseAPX);
 8565   match(Set dst (MulI dst src));
 8566   effect(KILL cr);
 8567 
 8568   ins_cost(300);
 8569   format %{ "imull   $dst, $src\t# int" %}
 8570   ins_encode %{
 8571     __ imull($dst$$Register, $src$$Register);
 8572   %}
 8573   ins_pipe(ialu_reg_reg_alu0);
 8574 %}
 8575 
 8576 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8577 %{
 8578   predicate(UseAPX);
 8579   match(Set dst (MulI src1 src2));
 8580   effect(KILL cr);
 8581 
 8582   ins_cost(300);
 8583   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8584   ins_encode %{
 8585     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8586   %}
 8587   ins_pipe(ialu_reg_reg_alu0);
 8588 %}
 8589 
 8590 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8591 %{
 8592   predicate(!UseAPX);
 8593   match(Set dst (MulI src imm));
 8594   effect(KILL cr);
 8595 
 8596   ins_cost(300);
 8597   format %{ "imull   $dst, $src, $imm\t# int" %}
 8598   ins_encode %{
 8599     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8600   %}
 8601   ins_pipe(ialu_reg_reg_alu0);
 8602 %}
 8603 
 8604 instruct mulI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8605 %{
 8606   predicate(UseAPX);
 8607   match(Set dst (MulI src1 src2));
 8608   effect(KILL cr);
 8609 
 8610   ins_cost(300);
 8611   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8612   ins_encode %{
 8613     __ eimull($dst$$Register, $src1$$Register, $src2$$constant, false);
 8614   %}
 8615   ins_pipe(ialu_reg_reg_alu0);
 8616 %}
 8617 
 8618 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8619 %{
 8620   predicate(!UseAPX);
 8621   match(Set dst (MulI dst (LoadI src)));
 8622   effect(KILL cr);
 8623 
 8624   ins_cost(350);
 8625   format %{ "imull   $dst, $src\t# int" %}
 8626   ins_encode %{
 8627     __ imull($dst$$Register, $src$$Address);
 8628   %}
 8629   ins_pipe(ialu_reg_mem_alu0);
 8630 %}
 8631 
 8632 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8633 %{
 8634   predicate(UseAPX);
 8635   match(Set dst (MulI src1 (LoadI src2)));
 8636   effect(KILL cr);
 8637 
 8638   ins_cost(350);
 8639   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8640   ins_encode %{
 8641     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8642   %}
 8643   ins_pipe(ialu_reg_mem_alu0);
 8644 %}
 8645 
 8646 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8647 %{
 8648   predicate(!UseAPX);
 8649   match(Set dst (MulI (LoadI src) imm));
 8650   effect(KILL cr);
 8651 
 8652   ins_cost(300);
 8653   format %{ "imull   $dst, $src, $imm\t# int" %}
 8654   ins_encode %{
 8655     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8656   %}
 8657   ins_pipe(ialu_reg_mem_alu0);
 8658 %}
 8659 
 8660 instruct mulI_rReg_mem_imm(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8661 %{
 8662   predicate(UseAPX);
 8663   match(Set dst (MulI (LoadI src1) src2));
 8664   effect(KILL cr);
 8665 
 8666   ins_cost(300);
 8667   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8668   ins_encode %{
 8669     __ eimull($dst$$Register, $src1$$Address, $src2$$constant, false);
 8670   %}
 8671   ins_pipe(ialu_reg_mem_alu0);
 8672 %}
 8673 
 8674 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8675 %{
 8676   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8677   effect(KILL cr, KILL src2);
 8678 
 8679   expand %{ mulI_rReg(dst, src1, cr);
 8680            mulI_rReg(src2, src3, cr);
 8681            addI_rReg(dst, src2, cr); %}
 8682 %}
 8683 
 8684 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8685 %{
 8686   predicate(!UseAPX);
 8687   match(Set dst (MulL dst src));
 8688   effect(KILL cr);
 8689 
 8690   ins_cost(300);
 8691   format %{ "imulq   $dst, $src\t# long" %}
 8692   ins_encode %{
 8693     __ imulq($dst$$Register, $src$$Register);
 8694   %}
 8695   ins_pipe(ialu_reg_reg_alu0);
 8696 %}
 8697 
 8698 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8699 %{
 8700   predicate(UseAPX);
 8701   match(Set dst (MulL src1 src2));
 8702   effect(KILL cr);
 8703 
 8704   ins_cost(300);
 8705   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8706   ins_encode %{
 8707     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8708   %}
 8709   ins_pipe(ialu_reg_reg_alu0);
 8710 %}
 8711 
 8712 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8713 %{
 8714   predicate(!UseAPX);
 8715   match(Set dst (MulL src imm));
 8716   effect(KILL cr);
 8717 
 8718   ins_cost(300);
 8719   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8720   ins_encode %{
 8721     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8722   %}
 8723   ins_pipe(ialu_reg_reg_alu0);
 8724 %}
 8725 
 8726 instruct mulL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8727 %{
 8728   predicate(UseAPX);
 8729   match(Set dst (MulL src1 src2));
 8730   effect(KILL cr);
 8731 
 8732   ins_cost(300);
 8733   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8734   ins_encode %{
 8735     __ eimulq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8736   %}
 8737   ins_pipe(ialu_reg_reg_alu0);
 8738 %}
 8739 
 8740 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8741 %{
 8742   predicate(!UseAPX);
 8743   match(Set dst (MulL dst (LoadL src)));
 8744   effect(KILL cr);
 8745 
 8746   ins_cost(350);
 8747   format %{ "imulq   $dst, $src\t# long" %}
 8748   ins_encode %{
 8749     __ imulq($dst$$Register, $src$$Address);
 8750   %}
 8751   ins_pipe(ialu_reg_mem_alu0);
 8752 %}
 8753 
 8754 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8755 %{
 8756   predicate(UseAPX);
 8757   match(Set dst (MulL src1 (LoadL src2)));
 8758   effect(KILL cr);
 8759 
 8760   ins_cost(350);
 8761   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8762   ins_encode %{
 8763     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8764   %}
 8765   ins_pipe(ialu_reg_mem_alu0);
 8766 %}
 8767 
 8768 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8769 %{
 8770   predicate(!UseAPX);
 8771   match(Set dst (MulL (LoadL src) imm));
 8772   effect(KILL cr);
 8773 
 8774   ins_cost(300);
 8775   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8776   ins_encode %{
 8777     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8778   %}
 8779   ins_pipe(ialu_reg_mem_alu0);
 8780 %}
 8781 
 8782 instruct mulL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8783 %{
 8784   predicate(UseAPX);
 8785   match(Set dst (MulL (LoadL src1) src2));
 8786   effect(KILL cr);
 8787 
 8788   ins_cost(300);
 8789   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8790   ins_encode %{
 8791     __ eimulq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8792   %}
 8793   ins_pipe(ialu_reg_mem_alu0);
 8794 %}
 8795 
 8796 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8797 %{
 8798   match(Set dst (MulHiL src rax));
 8799   effect(USE_KILL rax, KILL cr);
 8800 
 8801   ins_cost(300);
 8802   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8803   ins_encode %{
 8804     __ imulq($src$$Register);
 8805   %}
 8806   ins_pipe(ialu_reg_reg_alu0);
 8807 %}
 8808 
 8809 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8810 %{
 8811   match(Set dst (UMulHiL src rax));
 8812   effect(USE_KILL rax, KILL cr);
 8813 
 8814   ins_cost(300);
 8815   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8816   ins_encode %{
 8817     __ mulq($src$$Register);
 8818   %}
 8819   ins_pipe(ialu_reg_reg_alu0);
 8820 %}
 8821 
 8822 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8823                    rFlagsReg cr)
 8824 %{
 8825   match(Set rax (DivI rax div));
 8826   effect(KILL rdx, KILL cr);
 8827 
 8828   ins_cost(30*100+10*100); // XXX
 8829   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8830             "jne,s   normal\n\t"
 8831             "xorl    rdx, rdx\n\t"
 8832             "cmpl    $div, -1\n\t"
 8833             "je,s    done\n"
 8834     "normal: cdql\n\t"
 8835             "idivl   $div\n"
 8836     "done:"        %}
 8837   ins_encode(cdql_enc(div));
 8838   ins_pipe(ialu_reg_reg_alu0);
 8839 %}
 8840 
 8841 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8842                    rFlagsReg cr)
 8843 %{
 8844   match(Set rax (DivL rax div));
 8845   effect(KILL rdx, KILL cr);
 8846 
 8847   ins_cost(30*100+10*100); // XXX
 8848   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8849             "cmpq    rax, rdx\n\t"
 8850             "jne,s   normal\n\t"
 8851             "xorl    rdx, rdx\n\t"
 8852             "cmpq    $div, -1\n\t"
 8853             "je,s    done\n"
 8854     "normal: cdqq\n\t"
 8855             "idivq   $div\n"
 8856     "done:"        %}
 8857   ins_encode(cdqq_enc(div));
 8858   ins_pipe(ialu_reg_reg_alu0);
 8859 %}
 8860 
 8861 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8862 %{
 8863   match(Set rax (UDivI rax div));
 8864   effect(KILL rdx, KILL cr);
 8865 
 8866   ins_cost(300);
 8867   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8868   ins_encode %{
 8869     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8870   %}
 8871   ins_pipe(ialu_reg_reg_alu0);
 8872 %}
 8873 
 8874 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8875 %{
 8876   match(Set rax (UDivL rax div));
 8877   effect(KILL rdx, KILL cr);
 8878 
 8879   ins_cost(300);
 8880   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8881   ins_encode %{
 8882      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8883   %}
 8884   ins_pipe(ialu_reg_reg_alu0);
 8885 %}
 8886 
 8887 // Integer DIVMOD with Register, both quotient and mod results
 8888 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8889                              rFlagsReg cr)
 8890 %{
 8891   match(DivModI rax div);
 8892   effect(KILL cr);
 8893 
 8894   ins_cost(30*100+10*100); // XXX
 8895   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8896             "jne,s   normal\n\t"
 8897             "xorl    rdx, rdx\n\t"
 8898             "cmpl    $div, -1\n\t"
 8899             "je,s    done\n"
 8900     "normal: cdql\n\t"
 8901             "idivl   $div\n"
 8902     "done:"        %}
 8903   ins_encode(cdql_enc(div));
 8904   ins_pipe(pipe_slow);
 8905 %}
 8906 
 8907 // Long DIVMOD with Register, both quotient and mod results
 8908 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8909                              rFlagsReg cr)
 8910 %{
 8911   match(DivModL rax div);
 8912   effect(KILL cr);
 8913 
 8914   ins_cost(30*100+10*100); // XXX
 8915   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8916             "cmpq    rax, rdx\n\t"
 8917             "jne,s   normal\n\t"
 8918             "xorl    rdx, rdx\n\t"
 8919             "cmpq    $div, -1\n\t"
 8920             "je,s    done\n"
 8921     "normal: cdqq\n\t"
 8922             "idivq   $div\n"
 8923     "done:"        %}
 8924   ins_encode(cdqq_enc(div));
 8925   ins_pipe(pipe_slow);
 8926 %}
 8927 
 8928 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8929 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8930                               no_rax_rdx_RegI div, rFlagsReg cr)
 8931 %{
 8932   match(UDivModI rax div);
 8933   effect(TEMP tmp, KILL cr);
 8934 
 8935   ins_cost(300);
 8936   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8937             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8938           %}
 8939   ins_encode %{
 8940     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8941   %}
 8942   ins_pipe(pipe_slow);
 8943 %}
 8944 
 8945 // Unsigned long DIVMOD with Register, both quotient and mod results
 8946 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8947                               no_rax_rdx_RegL div, rFlagsReg cr)
 8948 %{
 8949   match(UDivModL rax div);
 8950   effect(TEMP tmp, KILL cr);
 8951 
 8952   ins_cost(300);
 8953   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8954             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8955           %}
 8956   ins_encode %{
 8957     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8958   %}
 8959   ins_pipe(pipe_slow);
 8960 %}
 8961 
 8962 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8963                    rFlagsReg cr)
 8964 %{
 8965   match(Set rdx (ModI rax div));
 8966   effect(KILL rax, KILL cr);
 8967 
 8968   ins_cost(300); // XXX
 8969   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8970             "jne,s   normal\n\t"
 8971             "xorl    rdx, rdx\n\t"
 8972             "cmpl    $div, -1\n\t"
 8973             "je,s    done\n"
 8974     "normal: cdql\n\t"
 8975             "idivl   $div\n"
 8976     "done:"        %}
 8977   ins_encode(cdql_enc(div));
 8978   ins_pipe(ialu_reg_reg_alu0);
 8979 %}
 8980 
 8981 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8982                    rFlagsReg cr)
 8983 %{
 8984   match(Set rdx (ModL rax div));
 8985   effect(KILL rax, KILL cr);
 8986 
 8987   ins_cost(300); // XXX
 8988   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8989             "cmpq    rax, rdx\n\t"
 8990             "jne,s   normal\n\t"
 8991             "xorl    rdx, rdx\n\t"
 8992             "cmpq    $div, -1\n\t"
 8993             "je,s    done\n"
 8994     "normal: cdqq\n\t"
 8995             "idivq   $div\n"
 8996     "done:"        %}
 8997   ins_encode(cdqq_enc(div));
 8998   ins_pipe(ialu_reg_reg_alu0);
 8999 %}
 9000 
 9001 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9002 %{
 9003   match(Set rdx (UModI rax div));
 9004   effect(KILL rax, KILL cr);
 9005 
 9006   ins_cost(300);
 9007   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9008   ins_encode %{
 9009     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9010   %}
 9011   ins_pipe(ialu_reg_reg_alu0);
 9012 %}
 9013 
 9014 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9015 %{
 9016   match(Set rdx (UModL rax div));
 9017   effect(KILL rax, KILL cr);
 9018 
 9019   ins_cost(300);
 9020   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9021   ins_encode %{
 9022     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9023   %}
 9024   ins_pipe(ialu_reg_reg_alu0);
 9025 %}
 9026 
 9027 // Integer Shift Instructions
 9028 // Shift Left by one, two, three
 9029 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9030 %{
 9031   predicate(!UseAPX);
 9032   match(Set dst (LShiftI dst shift));
 9033   effect(KILL cr);
 9034 
 9035   format %{ "sall    $dst, $shift" %}
 9036   ins_encode %{
 9037     __ sall($dst$$Register, $shift$$constant);
 9038   %}
 9039   ins_pipe(ialu_reg);
 9040 %}
 9041 
 9042 // Shift Left by one, two, three
 9043 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9044 %{
 9045   predicate(UseAPX);
 9046   match(Set dst (LShiftI src shift));
 9047   effect(KILL cr);
 9048 
 9049   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9050   ins_encode %{
 9051     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9052   %}
 9053   ins_pipe(ialu_reg);
 9054 %}
 9055 
 9056 // Shift Left by 8-bit immediate
 9057 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9058 %{
 9059   predicate(!UseAPX);
 9060   match(Set dst (LShiftI dst shift));
 9061   effect(KILL cr);
 9062 
 9063   format %{ "sall    $dst, $shift" %}
 9064   ins_encode %{
 9065     __ sall($dst$$Register, $shift$$constant);
 9066   %}
 9067   ins_pipe(ialu_reg);
 9068 %}
 9069 
 9070 // Shift Left by 8-bit immediate
 9071 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9072 %{
 9073   predicate(UseAPX);
 9074   match(Set dst (LShiftI src shift));
 9075   effect(KILL cr);
 9076 
 9077   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9078   ins_encode %{
 9079     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9080   %}
 9081   ins_pipe(ialu_reg);
 9082 %}
 9083 
 9084 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9085 %{
 9086   predicate(UseAPX);
 9087   match(Set dst (LShiftI (LoadI src) shift));
 9088   effect(KILL cr);
 9089 
 9090   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9091   ins_encode %{
 9092     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9093   %}
 9094   ins_pipe(ialu_reg);
 9095 %}
 9096 
 9097 // Shift Left by 8-bit immediate
 9098 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9099 %{
 9100   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9101   effect(KILL cr);
 9102 
 9103   format %{ "sall    $dst, $shift" %}
 9104   ins_encode %{
 9105     __ sall($dst$$Address, $shift$$constant);
 9106   %}
 9107   ins_pipe(ialu_mem_imm);
 9108 %}
 9109 
 9110 // Shift Left by variable
 9111 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9112 %{
 9113   predicate(!VM_Version::supports_bmi2());
 9114   match(Set dst (LShiftI dst shift));
 9115   effect(KILL cr);
 9116 
 9117   format %{ "sall    $dst, $shift" %}
 9118   ins_encode %{
 9119     __ sall($dst$$Register);
 9120   %}
 9121   ins_pipe(ialu_reg_reg);
 9122 %}
 9123 
 9124 // Shift Left by variable
 9125 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9126 %{
 9127   predicate(!VM_Version::supports_bmi2());
 9128   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9129   effect(KILL cr);
 9130 
 9131   format %{ "sall    $dst, $shift" %}
 9132   ins_encode %{
 9133     __ sall($dst$$Address);
 9134   %}
 9135   ins_pipe(ialu_mem_reg);
 9136 %}
 9137 
 9138 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9139 %{
 9140   predicate(VM_Version::supports_bmi2());
 9141   match(Set dst (LShiftI src shift));
 9142 
 9143   format %{ "shlxl   $dst, $src, $shift" %}
 9144   ins_encode %{
 9145     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9146   %}
 9147   ins_pipe(ialu_reg_reg);
 9148 %}
 9149 
 9150 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9151 %{
 9152   predicate(VM_Version::supports_bmi2());
 9153   match(Set dst (LShiftI (LoadI src) shift));
 9154   ins_cost(175);
 9155   format %{ "shlxl   $dst, $src, $shift" %}
 9156   ins_encode %{
 9157     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9158   %}
 9159   ins_pipe(ialu_reg_mem);
 9160 %}
 9161 
 9162 // Arithmetic Shift Right by 8-bit immediate
 9163 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9164 %{
 9165   predicate(!UseAPX);
 9166   match(Set dst (RShiftI dst shift));
 9167   effect(KILL cr);
 9168 
 9169   format %{ "sarl    $dst, $shift" %}
 9170   ins_encode %{
 9171     __ sarl($dst$$Register, $shift$$constant);
 9172   %}
 9173   ins_pipe(ialu_mem_imm);
 9174 %}
 9175 
 9176 // Arithmetic Shift Right by 8-bit immediate
 9177 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9178 %{
 9179   predicate(UseAPX);
 9180   match(Set dst (RShiftI src shift));
 9181   effect(KILL cr);
 9182 
 9183   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9184   ins_encode %{
 9185     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9186   %}
 9187   ins_pipe(ialu_mem_imm);
 9188 %}
 9189 
 9190 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9191 %{
 9192   predicate(UseAPX);
 9193   match(Set dst (RShiftI (LoadI src) shift));
 9194   effect(KILL cr);
 9195 
 9196   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9197   ins_encode %{
 9198     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9199   %}
 9200   ins_pipe(ialu_mem_imm);
 9201 %}
 9202 
 9203 // Arithmetic Shift Right by 8-bit immediate
 9204 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9205 %{
 9206   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9207   effect(KILL cr);
 9208 
 9209   format %{ "sarl    $dst, $shift" %}
 9210   ins_encode %{
 9211     __ sarl($dst$$Address, $shift$$constant);
 9212   %}
 9213   ins_pipe(ialu_mem_imm);
 9214 %}
 9215 
 9216 // Arithmetic Shift Right by variable
 9217 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9218 %{
 9219   predicate(!VM_Version::supports_bmi2());
 9220   match(Set dst (RShiftI dst shift));
 9221   effect(KILL cr);
 9222 
 9223   format %{ "sarl    $dst, $shift" %}
 9224   ins_encode %{
 9225     __ sarl($dst$$Register);
 9226   %}
 9227   ins_pipe(ialu_reg_reg);
 9228 %}
 9229 
 9230 // Arithmetic Shift Right by variable
 9231 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9232 %{
 9233   predicate(!VM_Version::supports_bmi2());
 9234   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9235   effect(KILL cr);
 9236 
 9237   format %{ "sarl    $dst, $shift" %}
 9238   ins_encode %{
 9239     __ sarl($dst$$Address);
 9240   %}
 9241   ins_pipe(ialu_mem_reg);
 9242 %}
 9243 
 9244 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9245 %{
 9246   predicate(VM_Version::supports_bmi2());
 9247   match(Set dst (RShiftI src shift));
 9248 
 9249   format %{ "sarxl   $dst, $src, $shift" %}
 9250   ins_encode %{
 9251     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9252   %}
 9253   ins_pipe(ialu_reg_reg);
 9254 %}
 9255 
 9256 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9257 %{
 9258   predicate(VM_Version::supports_bmi2());
 9259   match(Set dst (RShiftI (LoadI src) shift));
 9260   ins_cost(175);
 9261   format %{ "sarxl   $dst, $src, $shift" %}
 9262   ins_encode %{
 9263     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9264   %}
 9265   ins_pipe(ialu_reg_mem);
 9266 %}
 9267 
 9268 // Logical Shift Right by 8-bit immediate
 9269 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9270 %{
 9271   predicate(!UseAPX);
 9272   match(Set dst (URShiftI dst shift));
 9273   effect(KILL cr);
 9274 
 9275   format %{ "shrl    $dst, $shift" %}
 9276   ins_encode %{
 9277     __ shrl($dst$$Register, $shift$$constant);
 9278   %}
 9279   ins_pipe(ialu_reg);
 9280 %}
 9281 
 9282 // Logical Shift Right by 8-bit immediate
 9283 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9284 %{
 9285   predicate(UseAPX);
 9286   match(Set dst (URShiftI src shift));
 9287   effect(KILL cr);
 9288 
 9289   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9290   ins_encode %{
 9291     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9292   %}
 9293   ins_pipe(ialu_reg);
 9294 %}
 9295 
 9296 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9297 %{
 9298   predicate(UseAPX);
 9299   match(Set dst (URShiftI (LoadI src) shift));
 9300   effect(KILL cr);
 9301 
 9302   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9303   ins_encode %{
 9304     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9305   %}
 9306   ins_pipe(ialu_reg);
 9307 %}
 9308 
 9309 // Logical Shift Right by 8-bit immediate
 9310 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9311 %{
 9312   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9313   effect(KILL cr);
 9314 
 9315   format %{ "shrl    $dst, $shift" %}
 9316   ins_encode %{
 9317     __ shrl($dst$$Address, $shift$$constant);
 9318   %}
 9319   ins_pipe(ialu_mem_imm);
 9320 %}
 9321 
 9322 // Logical Shift Right by variable
 9323 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9324 %{
 9325   predicate(!VM_Version::supports_bmi2());
 9326   match(Set dst (URShiftI dst shift));
 9327   effect(KILL cr);
 9328 
 9329   format %{ "shrl    $dst, $shift" %}
 9330   ins_encode %{
 9331     __ shrl($dst$$Register);
 9332   %}
 9333   ins_pipe(ialu_reg_reg);
 9334 %}
 9335 
 9336 // Logical Shift Right by variable
 9337 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9338 %{
 9339   predicate(!VM_Version::supports_bmi2());
 9340   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9341   effect(KILL cr);
 9342 
 9343   format %{ "shrl    $dst, $shift" %}
 9344   ins_encode %{
 9345     __ shrl($dst$$Address);
 9346   %}
 9347   ins_pipe(ialu_mem_reg);
 9348 %}
 9349 
 9350 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9351 %{
 9352   predicate(VM_Version::supports_bmi2());
 9353   match(Set dst (URShiftI src shift));
 9354 
 9355   format %{ "shrxl   $dst, $src, $shift" %}
 9356   ins_encode %{
 9357     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9358   %}
 9359   ins_pipe(ialu_reg_reg);
 9360 %}
 9361 
 9362 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9363 %{
 9364   predicate(VM_Version::supports_bmi2());
 9365   match(Set dst (URShiftI (LoadI src) shift));
 9366   ins_cost(175);
 9367   format %{ "shrxl   $dst, $src, $shift" %}
 9368   ins_encode %{
 9369     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9370   %}
 9371   ins_pipe(ialu_reg_mem);
 9372 %}
 9373 
 9374 // Long Shift Instructions
 9375 // Shift Left by one, two, three
 9376 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9377 %{
 9378   predicate(!UseAPX);
 9379   match(Set dst (LShiftL dst shift));
 9380   effect(KILL cr);
 9381 
 9382   format %{ "salq    $dst, $shift" %}
 9383   ins_encode %{
 9384     __ salq($dst$$Register, $shift$$constant);
 9385   %}
 9386   ins_pipe(ialu_reg);
 9387 %}
 9388 
 9389 // Shift Left by one, two, three
 9390 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9391 %{
 9392   predicate(UseAPX);
 9393   match(Set dst (LShiftL src shift));
 9394   effect(KILL cr);
 9395 
 9396   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9397   ins_encode %{
 9398     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9399   %}
 9400   ins_pipe(ialu_reg);
 9401 %}
 9402 
 9403 // Shift Left by 8-bit immediate
 9404 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9405 %{
 9406   predicate(!UseAPX);
 9407   match(Set dst (LShiftL dst shift));
 9408   effect(KILL cr);
 9409 
 9410   format %{ "salq    $dst, $shift" %}
 9411   ins_encode %{
 9412     __ salq($dst$$Register, $shift$$constant);
 9413   %}
 9414   ins_pipe(ialu_reg);
 9415 %}
 9416 
 9417 // Shift Left by 8-bit immediate
 9418 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9419 %{
 9420   predicate(UseAPX);
 9421   match(Set dst (LShiftL src shift));
 9422   effect(KILL cr);
 9423 
 9424   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9425   ins_encode %{
 9426     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9427   %}
 9428   ins_pipe(ialu_reg);
 9429 %}
 9430 
 9431 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9432 %{
 9433   predicate(UseAPX);
 9434   match(Set dst (LShiftL (LoadL src) shift));
 9435   effect(KILL cr);
 9436 
 9437   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9438   ins_encode %{
 9439     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9440   %}
 9441   ins_pipe(ialu_reg);
 9442 %}
 9443 
 9444 // Shift Left by 8-bit immediate
 9445 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9446 %{
 9447   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9448   effect(KILL cr);
 9449 
 9450   format %{ "salq    $dst, $shift" %}
 9451   ins_encode %{
 9452     __ salq($dst$$Address, $shift$$constant);
 9453   %}
 9454   ins_pipe(ialu_mem_imm);
 9455 %}
 9456 
 9457 // Shift Left by variable
 9458 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9459 %{
 9460   predicate(!VM_Version::supports_bmi2());
 9461   match(Set dst (LShiftL dst shift));
 9462   effect(KILL cr);
 9463 
 9464   format %{ "salq    $dst, $shift" %}
 9465   ins_encode %{
 9466     __ salq($dst$$Register);
 9467   %}
 9468   ins_pipe(ialu_reg_reg);
 9469 %}
 9470 
 9471 // Shift Left by variable
 9472 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9473 %{
 9474   predicate(!VM_Version::supports_bmi2());
 9475   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9476   effect(KILL cr);
 9477 
 9478   format %{ "salq    $dst, $shift" %}
 9479   ins_encode %{
 9480     __ salq($dst$$Address);
 9481   %}
 9482   ins_pipe(ialu_mem_reg);
 9483 %}
 9484 
 9485 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9486 %{
 9487   predicate(VM_Version::supports_bmi2());
 9488   match(Set dst (LShiftL src shift));
 9489 
 9490   format %{ "shlxq   $dst, $src, $shift" %}
 9491   ins_encode %{
 9492     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9493   %}
 9494   ins_pipe(ialu_reg_reg);
 9495 %}
 9496 
 9497 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9498 %{
 9499   predicate(VM_Version::supports_bmi2());
 9500   match(Set dst (LShiftL (LoadL src) shift));
 9501   ins_cost(175);
 9502   format %{ "shlxq   $dst, $src, $shift" %}
 9503   ins_encode %{
 9504     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9505   %}
 9506   ins_pipe(ialu_reg_mem);
 9507 %}
 9508 
 9509 // Arithmetic Shift Right by 8-bit immediate
 9510 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9511 %{
 9512   predicate(!UseAPX);
 9513   match(Set dst (RShiftL dst shift));
 9514   effect(KILL cr);
 9515 
 9516   format %{ "sarq    $dst, $shift" %}
 9517   ins_encode %{
 9518     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9519   %}
 9520   ins_pipe(ialu_mem_imm);
 9521 %}
 9522 
 9523 // Arithmetic Shift Right by 8-bit immediate
 9524 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9525 %{
 9526   predicate(UseAPX);
 9527   match(Set dst (RShiftL src shift));
 9528   effect(KILL cr);
 9529 
 9530   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9531   ins_encode %{
 9532     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9533   %}
 9534   ins_pipe(ialu_mem_imm);
 9535 %}
 9536 
 9537 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9538 %{
 9539   predicate(UseAPX);
 9540   match(Set dst (RShiftL (LoadL src) shift));
 9541   effect(KILL cr);
 9542 
 9543   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9544   ins_encode %{
 9545     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9546   %}
 9547   ins_pipe(ialu_mem_imm);
 9548 %}
 9549 
 9550 // Arithmetic Shift Right by 8-bit immediate
 9551 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9552 %{
 9553   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9554   effect(KILL cr);
 9555 
 9556   format %{ "sarq    $dst, $shift" %}
 9557   ins_encode %{
 9558     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9559   %}
 9560   ins_pipe(ialu_mem_imm);
 9561 %}
 9562 
 9563 // Arithmetic Shift Right by variable
 9564 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9565 %{
 9566   predicate(!VM_Version::supports_bmi2());
 9567   match(Set dst (RShiftL dst shift));
 9568   effect(KILL cr);
 9569 
 9570   format %{ "sarq    $dst, $shift" %}
 9571   ins_encode %{
 9572     __ sarq($dst$$Register);
 9573   %}
 9574   ins_pipe(ialu_reg_reg);
 9575 %}
 9576 
 9577 // Arithmetic Shift Right by variable
 9578 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9579 %{
 9580   predicate(!VM_Version::supports_bmi2());
 9581   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9582   effect(KILL cr);
 9583 
 9584   format %{ "sarq    $dst, $shift" %}
 9585   ins_encode %{
 9586     __ sarq($dst$$Address);
 9587   %}
 9588   ins_pipe(ialu_mem_reg);
 9589 %}
 9590 
 9591 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9592 %{
 9593   predicate(VM_Version::supports_bmi2());
 9594   match(Set dst (RShiftL src shift));
 9595 
 9596   format %{ "sarxq   $dst, $src, $shift" %}
 9597   ins_encode %{
 9598     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9599   %}
 9600   ins_pipe(ialu_reg_reg);
 9601 %}
 9602 
 9603 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9604 %{
 9605   predicate(VM_Version::supports_bmi2());
 9606   match(Set dst (RShiftL (LoadL src) shift));
 9607   ins_cost(175);
 9608   format %{ "sarxq   $dst, $src, $shift" %}
 9609   ins_encode %{
 9610     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9611   %}
 9612   ins_pipe(ialu_reg_mem);
 9613 %}
 9614 
 9615 // Logical Shift Right by 8-bit immediate
 9616 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9617 %{
 9618   predicate(!UseAPX);
 9619   match(Set dst (URShiftL dst shift));
 9620   effect(KILL cr);
 9621 
 9622   format %{ "shrq    $dst, $shift" %}
 9623   ins_encode %{
 9624     __ shrq($dst$$Register, $shift$$constant);
 9625   %}
 9626   ins_pipe(ialu_reg);
 9627 %}
 9628 
 9629 // Logical Shift Right by 8-bit immediate
 9630 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9631 %{
 9632   predicate(UseAPX);
 9633   match(Set dst (URShiftL src shift));
 9634   effect(KILL cr);
 9635 
 9636   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9637   ins_encode %{
 9638     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9639   %}
 9640   ins_pipe(ialu_reg);
 9641 %}
 9642 
 9643 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9644 %{
 9645   predicate(UseAPX);
 9646   match(Set dst (URShiftL (LoadL src) shift));
 9647   effect(KILL cr);
 9648 
 9649   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9650   ins_encode %{
 9651     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9652   %}
 9653   ins_pipe(ialu_reg);
 9654 %}
 9655 
 9656 // Logical Shift Right by 8-bit immediate
 9657 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9658 %{
 9659   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9660   effect(KILL cr);
 9661 
 9662   format %{ "shrq    $dst, $shift" %}
 9663   ins_encode %{
 9664     __ shrq($dst$$Address, $shift$$constant);
 9665   %}
 9666   ins_pipe(ialu_mem_imm);
 9667 %}
 9668 
 9669 // Logical Shift Right by variable
 9670 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9671 %{
 9672   predicate(!VM_Version::supports_bmi2());
 9673   match(Set dst (URShiftL dst shift));
 9674   effect(KILL cr);
 9675 
 9676   format %{ "shrq    $dst, $shift" %}
 9677   ins_encode %{
 9678     __ shrq($dst$$Register);
 9679   %}
 9680   ins_pipe(ialu_reg_reg);
 9681 %}
 9682 
 9683 // Logical Shift Right by variable
 9684 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9685 %{
 9686   predicate(!VM_Version::supports_bmi2());
 9687   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9688   effect(KILL cr);
 9689 
 9690   format %{ "shrq    $dst, $shift" %}
 9691   ins_encode %{
 9692     __ shrq($dst$$Address);
 9693   %}
 9694   ins_pipe(ialu_mem_reg);
 9695 %}
 9696 
 9697 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9698 %{
 9699   predicate(VM_Version::supports_bmi2());
 9700   match(Set dst (URShiftL src shift));
 9701 
 9702   format %{ "shrxq   $dst, $src, $shift" %}
 9703   ins_encode %{
 9704     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9705   %}
 9706   ins_pipe(ialu_reg_reg);
 9707 %}
 9708 
 9709 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9710 %{
 9711   predicate(VM_Version::supports_bmi2());
 9712   match(Set dst (URShiftL (LoadL src) shift));
 9713   ins_cost(175);
 9714   format %{ "shrxq   $dst, $src, $shift" %}
 9715   ins_encode %{
 9716     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9717   %}
 9718   ins_pipe(ialu_reg_mem);
 9719 %}
 9720 
 9721 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9722 // This idiom is used by the compiler for the i2b bytecode.
 9723 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9724 %{
 9725   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9726 
 9727   format %{ "movsbl  $dst, $src\t# i2b" %}
 9728   ins_encode %{
 9729     __ movsbl($dst$$Register, $src$$Register);
 9730   %}
 9731   ins_pipe(ialu_reg_reg);
 9732 %}
 9733 
 9734 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9735 // This idiom is used by the compiler the i2s bytecode.
 9736 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9737 %{
 9738   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9739 
 9740   format %{ "movswl  $dst, $src\t# i2s" %}
 9741   ins_encode %{
 9742     __ movswl($dst$$Register, $src$$Register);
 9743   %}
 9744   ins_pipe(ialu_reg_reg);
 9745 %}
 9746 
 9747 // ROL/ROR instructions
 9748 
 9749 // Rotate left by constant.
 9750 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9751 %{
 9752   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9753   match(Set dst (RotateLeft dst shift));
 9754   effect(KILL cr);
 9755   format %{ "roll    $dst, $shift" %}
 9756   ins_encode %{
 9757     __ roll($dst$$Register, $shift$$constant);
 9758   %}
 9759   ins_pipe(ialu_reg);
 9760 %}
 9761 
 9762 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9763 %{
 9764   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9765   match(Set dst (RotateLeft src shift));
 9766   format %{ "rolxl   $dst, $src, $shift" %}
 9767   ins_encode %{
 9768     int shift = 32 - ($shift$$constant & 31);
 9769     __ rorxl($dst$$Register, $src$$Register, shift);
 9770   %}
 9771   ins_pipe(ialu_reg_reg);
 9772 %}
 9773 
 9774 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9775 %{
 9776   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9777   match(Set dst (RotateLeft (LoadI src) shift));
 9778   ins_cost(175);
 9779   format %{ "rolxl   $dst, $src, $shift" %}
 9780   ins_encode %{
 9781     int shift = 32 - ($shift$$constant & 31);
 9782     __ rorxl($dst$$Register, $src$$Address, shift);
 9783   %}
 9784   ins_pipe(ialu_reg_mem);
 9785 %}
 9786 
 9787 // Rotate Left by variable
 9788 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9789 %{
 9790   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9791   match(Set dst (RotateLeft dst shift));
 9792   effect(KILL cr);
 9793   format %{ "roll    $dst, $shift" %}
 9794   ins_encode %{
 9795     __ roll($dst$$Register);
 9796   %}
 9797   ins_pipe(ialu_reg_reg);
 9798 %}
 9799 
 9800 // Rotate Left by variable
 9801 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9802 %{
 9803   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9804   match(Set dst (RotateLeft src shift));
 9805   effect(KILL cr);
 9806 
 9807   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9808   ins_encode %{
 9809     __ eroll($dst$$Register, $src$$Register, false);
 9810   %}
 9811   ins_pipe(ialu_reg_reg);
 9812 %}
 9813 
 9814 // Rotate Right by constant.
 9815 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9816 %{
 9817   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9818   match(Set dst (RotateRight dst shift));
 9819   effect(KILL cr);
 9820   format %{ "rorl    $dst, $shift" %}
 9821   ins_encode %{
 9822     __ rorl($dst$$Register, $shift$$constant);
 9823   %}
 9824   ins_pipe(ialu_reg);
 9825 %}
 9826 
 9827 // Rotate Right by constant.
 9828 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9829 %{
 9830   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9831   match(Set dst (RotateRight src shift));
 9832   format %{ "rorxl   $dst, $src, $shift" %}
 9833   ins_encode %{
 9834     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9835   %}
 9836   ins_pipe(ialu_reg_reg);
 9837 %}
 9838 
 9839 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9840 %{
 9841   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9842   match(Set dst (RotateRight (LoadI src) shift));
 9843   ins_cost(175);
 9844   format %{ "rorxl   $dst, $src, $shift" %}
 9845   ins_encode %{
 9846     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9847   %}
 9848   ins_pipe(ialu_reg_mem);
 9849 %}
 9850 
 9851 // Rotate Right by variable
 9852 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9853 %{
 9854   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9855   match(Set dst (RotateRight dst shift));
 9856   effect(KILL cr);
 9857   format %{ "rorl    $dst, $shift" %}
 9858   ins_encode %{
 9859     __ rorl($dst$$Register);
 9860   %}
 9861   ins_pipe(ialu_reg_reg);
 9862 %}
 9863 
 9864 // Rotate Right by variable
 9865 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9866 %{
 9867   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9868   match(Set dst (RotateRight src shift));
 9869   effect(KILL cr);
 9870 
 9871   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9872   ins_encode %{
 9873     __ erorl($dst$$Register, $src$$Register, false);
 9874   %}
 9875   ins_pipe(ialu_reg_reg);
 9876 %}
 9877 
 9878 // Rotate Left by constant.
 9879 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9880 %{
 9881   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9882   match(Set dst (RotateLeft dst shift));
 9883   effect(KILL cr);
 9884   format %{ "rolq    $dst, $shift" %}
 9885   ins_encode %{
 9886     __ rolq($dst$$Register, $shift$$constant);
 9887   %}
 9888   ins_pipe(ialu_reg);
 9889 %}
 9890 
 9891 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9892 %{
 9893   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9894   match(Set dst (RotateLeft src shift));
 9895   format %{ "rolxq   $dst, $src, $shift" %}
 9896   ins_encode %{
 9897     int shift = 64 - ($shift$$constant & 63);
 9898     __ rorxq($dst$$Register, $src$$Register, shift);
 9899   %}
 9900   ins_pipe(ialu_reg_reg);
 9901 %}
 9902 
 9903 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9904 %{
 9905   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9906   match(Set dst (RotateLeft (LoadL src) shift));
 9907   ins_cost(175);
 9908   format %{ "rolxq   $dst, $src, $shift" %}
 9909   ins_encode %{
 9910     int shift = 64 - ($shift$$constant & 63);
 9911     __ rorxq($dst$$Register, $src$$Address, shift);
 9912   %}
 9913   ins_pipe(ialu_reg_mem);
 9914 %}
 9915 
 9916 // Rotate Left by variable
 9917 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9918 %{
 9919   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9920   match(Set dst (RotateLeft dst shift));
 9921   effect(KILL cr);
 9922   format %{ "rolq    $dst, $shift" %}
 9923   ins_encode %{
 9924     __ rolq($dst$$Register);
 9925   %}
 9926   ins_pipe(ialu_reg_reg);
 9927 %}
 9928 
 9929 // Rotate Left by variable
 9930 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9931 %{
 9932   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9933   match(Set dst (RotateLeft src shift));
 9934   effect(KILL cr);
 9935 
 9936   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9937   ins_encode %{
 9938     __ erolq($dst$$Register, $src$$Register, false);
 9939   %}
 9940   ins_pipe(ialu_reg_reg);
 9941 %}
 9942 
 9943 // Rotate Right by constant.
 9944 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9945 %{
 9946   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9947   match(Set dst (RotateRight dst shift));
 9948   effect(KILL cr);
 9949   format %{ "rorq    $dst, $shift" %}
 9950   ins_encode %{
 9951     __ rorq($dst$$Register, $shift$$constant);
 9952   %}
 9953   ins_pipe(ialu_reg);
 9954 %}
 9955 
 9956 // Rotate Right by constant
 9957 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9958 %{
 9959   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9960   match(Set dst (RotateRight src shift));
 9961   format %{ "rorxq   $dst, $src, $shift" %}
 9962   ins_encode %{
 9963     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9964   %}
 9965   ins_pipe(ialu_reg_reg);
 9966 %}
 9967 
 9968 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9969 %{
 9970   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9971   match(Set dst (RotateRight (LoadL src) shift));
 9972   ins_cost(175);
 9973   format %{ "rorxq   $dst, $src, $shift" %}
 9974   ins_encode %{
 9975     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9976   %}
 9977   ins_pipe(ialu_reg_mem);
 9978 %}
 9979 
 9980 // Rotate Right by variable
 9981 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9982 %{
 9983   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9984   match(Set dst (RotateRight dst shift));
 9985   effect(KILL cr);
 9986   format %{ "rorq    $dst, $shift" %}
 9987   ins_encode %{
 9988     __ rorq($dst$$Register);
 9989   %}
 9990   ins_pipe(ialu_reg_reg);
 9991 %}
 9992 
 9993 // Rotate Right by variable
 9994 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9995 %{
 9996   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9997   match(Set dst (RotateRight src shift));
 9998   effect(KILL cr);
 9999 
10000   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
10001   ins_encode %{
10002     __ erorq($dst$$Register, $src$$Register, false);
10003   %}
10004   ins_pipe(ialu_reg_reg);
10005 %}
10006 
10007 //----------------------------- CompressBits/ExpandBits ------------------------
10008 
10009 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10010   predicate(n->bottom_type()->isa_long());
10011   match(Set dst (CompressBits src mask));
10012   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10013   ins_encode %{
10014     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
10015   %}
10016   ins_pipe( pipe_slow );
10017 %}
10018 
10019 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10020   predicate(n->bottom_type()->isa_long());
10021   match(Set dst (ExpandBits src mask));
10022   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10023   ins_encode %{
10024     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10025   %}
10026   ins_pipe( pipe_slow );
10027 %}
10028 
10029 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10030   predicate(n->bottom_type()->isa_long());
10031   match(Set dst (CompressBits src (LoadL mask)));
10032   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10033   ins_encode %{
10034     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10035   %}
10036   ins_pipe( pipe_slow );
10037 %}
10038 
10039 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10040   predicate(n->bottom_type()->isa_long());
10041   match(Set dst (ExpandBits src (LoadL mask)));
10042   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10043   ins_encode %{
10044     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10045   %}
10046   ins_pipe( pipe_slow );
10047 %}
10048 
10049 
10050 // Logical Instructions
10051 
10052 // Integer Logical Instructions
10053 
10054 // And Instructions
10055 // And Register with Register
10056 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10057 %{
10058   predicate(!UseAPX);
10059   match(Set dst (AndI dst src));
10060   effect(KILL cr);
10061   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10062 
10063   format %{ "andl    $dst, $src\t# int" %}
10064   ins_encode %{
10065     __ andl($dst$$Register, $src$$Register);
10066   %}
10067   ins_pipe(ialu_reg_reg);
10068 %}
10069 
10070 // And Register with Register using New Data Destination (NDD)
10071 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10072 %{
10073   predicate(UseAPX);
10074   match(Set dst (AndI src1 src2));
10075   effect(KILL cr);
10076   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10077 
10078   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10079   ins_encode %{
10080     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10081 
10082   %}
10083   ins_pipe(ialu_reg_reg);
10084 %}
10085 
10086 // And Register with Immediate 255
10087 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10088 %{
10089   match(Set dst (AndI src mask));
10090 
10091   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10092   ins_encode %{
10093     __ movzbl($dst$$Register, $src$$Register);
10094   %}
10095   ins_pipe(ialu_reg);
10096 %}
10097 
10098 // And Register with Immediate 255 and promote to long
10099 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10100 %{
10101   match(Set dst (ConvI2L (AndI src mask)));
10102 
10103   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10104   ins_encode %{
10105     __ movzbl($dst$$Register, $src$$Register);
10106   %}
10107   ins_pipe(ialu_reg);
10108 %}
10109 
10110 // And Register with Immediate 65535
10111 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10112 %{
10113   match(Set dst (AndI src mask));
10114 
10115   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10116   ins_encode %{
10117     __ movzwl($dst$$Register, $src$$Register);
10118   %}
10119   ins_pipe(ialu_reg);
10120 %}
10121 
10122 // And Register with Immediate 65535 and promote to long
10123 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10124 %{
10125   match(Set dst (ConvI2L (AndI src mask)));
10126 
10127   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10128   ins_encode %{
10129     __ movzwl($dst$$Register, $src$$Register);
10130   %}
10131   ins_pipe(ialu_reg);
10132 %}
10133 
10134 // Can skip int2long conversions after AND with small bitmask
10135 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10136 %{
10137   predicate(VM_Version::supports_bmi2());
10138   ins_cost(125);
10139   effect(TEMP tmp, KILL cr);
10140   match(Set dst (ConvI2L (AndI src mask)));
10141   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10142   ins_encode %{
10143     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10144     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10145   %}
10146   ins_pipe(ialu_reg_reg);
10147 %}
10148 
10149 // And Register with Immediate
10150 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10151 %{
10152   predicate(!UseAPX);
10153   match(Set dst (AndI dst src));
10154   effect(KILL cr);
10155   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10156 
10157   format %{ "andl    $dst, $src\t# int" %}
10158   ins_encode %{
10159     __ andl($dst$$Register, $src$$constant);
10160   %}
10161   ins_pipe(ialu_reg);
10162 %}
10163 
10164 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10165 %{
10166   predicate(UseAPX);
10167   match(Set dst (AndI src1 src2));
10168   effect(KILL cr);
10169   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10170 
10171   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10172   ins_encode %{
10173     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10174   %}
10175   ins_pipe(ialu_reg);
10176 %}
10177 
10178 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10179 %{
10180   predicate(UseAPX);
10181   match(Set dst (AndI (LoadI src1) src2));
10182   effect(KILL cr);
10183   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10184 
10185   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10186   ins_encode %{
10187     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10188   %}
10189   ins_pipe(ialu_reg);
10190 %}
10191 
10192 // And Register with Memory
10193 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10194 %{
10195   predicate(!UseAPX);
10196   match(Set dst (AndI dst (LoadI src)));
10197   effect(KILL cr);
10198   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10199 
10200   ins_cost(150);
10201   format %{ "andl    $dst, $src\t# int" %}
10202   ins_encode %{
10203     __ andl($dst$$Register, $src$$Address);
10204   %}
10205   ins_pipe(ialu_reg_mem);
10206 %}
10207 
10208 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10209 %{
10210   predicate(UseAPX);
10211   match(Set dst (AndI src1 (LoadI src2)));
10212   effect(KILL cr);
10213   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10214 
10215   ins_cost(150);
10216   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10217   ins_encode %{
10218     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10219   %}
10220   ins_pipe(ialu_reg_mem);
10221 %}
10222 
10223 // And Memory with Register
10224 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10225 %{
10226   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10227   effect(KILL cr);
10228   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10229 
10230   ins_cost(150);
10231   format %{ "andb    $dst, $src\t# byte" %}
10232   ins_encode %{
10233     __ andb($dst$$Address, $src$$Register);
10234   %}
10235   ins_pipe(ialu_mem_reg);
10236 %}
10237 
10238 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10239 %{
10240   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10241   effect(KILL cr);
10242   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10243 
10244   ins_cost(150);
10245   format %{ "andl    $dst, $src\t# int" %}
10246   ins_encode %{
10247     __ andl($dst$$Address, $src$$Register);
10248   %}
10249   ins_pipe(ialu_mem_reg);
10250 %}
10251 
10252 // And Memory with Immediate
10253 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10254 %{
10255   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10256   effect(KILL cr);
10257   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10258 
10259   ins_cost(125);
10260   format %{ "andl    $dst, $src\t# int" %}
10261   ins_encode %{
10262     __ andl($dst$$Address, $src$$constant);
10263   %}
10264   ins_pipe(ialu_mem_imm);
10265 %}
10266 
10267 // BMI1 instructions
10268 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10269   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10270   predicate(UseBMI1Instructions);
10271   effect(KILL cr);
10272   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10273 
10274   ins_cost(125);
10275   format %{ "andnl  $dst, $src1, $src2" %}
10276 
10277   ins_encode %{
10278     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10279   %}
10280   ins_pipe(ialu_reg_mem);
10281 %}
10282 
10283 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10284   match(Set dst (AndI (XorI src1 minus_1) src2));
10285   predicate(UseBMI1Instructions);
10286   effect(KILL cr);
10287   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10288 
10289   format %{ "andnl  $dst, $src1, $src2" %}
10290 
10291   ins_encode %{
10292     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10293   %}
10294   ins_pipe(ialu_reg);
10295 %}
10296 
10297 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10298   match(Set dst (AndI (SubI imm_zero src) src));
10299   predicate(UseBMI1Instructions);
10300   effect(KILL cr);
10301   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10302 
10303   format %{ "blsil  $dst, $src" %}
10304 
10305   ins_encode %{
10306     __ blsil($dst$$Register, $src$$Register);
10307   %}
10308   ins_pipe(ialu_reg);
10309 %}
10310 
10311 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10312   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10313   predicate(UseBMI1Instructions);
10314   effect(KILL cr);
10315   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10316 
10317   ins_cost(125);
10318   format %{ "blsil  $dst, $src" %}
10319 
10320   ins_encode %{
10321     __ blsil($dst$$Register, $src$$Address);
10322   %}
10323   ins_pipe(ialu_reg_mem);
10324 %}
10325 
10326 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10327 %{
10328   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10329   predicate(UseBMI1Instructions);
10330   effect(KILL cr);
10331   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10332 
10333   ins_cost(125);
10334   format %{ "blsmskl $dst, $src" %}
10335 
10336   ins_encode %{
10337     __ blsmskl($dst$$Register, $src$$Address);
10338   %}
10339   ins_pipe(ialu_reg_mem);
10340 %}
10341 
10342 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10343 %{
10344   match(Set dst (XorI (AddI src minus_1) src));
10345   predicate(UseBMI1Instructions);
10346   effect(KILL cr);
10347   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10348 
10349   format %{ "blsmskl $dst, $src" %}
10350 
10351   ins_encode %{
10352     __ blsmskl($dst$$Register, $src$$Register);
10353   %}
10354 
10355   ins_pipe(ialu_reg);
10356 %}
10357 
10358 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10359 %{
10360   match(Set dst (AndI (AddI src minus_1) src) );
10361   predicate(UseBMI1Instructions);
10362   effect(KILL cr);
10363   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10364 
10365   format %{ "blsrl  $dst, $src" %}
10366 
10367   ins_encode %{
10368     __ blsrl($dst$$Register, $src$$Register);
10369   %}
10370 
10371   ins_pipe(ialu_reg_mem);
10372 %}
10373 
10374 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10375 %{
10376   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10377   predicate(UseBMI1Instructions);
10378   effect(KILL cr);
10379   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10380 
10381   ins_cost(125);
10382   format %{ "blsrl  $dst, $src" %}
10383 
10384   ins_encode %{
10385     __ blsrl($dst$$Register, $src$$Address);
10386   %}
10387 
10388   ins_pipe(ialu_reg);
10389 %}
10390 
10391 // Or Instructions
10392 // Or Register with Register
10393 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10394 %{
10395   predicate(!UseAPX);
10396   match(Set dst (OrI dst src));
10397   effect(KILL cr);
10398   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10399 
10400   format %{ "orl     $dst, $src\t# int" %}
10401   ins_encode %{
10402     __ orl($dst$$Register, $src$$Register);
10403   %}
10404   ins_pipe(ialu_reg_reg);
10405 %}
10406 
10407 // Or Register with Register using New Data Destination (NDD)
10408 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10409 %{
10410   predicate(UseAPX);
10411   match(Set dst (OrI src1 src2));
10412   effect(KILL cr);
10413   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10414 
10415   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10416   ins_encode %{
10417     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10418   %}
10419   ins_pipe(ialu_reg_reg);
10420 %}
10421 
10422 // Or Register with Immediate
10423 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10424 %{
10425   predicate(!UseAPX);
10426   match(Set dst (OrI dst src));
10427   effect(KILL cr);
10428   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10429 
10430   format %{ "orl     $dst, $src\t# int" %}
10431   ins_encode %{
10432     __ orl($dst$$Register, $src$$constant);
10433   %}
10434   ins_pipe(ialu_reg);
10435 %}
10436 
10437 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10438 %{
10439   predicate(UseAPX);
10440   match(Set dst (OrI src1 src2));
10441   effect(KILL cr);
10442   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10443 
10444   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10445   ins_encode %{
10446     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10447   %}
10448   ins_pipe(ialu_reg);
10449 %}
10450 
10451 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10452 %{
10453   predicate(UseAPX);
10454   match(Set dst (OrI src1 src2));
10455   effect(KILL cr);
10456   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10457 
10458   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10459   ins_encode %{
10460     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10461   %}
10462   ins_pipe(ialu_reg);
10463 %}
10464 
10465 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10466 %{
10467   predicate(UseAPX);
10468   match(Set dst (OrI (LoadI src1) src2));
10469   effect(KILL cr);
10470   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10471 
10472   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10473   ins_encode %{
10474     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10475   %}
10476   ins_pipe(ialu_reg);
10477 %}
10478 
10479 // Or Register with Memory
10480 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10481 %{
10482   predicate(!UseAPX);
10483   match(Set dst (OrI dst (LoadI src)));
10484   effect(KILL cr);
10485   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10486 
10487   ins_cost(150);
10488   format %{ "orl     $dst, $src\t# int" %}
10489   ins_encode %{
10490     __ orl($dst$$Register, $src$$Address);
10491   %}
10492   ins_pipe(ialu_reg_mem);
10493 %}
10494 
10495 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10496 %{
10497   predicate(UseAPX);
10498   match(Set dst (OrI src1 (LoadI src2)));
10499   effect(KILL cr);
10500   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10501 
10502   ins_cost(150);
10503   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10504   ins_encode %{
10505     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10506   %}
10507   ins_pipe(ialu_reg_mem);
10508 %}
10509 
10510 // Or Memory with Register
10511 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10512 %{
10513   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10514   effect(KILL cr);
10515   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10516 
10517   ins_cost(150);
10518   format %{ "orb    $dst, $src\t# byte" %}
10519   ins_encode %{
10520     __ orb($dst$$Address, $src$$Register);
10521   %}
10522   ins_pipe(ialu_mem_reg);
10523 %}
10524 
10525 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10526 %{
10527   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10528   effect(KILL cr);
10529   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10530 
10531   ins_cost(150);
10532   format %{ "orl     $dst, $src\t# int" %}
10533   ins_encode %{
10534     __ orl($dst$$Address, $src$$Register);
10535   %}
10536   ins_pipe(ialu_mem_reg);
10537 %}
10538 
10539 // Or Memory with Immediate
10540 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10541 %{
10542   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10543   effect(KILL cr);
10544   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10545 
10546   ins_cost(125);
10547   format %{ "orl     $dst, $src\t# int" %}
10548   ins_encode %{
10549     __ orl($dst$$Address, $src$$constant);
10550   %}
10551   ins_pipe(ialu_mem_imm);
10552 %}
10553 
10554 // Xor Instructions
10555 // Xor Register with Register
10556 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10557 %{
10558   predicate(!UseAPX);
10559   match(Set dst (XorI dst src));
10560   effect(KILL cr);
10561   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10562 
10563   format %{ "xorl    $dst, $src\t# int" %}
10564   ins_encode %{
10565     __ xorl($dst$$Register, $src$$Register);
10566   %}
10567   ins_pipe(ialu_reg_reg);
10568 %}
10569 
10570 // Xor Register with Register using New Data Destination (NDD)
10571 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10572 %{
10573   predicate(UseAPX);
10574   match(Set dst (XorI src1 src2));
10575   effect(KILL cr);
10576   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10577 
10578   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10579   ins_encode %{
10580     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10581   %}
10582   ins_pipe(ialu_reg_reg);
10583 %}
10584 
10585 // Xor Register with Immediate -1
10586 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10587 %{
10588   predicate(!UseAPX);
10589   match(Set dst (XorI dst imm));
10590 
10591   format %{ "notl    $dst" %}
10592   ins_encode %{
10593      __ notl($dst$$Register);
10594   %}
10595   ins_pipe(ialu_reg);
10596 %}
10597 
10598 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10599 %{
10600   match(Set dst (XorI src imm));
10601   predicate(UseAPX);
10602 
10603   format %{ "enotl    $dst, $src" %}
10604   ins_encode %{
10605      __ enotl($dst$$Register, $src$$Register);
10606   %}
10607   ins_pipe(ialu_reg);
10608 %}
10609 
10610 // Xor Register with Immediate
10611 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10612 %{
10613   predicate(!UseAPX);
10614   match(Set dst (XorI dst src));
10615   effect(KILL cr);
10616   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10617 
10618   format %{ "xorl    $dst, $src\t# int" %}
10619   ins_encode %{
10620     __ xorl($dst$$Register, $src$$constant);
10621   %}
10622   ins_pipe(ialu_reg);
10623 %}
10624 
10625 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10626 %{
10627   predicate(UseAPX);
10628   match(Set dst (XorI src1 src2));
10629   effect(KILL cr);
10630   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10631 
10632   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10633   ins_encode %{
10634     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10635   %}
10636   ins_pipe(ialu_reg);
10637 %}
10638 
10639 // Xor Memory with Immediate
10640 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10641 %{
10642   predicate(UseAPX);
10643   match(Set dst (XorI (LoadI src1) src2));
10644   effect(KILL cr);
10645   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10646 
10647   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10648   ins_encode %{
10649     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10650   %}
10651   ins_pipe(ialu_reg);
10652 %}
10653 
10654 // Xor Register with Memory
10655 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10656 %{
10657   predicate(!UseAPX);
10658   match(Set dst (XorI dst (LoadI src)));
10659   effect(KILL cr);
10660   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10661 
10662   ins_cost(150);
10663   format %{ "xorl    $dst, $src\t# int" %}
10664   ins_encode %{
10665     __ xorl($dst$$Register, $src$$Address);
10666   %}
10667   ins_pipe(ialu_reg_mem);
10668 %}
10669 
10670 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10671 %{
10672   predicate(UseAPX);
10673   match(Set dst (XorI src1 (LoadI src2)));
10674   effect(KILL cr);
10675   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10676 
10677   ins_cost(150);
10678   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10679   ins_encode %{
10680     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10681   %}
10682   ins_pipe(ialu_reg_mem);
10683 %}
10684 
10685 instruct xorI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
10686 %{
10687   predicate(UseAPX);
10688   match(Set dst (XorI (LoadI src1) src2));
10689   effect(KILL cr);
10690   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10691 
10692   ins_cost(150);
10693   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10694   ins_encode %{
10695     __ exorl($dst$$Register, $src1$$Address, $src2$$Register, false);
10696   %}
10697   ins_pipe(ialu_reg_mem);
10698 %}
10699 
10700 // Xor Memory with Register
10701 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10702 %{
10703   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10704   effect(KILL cr);
10705   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10706 
10707   ins_cost(150);
10708   format %{ "xorb    $dst, $src\t# byte" %}
10709   ins_encode %{
10710     __ xorb($dst$$Address, $src$$Register);
10711   %}
10712   ins_pipe(ialu_mem_reg);
10713 %}
10714 
10715 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10716 %{
10717   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10718   effect(KILL cr);
10719   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10720 
10721   ins_cost(150);
10722   format %{ "xorl    $dst, $src\t# int" %}
10723   ins_encode %{
10724     __ xorl($dst$$Address, $src$$Register);
10725   %}
10726   ins_pipe(ialu_mem_reg);
10727 %}
10728 
10729 // Xor Memory with Immediate
10730 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10731 %{
10732   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10733   effect(KILL cr);
10734   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10735 
10736   ins_cost(125);
10737   format %{ "xorl    $dst, $src\t# int" %}
10738   ins_encode %{
10739     __ xorl($dst$$Address, $src$$constant);
10740   %}
10741   ins_pipe(ialu_mem_imm);
10742 %}
10743 
10744 
10745 // Long Logical Instructions
10746 
10747 // And Instructions
10748 // And Register with Register
10749 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10750 %{
10751   predicate(!UseAPX);
10752   match(Set dst (AndL dst src));
10753   effect(KILL cr);
10754   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10755 
10756   format %{ "andq    $dst, $src\t# long" %}
10757   ins_encode %{
10758     __ andq($dst$$Register, $src$$Register);
10759   %}
10760   ins_pipe(ialu_reg_reg);
10761 %}
10762 
10763 // And Register with Register using New Data Destination (NDD)
10764 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10765 %{
10766   predicate(UseAPX);
10767   match(Set dst (AndL src1 src2));
10768   effect(KILL cr);
10769   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10770 
10771   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10772   ins_encode %{
10773     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10774 
10775   %}
10776   ins_pipe(ialu_reg_reg);
10777 %}
10778 
10779 // And Register with Immediate 255
10780 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10781 %{
10782   match(Set dst (AndL src mask));
10783 
10784   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10785   ins_encode %{
10786     // movzbl zeroes out the upper 32-bit and does not need REX.W
10787     __ movzbl($dst$$Register, $src$$Register);
10788   %}
10789   ins_pipe(ialu_reg);
10790 %}
10791 
10792 // And Register with Immediate 65535
10793 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10794 %{
10795   match(Set dst (AndL src mask));
10796 
10797   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10798   ins_encode %{
10799     // movzwl zeroes out the upper 32-bit and does not need REX.W
10800     __ movzwl($dst$$Register, $src$$Register);
10801   %}
10802   ins_pipe(ialu_reg);
10803 %}
10804 
10805 // And Register with Immediate
10806 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10807 %{
10808   predicate(!UseAPX);
10809   match(Set dst (AndL dst src));
10810   effect(KILL cr);
10811   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10812 
10813   format %{ "andq    $dst, $src\t# long" %}
10814   ins_encode %{
10815     __ andq($dst$$Register, $src$$constant);
10816   %}
10817   ins_pipe(ialu_reg);
10818 %}
10819 
10820 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10821 %{
10822   predicate(UseAPX);
10823   match(Set dst (AndL src1 src2));
10824   effect(KILL cr);
10825   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10826 
10827   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10828   ins_encode %{
10829     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10830   %}
10831   ins_pipe(ialu_reg);
10832 %}
10833 
10834 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10835 %{
10836   predicate(UseAPX);
10837   match(Set dst (AndL (LoadL src1) src2));
10838   effect(KILL cr);
10839   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10840 
10841   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10842   ins_encode %{
10843     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10844   %}
10845   ins_pipe(ialu_reg);
10846 %}
10847 
10848 // And Register with Memory
10849 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10850 %{
10851   predicate(!UseAPX);
10852   match(Set dst (AndL dst (LoadL src)));
10853   effect(KILL cr);
10854   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10855 
10856   ins_cost(150);
10857   format %{ "andq    $dst, $src\t# long" %}
10858   ins_encode %{
10859     __ andq($dst$$Register, $src$$Address);
10860   %}
10861   ins_pipe(ialu_reg_mem);
10862 %}
10863 
10864 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10865 %{
10866   predicate(UseAPX);
10867   match(Set dst (AndL src1 (LoadL src2)));
10868   effect(KILL cr);
10869   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10870 
10871   ins_cost(150);
10872   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10873   ins_encode %{
10874     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10875   %}
10876   ins_pipe(ialu_reg_mem);
10877 %}
10878 
10879 instruct andL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
10880 %{
10881   predicate(UseAPX);
10882   match(Set dst (AndL (LoadL src1) src2));
10883   effect(KILL cr);
10884   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10885 
10886   ins_cost(150);
10887   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10888   ins_encode %{
10889     __ eandq($dst$$Register, $src1$$Address, $src2$$Register, false);
10890   %}
10891   ins_pipe(ialu_reg_mem);
10892 %}
10893 
10894 // And Memory with Register
10895 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10896 %{
10897   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10898   effect(KILL cr);
10899   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10900 
10901   ins_cost(150);
10902   format %{ "andq    $dst, $src\t# long" %}
10903   ins_encode %{
10904     __ andq($dst$$Address, $src$$Register);
10905   %}
10906   ins_pipe(ialu_mem_reg);
10907 %}
10908 
10909 // And Memory with Immediate
10910 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10911 %{
10912   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10913   effect(KILL cr);
10914   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10915 
10916   ins_cost(125);
10917   format %{ "andq    $dst, $src\t# long" %}
10918   ins_encode %{
10919     __ andq($dst$$Address, $src$$constant);
10920   %}
10921   ins_pipe(ialu_mem_imm);
10922 %}
10923 
10924 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10925 %{
10926   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10927   // because AND/OR works well enough for 8/32-bit values.
10928   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10929 
10930   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10931   effect(KILL cr);
10932 
10933   ins_cost(125);
10934   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10935   ins_encode %{
10936     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10937   %}
10938   ins_pipe(ialu_mem_imm);
10939 %}
10940 
10941 // BMI1 instructions
10942 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10943   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10944   predicate(UseBMI1Instructions);
10945   effect(KILL cr);
10946   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10947 
10948   ins_cost(125);
10949   format %{ "andnq  $dst, $src1, $src2" %}
10950 
10951   ins_encode %{
10952     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10953   %}
10954   ins_pipe(ialu_reg_mem);
10955 %}
10956 
10957 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10958   match(Set dst (AndL (XorL src1 minus_1) src2));
10959   predicate(UseBMI1Instructions);
10960   effect(KILL cr);
10961   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10962 
10963   format %{ "andnq  $dst, $src1, $src2" %}
10964 
10965   ins_encode %{
10966   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10967   %}
10968   ins_pipe(ialu_reg_mem);
10969 %}
10970 
10971 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10972   match(Set dst (AndL (SubL imm_zero src) src));
10973   predicate(UseBMI1Instructions);
10974   effect(KILL cr);
10975   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10976 
10977   format %{ "blsiq  $dst, $src" %}
10978 
10979   ins_encode %{
10980     __ blsiq($dst$$Register, $src$$Register);
10981   %}
10982   ins_pipe(ialu_reg);
10983 %}
10984 
10985 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10986   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10987   predicate(UseBMI1Instructions);
10988   effect(KILL cr);
10989   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10990 
10991   ins_cost(125);
10992   format %{ "blsiq  $dst, $src" %}
10993 
10994   ins_encode %{
10995     __ blsiq($dst$$Register, $src$$Address);
10996   %}
10997   ins_pipe(ialu_reg_mem);
10998 %}
10999 
11000 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11001 %{
11002   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
11003   predicate(UseBMI1Instructions);
11004   effect(KILL cr);
11005   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11006 
11007   ins_cost(125);
11008   format %{ "blsmskq $dst, $src" %}
11009 
11010   ins_encode %{
11011     __ blsmskq($dst$$Register, $src$$Address);
11012   %}
11013   ins_pipe(ialu_reg_mem);
11014 %}
11015 
11016 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11017 %{
11018   match(Set dst (XorL (AddL src minus_1) src));
11019   predicate(UseBMI1Instructions);
11020   effect(KILL cr);
11021   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11022 
11023   format %{ "blsmskq $dst, $src" %}
11024 
11025   ins_encode %{
11026     __ blsmskq($dst$$Register, $src$$Register);
11027   %}
11028 
11029   ins_pipe(ialu_reg);
11030 %}
11031 
11032 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11033 %{
11034   match(Set dst (AndL (AddL src minus_1) src) );
11035   predicate(UseBMI1Instructions);
11036   effect(KILL cr);
11037   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11038 
11039   format %{ "blsrq  $dst, $src" %}
11040 
11041   ins_encode %{
11042     __ blsrq($dst$$Register, $src$$Register);
11043   %}
11044 
11045   ins_pipe(ialu_reg);
11046 %}
11047 
11048 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11049 %{
11050   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
11051   predicate(UseBMI1Instructions);
11052   effect(KILL cr);
11053   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11054 
11055   ins_cost(125);
11056   format %{ "blsrq  $dst, $src" %}
11057 
11058   ins_encode %{
11059     __ blsrq($dst$$Register, $src$$Address);
11060   %}
11061 
11062   ins_pipe(ialu_reg);
11063 %}
11064 
11065 // Or Instructions
11066 // Or Register with Register
11067 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11068 %{
11069   predicate(!UseAPX);
11070   match(Set dst (OrL dst src));
11071   effect(KILL cr);
11072   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11073 
11074   format %{ "orq     $dst, $src\t# long" %}
11075   ins_encode %{
11076     __ orq($dst$$Register, $src$$Register);
11077   %}
11078   ins_pipe(ialu_reg_reg);
11079 %}
11080 
11081 // Or Register with Register using New Data Destination (NDD)
11082 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11083 %{
11084   predicate(UseAPX);
11085   match(Set dst (OrL src1 src2));
11086   effect(KILL cr);
11087   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11088 
11089   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11090   ins_encode %{
11091     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11092 
11093   %}
11094   ins_pipe(ialu_reg_reg);
11095 %}
11096 
11097 // Use any_RegP to match R15 (TLS register) without spilling.
11098 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11099   match(Set dst (OrL dst (CastP2X src)));
11100   effect(KILL cr);
11101   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11102 
11103   format %{ "orq     $dst, $src\t# long" %}
11104   ins_encode %{
11105     __ orq($dst$$Register, $src$$Register);
11106   %}
11107   ins_pipe(ialu_reg_reg);
11108 %}
11109 
11110 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11111   match(Set dst (OrL src1 (CastP2X src2)));
11112   effect(KILL cr);
11113   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11114 
11115   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11116   ins_encode %{
11117     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11118   %}
11119   ins_pipe(ialu_reg_reg);
11120 %}
11121 
11122 // Or Register with Immediate
11123 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11124 %{
11125   predicate(!UseAPX);
11126   match(Set dst (OrL dst src));
11127   effect(KILL cr);
11128   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11129 
11130   format %{ "orq     $dst, $src\t# long" %}
11131   ins_encode %{
11132     __ orq($dst$$Register, $src$$constant);
11133   %}
11134   ins_pipe(ialu_reg);
11135 %}
11136 
11137 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11138 %{
11139   predicate(UseAPX);
11140   match(Set dst (OrL src1 src2));
11141   effect(KILL cr);
11142   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11143 
11144   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11145   ins_encode %{
11146     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11147   %}
11148   ins_pipe(ialu_reg);
11149 %}
11150 
11151 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11152 %{
11153   predicate(UseAPX);
11154   match(Set dst (OrL src1 src2));
11155   effect(KILL cr);
11156   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11157 
11158   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11159   ins_encode %{
11160     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11161   %}
11162   ins_pipe(ialu_reg);
11163 %}
11164 
11165 // Or Memory with Immediate
11166 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11167 %{
11168   predicate(UseAPX);
11169   match(Set dst (OrL (LoadL src1) src2));
11170   effect(KILL cr);
11171   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11172 
11173   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11174   ins_encode %{
11175     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11176   %}
11177   ins_pipe(ialu_reg);
11178 %}
11179 
11180 // Or Register with Memory
11181 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11182 %{
11183   predicate(!UseAPX);
11184   match(Set dst (OrL dst (LoadL src)));
11185   effect(KILL cr);
11186   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11187 
11188   ins_cost(150);
11189   format %{ "orq     $dst, $src\t# long" %}
11190   ins_encode %{
11191     __ orq($dst$$Register, $src$$Address);
11192   %}
11193   ins_pipe(ialu_reg_mem);
11194 %}
11195 
11196 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11197 %{
11198   predicate(UseAPX);
11199   match(Set dst (OrL src1 (LoadL src2)));
11200   effect(KILL cr);
11201   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11202 
11203   ins_cost(150);
11204   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11205   ins_encode %{
11206     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11207   %}
11208   ins_pipe(ialu_reg_mem);
11209 %}
11210 
11211 // Or Memory with Register
11212 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11213 %{
11214   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11215   effect(KILL cr);
11216   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11217 
11218   ins_cost(150);
11219   format %{ "orq     $dst, $src\t# long" %}
11220   ins_encode %{
11221     __ orq($dst$$Address, $src$$Register);
11222   %}
11223   ins_pipe(ialu_mem_reg);
11224 %}
11225 
11226 // Or Memory with Immediate
11227 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11228 %{
11229   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11230   effect(KILL cr);
11231   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11232 
11233   ins_cost(125);
11234   format %{ "orq     $dst, $src\t# long" %}
11235   ins_encode %{
11236     __ orq($dst$$Address, $src$$constant);
11237   %}
11238   ins_pipe(ialu_mem_imm);
11239 %}
11240 
11241 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11242 %{
11243   // con should be a pure 64-bit power of 2 immediate
11244   // because AND/OR works well enough for 8/32-bit values.
11245   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11246 
11247   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11248   effect(KILL cr);
11249 
11250   ins_cost(125);
11251   format %{ "btsq    $dst, log2($con)\t# long" %}
11252   ins_encode %{
11253     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11254   %}
11255   ins_pipe(ialu_mem_imm);
11256 %}
11257 
11258 // Xor Instructions
11259 // Xor Register with Register
11260 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11261 %{
11262   predicate(!UseAPX);
11263   match(Set dst (XorL dst src));
11264   effect(KILL cr);
11265   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11266 
11267   format %{ "xorq    $dst, $src\t# long" %}
11268   ins_encode %{
11269     __ xorq($dst$$Register, $src$$Register);
11270   %}
11271   ins_pipe(ialu_reg_reg);
11272 %}
11273 
11274 // Xor Register with Register using New Data Destination (NDD)
11275 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11276 %{
11277   predicate(UseAPX);
11278   match(Set dst (XorL src1 src2));
11279   effect(KILL cr);
11280   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11281 
11282   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11283   ins_encode %{
11284     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11285   %}
11286   ins_pipe(ialu_reg_reg);
11287 %}
11288 
11289 // Xor Register with Immediate -1
11290 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11291 %{
11292   predicate(!UseAPX);
11293   match(Set dst (XorL dst imm));
11294 
11295   format %{ "notq   $dst" %}
11296   ins_encode %{
11297      __ notq($dst$$Register);
11298   %}
11299   ins_pipe(ialu_reg);
11300 %}
11301 
11302 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11303 %{
11304   predicate(UseAPX);
11305   match(Set dst (XorL src imm));
11306 
11307   format %{ "enotq   $dst, $src" %}
11308   ins_encode %{
11309     __ enotq($dst$$Register, $src$$Register);
11310   %}
11311   ins_pipe(ialu_reg);
11312 %}
11313 
11314 // Xor Register with Immediate
11315 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11316 %{
11317   predicate(!UseAPX);
11318   match(Set dst (XorL dst src));
11319   effect(KILL cr);
11320   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11321 
11322   format %{ "xorq    $dst, $src\t# long" %}
11323   ins_encode %{
11324     __ xorq($dst$$Register, $src$$constant);
11325   %}
11326   ins_pipe(ialu_reg);
11327 %}
11328 
11329 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11330 %{
11331   predicate(UseAPX);
11332   match(Set dst (XorL src1 src2));
11333   effect(KILL cr);
11334   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11335 
11336   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11337   ins_encode %{
11338     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11339   %}
11340   ins_pipe(ialu_reg);
11341 %}
11342 
11343 // Xor Memory with Immediate
11344 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11345 %{
11346   predicate(UseAPX);
11347   match(Set dst (XorL (LoadL src1) src2));
11348   effect(KILL cr);
11349   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11350 
11351   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11352   ins_encode %{
11353     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11354   %}
11355   ins_pipe(ialu_reg);
11356 %}
11357 
11358 // Xor Register with Memory
11359 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11360 %{
11361   predicate(!UseAPX);
11362   match(Set dst (XorL dst (LoadL src)));
11363   effect(KILL cr);
11364   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11365 
11366   ins_cost(150);
11367   format %{ "xorq    $dst, $src\t# long" %}
11368   ins_encode %{
11369     __ xorq($dst$$Register, $src$$Address);
11370   %}
11371   ins_pipe(ialu_reg_mem);
11372 %}
11373 
11374 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11375 %{
11376   predicate(UseAPX);
11377   match(Set dst (XorL src1 (LoadL src2)));
11378   effect(KILL cr);
11379   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11380 
11381   ins_cost(150);
11382   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11383   ins_encode %{
11384     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11385   %}
11386   ins_pipe(ialu_reg_mem);
11387 %}
11388 
11389 instruct xorL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
11390 %{
11391   predicate(UseAPX);
11392   match(Set dst (XorL (LoadL src1) src2));
11393   effect(KILL cr);
11394   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11395 
11396   ins_cost(150);
11397   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11398   ins_encode %{
11399     __ exorq($dst$$Register, $src1$$Address, $src1$$Register, false);
11400   %}
11401   ins_pipe(ialu_reg_mem);
11402 %}
11403 
11404 // Xor Memory with Register
11405 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11406 %{
11407   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11408   effect(KILL cr);
11409   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11410 
11411   ins_cost(150);
11412   format %{ "xorq    $dst, $src\t# long" %}
11413   ins_encode %{
11414     __ xorq($dst$$Address, $src$$Register);
11415   %}
11416   ins_pipe(ialu_mem_reg);
11417 %}
11418 
11419 // Xor Memory with Immediate
11420 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11421 %{
11422   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11423   effect(KILL cr);
11424   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11425 
11426   ins_cost(125);
11427   format %{ "xorq    $dst, $src\t# long" %}
11428   ins_encode %{
11429     __ xorq($dst$$Address, $src$$constant);
11430   %}
11431   ins_pipe(ialu_mem_imm);
11432 %}
11433 
11434 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11435 %{
11436   match(Set dst (CmpLTMask p q));
11437   effect(KILL cr);
11438 
11439   ins_cost(400);
11440   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11441             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11442             "negl    $dst" %}
11443   ins_encode %{
11444     __ cmpl($p$$Register, $q$$Register);
11445     __ setcc(Assembler::less, $dst$$Register);
11446     __ negl($dst$$Register);
11447   %}
11448   ins_pipe(pipe_slow);
11449 %}
11450 
11451 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11452 %{
11453   match(Set dst (CmpLTMask dst zero));
11454   effect(KILL cr);
11455 
11456   ins_cost(100);
11457   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11458   ins_encode %{
11459     __ sarl($dst$$Register, 31);
11460   %}
11461   ins_pipe(ialu_reg);
11462 %}
11463 
11464 /* Better to save a register than avoid a branch */
11465 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11466 %{
11467   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11468   effect(KILL cr);
11469   ins_cost(300);
11470   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11471             "jge     done\n\t"
11472             "addl    $p,$y\n"
11473             "done:   " %}
11474   ins_encode %{
11475     Register Rp = $p$$Register;
11476     Register Rq = $q$$Register;
11477     Register Ry = $y$$Register;
11478     Label done;
11479     __ subl(Rp, Rq);
11480     __ jccb(Assembler::greaterEqual, done);
11481     __ addl(Rp, Ry);
11482     __ bind(done);
11483   %}
11484   ins_pipe(pipe_cmplt);
11485 %}
11486 
11487 /* Better to save a register than avoid a branch */
11488 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11489 %{
11490   match(Set y (AndI (CmpLTMask p q) y));
11491   effect(KILL cr);
11492 
11493   ins_cost(300);
11494 
11495   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11496             "jlt     done\n\t"
11497             "xorl    $y, $y\n"
11498             "done:   " %}
11499   ins_encode %{
11500     Register Rp = $p$$Register;
11501     Register Rq = $q$$Register;
11502     Register Ry = $y$$Register;
11503     Label done;
11504     __ cmpl(Rp, Rq);
11505     __ jccb(Assembler::less, done);
11506     __ xorl(Ry, Ry);
11507     __ bind(done);
11508   %}
11509   ins_pipe(pipe_cmplt);
11510 %}
11511 
11512 
11513 //---------- FP Instructions------------------------------------------------
11514 
11515 // Really expensive, avoid
11516 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11517 %{
11518   match(Set cr (CmpF src1 src2));
11519 
11520   ins_cost(500);
11521   format %{ "ucomiss $src1, $src2\n\t"
11522             "jnp,s   exit\n\t"
11523             "pushfq\t# saw NaN, set CF\n\t"
11524             "andq    [rsp], #0xffffff2b\n\t"
11525             "popfq\n"
11526     "exit:" %}
11527   ins_encode %{
11528     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11529     emit_cmpfp_fixup(masm);
11530   %}
11531   ins_pipe(pipe_slow);
11532 %}
11533 
11534 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11535   match(Set cr (CmpF src1 src2));
11536 
11537   ins_cost(100);
11538   format %{ "ucomiss $src1, $src2" %}
11539   ins_encode %{
11540     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11541   %}
11542   ins_pipe(pipe_slow);
11543 %}
11544 
11545 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11546   match(Set cr (CmpF src1 (LoadF src2)));
11547 
11548   ins_cost(100);
11549   format %{ "ucomiss $src1, $src2" %}
11550   ins_encode %{
11551     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11552   %}
11553   ins_pipe(pipe_slow);
11554 %}
11555 
11556 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11557   match(Set cr (CmpF src con));
11558   ins_cost(100);
11559   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11560   ins_encode %{
11561     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11562   %}
11563   ins_pipe(pipe_slow);
11564 %}
11565 
11566 // Really expensive, avoid
11567 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11568 %{
11569   match(Set cr (CmpD src1 src2));
11570 
11571   ins_cost(500);
11572   format %{ "ucomisd $src1, $src2\n\t"
11573             "jnp,s   exit\n\t"
11574             "pushfq\t# saw NaN, set CF\n\t"
11575             "andq    [rsp], #0xffffff2b\n\t"
11576             "popfq\n"
11577     "exit:" %}
11578   ins_encode %{
11579     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11580     emit_cmpfp_fixup(masm);
11581   %}
11582   ins_pipe(pipe_slow);
11583 %}
11584 
11585 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11586   match(Set cr (CmpD src1 src2));
11587 
11588   ins_cost(100);
11589   format %{ "ucomisd $src1, $src2 test" %}
11590   ins_encode %{
11591     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11592   %}
11593   ins_pipe(pipe_slow);
11594 %}
11595 
11596 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11597   match(Set cr (CmpD src1 (LoadD src2)));
11598 
11599   ins_cost(100);
11600   format %{ "ucomisd $src1, $src2" %}
11601   ins_encode %{
11602     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11603   %}
11604   ins_pipe(pipe_slow);
11605 %}
11606 
11607 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11608   match(Set cr (CmpD src con));
11609   ins_cost(100);
11610   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11611   ins_encode %{
11612     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11613   %}
11614   ins_pipe(pipe_slow);
11615 %}
11616 
11617 // Compare into -1,0,1
11618 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11619 %{
11620   match(Set dst (CmpF3 src1 src2));
11621   effect(KILL cr);
11622 
11623   ins_cost(275);
11624   format %{ "ucomiss $src1, $src2\n\t"
11625             "movl    $dst, #-1\n\t"
11626             "jp,s    done\n\t"
11627             "jb,s    done\n\t"
11628             "setne   $dst\n\t"
11629             "movzbl  $dst, $dst\n"
11630     "done:" %}
11631   ins_encode %{
11632     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11633     emit_cmpfp3(masm, $dst$$Register);
11634   %}
11635   ins_pipe(pipe_slow);
11636 %}
11637 
11638 // Compare into -1,0,1
11639 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11640 %{
11641   match(Set dst (CmpF3 src1 (LoadF src2)));
11642   effect(KILL cr);
11643 
11644   ins_cost(275);
11645   format %{ "ucomiss $src1, $src2\n\t"
11646             "movl    $dst, #-1\n\t"
11647             "jp,s    done\n\t"
11648             "jb,s    done\n\t"
11649             "setne   $dst\n\t"
11650             "movzbl  $dst, $dst\n"
11651     "done:" %}
11652   ins_encode %{
11653     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11654     emit_cmpfp3(masm, $dst$$Register);
11655   %}
11656   ins_pipe(pipe_slow);
11657 %}
11658 
11659 // Compare into -1,0,1
11660 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11661   match(Set dst (CmpF3 src con));
11662   effect(KILL cr);
11663 
11664   ins_cost(275);
11665   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11666             "movl    $dst, #-1\n\t"
11667             "jp,s    done\n\t"
11668             "jb,s    done\n\t"
11669             "setne   $dst\n\t"
11670             "movzbl  $dst, $dst\n"
11671     "done:" %}
11672   ins_encode %{
11673     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11674     emit_cmpfp3(masm, $dst$$Register);
11675   %}
11676   ins_pipe(pipe_slow);
11677 %}
11678 
11679 // Compare into -1,0,1
11680 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11681 %{
11682   match(Set dst (CmpD3 src1 src2));
11683   effect(KILL cr);
11684 
11685   ins_cost(275);
11686   format %{ "ucomisd $src1, $src2\n\t"
11687             "movl    $dst, #-1\n\t"
11688             "jp,s    done\n\t"
11689             "jb,s    done\n\t"
11690             "setne   $dst\n\t"
11691             "movzbl  $dst, $dst\n"
11692     "done:" %}
11693   ins_encode %{
11694     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11695     emit_cmpfp3(masm, $dst$$Register);
11696   %}
11697   ins_pipe(pipe_slow);
11698 %}
11699 
11700 // Compare into -1,0,1
11701 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11702 %{
11703   match(Set dst (CmpD3 src1 (LoadD src2)));
11704   effect(KILL cr);
11705 
11706   ins_cost(275);
11707   format %{ "ucomisd $src1, $src2\n\t"
11708             "movl    $dst, #-1\n\t"
11709             "jp,s    done\n\t"
11710             "jb,s    done\n\t"
11711             "setne   $dst\n\t"
11712             "movzbl  $dst, $dst\n"
11713     "done:" %}
11714   ins_encode %{
11715     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11716     emit_cmpfp3(masm, $dst$$Register);
11717   %}
11718   ins_pipe(pipe_slow);
11719 %}
11720 
11721 // Compare into -1,0,1
11722 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11723   match(Set dst (CmpD3 src con));
11724   effect(KILL cr);
11725 
11726   ins_cost(275);
11727   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11728             "movl    $dst, #-1\n\t"
11729             "jp,s    done\n\t"
11730             "jb,s    done\n\t"
11731             "setne   $dst\n\t"
11732             "movzbl  $dst, $dst\n"
11733     "done:" %}
11734   ins_encode %{
11735     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11736     emit_cmpfp3(masm, $dst$$Register);
11737   %}
11738   ins_pipe(pipe_slow);
11739 %}
11740 
11741 //----------Arithmetic Conversion Instructions---------------------------------
11742 
11743 instruct convF2D_reg_reg(regD dst, regF src)
11744 %{
11745   match(Set dst (ConvF2D src));
11746 
11747   format %{ "cvtss2sd $dst, $src" %}
11748   ins_encode %{
11749     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11750   %}
11751   ins_pipe(pipe_slow); // XXX
11752 %}
11753 
11754 instruct convF2D_reg_mem(regD dst, memory src)
11755 %{
11756   predicate(UseAVX == 0);
11757   match(Set dst (ConvF2D (LoadF src)));
11758 
11759   format %{ "cvtss2sd $dst, $src" %}
11760   ins_encode %{
11761     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11762   %}
11763   ins_pipe(pipe_slow); // XXX
11764 %}
11765 
11766 instruct convD2F_reg_reg(regF dst, regD src)
11767 %{
11768   match(Set dst (ConvD2F src));
11769 
11770   format %{ "cvtsd2ss $dst, $src" %}
11771   ins_encode %{
11772     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11773   %}
11774   ins_pipe(pipe_slow); // XXX
11775 %}
11776 
11777 instruct convD2F_reg_mem(regF dst, memory src)
11778 %{
11779   predicate(UseAVX == 0);
11780   match(Set dst (ConvD2F (LoadD src)));
11781 
11782   format %{ "cvtsd2ss $dst, $src" %}
11783   ins_encode %{
11784     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11785   %}
11786   ins_pipe(pipe_slow); // XXX
11787 %}
11788 
11789 // XXX do mem variants
11790 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11791 %{
11792   match(Set dst (ConvF2I src));
11793   effect(KILL cr);
11794   format %{ "convert_f2i $dst, $src" %}
11795   ins_encode %{
11796     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11797   %}
11798   ins_pipe(pipe_slow);
11799 %}
11800 
11801 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11802 %{
11803   match(Set dst (ConvF2L src));
11804   effect(KILL cr);
11805   format %{ "convert_f2l $dst, $src"%}
11806   ins_encode %{
11807     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11808   %}
11809   ins_pipe(pipe_slow);
11810 %}
11811 
11812 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11813 %{
11814   match(Set dst (ConvD2I src));
11815   effect(KILL cr);
11816   format %{ "convert_d2i $dst, $src"%}
11817   ins_encode %{
11818     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11819   %}
11820   ins_pipe(pipe_slow);
11821 %}
11822 
11823 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11824 %{
11825   match(Set dst (ConvD2L src));
11826   effect(KILL cr);
11827   format %{ "convert_d2l $dst, $src"%}
11828   ins_encode %{
11829     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11830   %}
11831   ins_pipe(pipe_slow);
11832 %}
11833 
11834 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11835 %{
11836   match(Set dst (RoundD src));
11837   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11838   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11839   ins_encode %{
11840     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11841   %}
11842   ins_pipe(pipe_slow);
11843 %}
11844 
11845 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11846 %{
11847   match(Set dst (RoundF src));
11848   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11849   format %{ "round_float $dst,$src" %}
11850   ins_encode %{
11851     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11852   %}
11853   ins_pipe(pipe_slow);
11854 %}
11855 
11856 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11857 %{
11858   predicate(!UseXmmI2F);
11859   match(Set dst (ConvI2F src));
11860 
11861   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11862   ins_encode %{
11863     if (UseAVX > 0) {
11864       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11865     }
11866     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11867   %}
11868   ins_pipe(pipe_slow); // XXX
11869 %}
11870 
11871 instruct convI2F_reg_mem(regF dst, memory src)
11872 %{
11873   predicate(UseAVX == 0);
11874   match(Set dst (ConvI2F (LoadI src)));
11875 
11876   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11877   ins_encode %{
11878     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11879   %}
11880   ins_pipe(pipe_slow); // XXX
11881 %}
11882 
11883 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11884 %{
11885   predicate(!UseXmmI2D);
11886   match(Set dst (ConvI2D src));
11887 
11888   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11889   ins_encode %{
11890     if (UseAVX > 0) {
11891       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11892     }
11893     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11894   %}
11895   ins_pipe(pipe_slow); // XXX
11896 %}
11897 
11898 instruct convI2D_reg_mem(regD dst, memory src)
11899 %{
11900   predicate(UseAVX == 0);
11901   match(Set dst (ConvI2D (LoadI src)));
11902 
11903   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11904   ins_encode %{
11905     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11906   %}
11907   ins_pipe(pipe_slow); // XXX
11908 %}
11909 
11910 instruct convXI2F_reg(regF dst, rRegI src)
11911 %{
11912   predicate(UseXmmI2F);
11913   match(Set dst (ConvI2F src));
11914 
11915   format %{ "movdl $dst, $src\n\t"
11916             "cvtdq2psl $dst, $dst\t# i2f" %}
11917   ins_encode %{
11918     __ movdl($dst$$XMMRegister, $src$$Register);
11919     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11920   %}
11921   ins_pipe(pipe_slow); // XXX
11922 %}
11923 
11924 instruct convXI2D_reg(regD dst, rRegI src)
11925 %{
11926   predicate(UseXmmI2D);
11927   match(Set dst (ConvI2D src));
11928 
11929   format %{ "movdl $dst, $src\n\t"
11930             "cvtdq2pdl $dst, $dst\t# i2d" %}
11931   ins_encode %{
11932     __ movdl($dst$$XMMRegister, $src$$Register);
11933     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11934   %}
11935   ins_pipe(pipe_slow); // XXX
11936 %}
11937 
11938 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11939 %{
11940   match(Set dst (ConvL2F src));
11941 
11942   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11943   ins_encode %{
11944     if (UseAVX > 0) {
11945       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11946     }
11947     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11948   %}
11949   ins_pipe(pipe_slow); // XXX
11950 %}
11951 
11952 instruct convL2F_reg_mem(regF dst, memory src)
11953 %{
11954   predicate(UseAVX == 0);
11955   match(Set dst (ConvL2F (LoadL src)));
11956 
11957   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11958   ins_encode %{
11959     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11960   %}
11961   ins_pipe(pipe_slow); // XXX
11962 %}
11963 
11964 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11965 %{
11966   match(Set dst (ConvL2D src));
11967 
11968   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11969   ins_encode %{
11970     if (UseAVX > 0) {
11971       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11972     }
11973     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11974   %}
11975   ins_pipe(pipe_slow); // XXX
11976 %}
11977 
11978 instruct convL2D_reg_mem(regD dst, memory src)
11979 %{
11980   predicate(UseAVX == 0);
11981   match(Set dst (ConvL2D (LoadL src)));
11982 
11983   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11984   ins_encode %{
11985     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11986   %}
11987   ins_pipe(pipe_slow); // XXX
11988 %}
11989 
11990 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11991 %{
11992   match(Set dst (ConvI2L src));
11993 
11994   ins_cost(125);
11995   format %{ "movslq  $dst, $src\t# i2l" %}
11996   ins_encode %{
11997     __ movslq($dst$$Register, $src$$Register);
11998   %}
11999   ins_pipe(ialu_reg_reg);
12000 %}
12001 
12002 // Zero-extend convert int to long
12003 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
12004 %{
12005   match(Set dst (AndL (ConvI2L src) mask));
12006 
12007   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12008   ins_encode %{
12009     if ($dst$$reg != $src$$reg) {
12010       __ movl($dst$$Register, $src$$Register);
12011     }
12012   %}
12013   ins_pipe(ialu_reg_reg);
12014 %}
12015 
12016 // Zero-extend convert int to long
12017 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
12018 %{
12019   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
12020 
12021   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12022   ins_encode %{
12023     __ movl($dst$$Register, $src$$Address);
12024   %}
12025   ins_pipe(ialu_reg_mem);
12026 %}
12027 
12028 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
12029 %{
12030   match(Set dst (AndL src mask));
12031 
12032   format %{ "movl    $dst, $src\t# zero-extend long" %}
12033   ins_encode %{
12034     __ movl($dst$$Register, $src$$Register);
12035   %}
12036   ins_pipe(ialu_reg_reg);
12037 %}
12038 
12039 instruct convL2I_reg_reg(rRegI dst, rRegL src)
12040 %{
12041   match(Set dst (ConvL2I src));
12042 
12043   format %{ "movl    $dst, $src\t# l2i" %}
12044   ins_encode %{
12045     __ movl($dst$$Register, $src$$Register);
12046   %}
12047   ins_pipe(ialu_reg_reg);
12048 %}
12049 
12050 
12051 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
12052   match(Set dst (MoveF2I src));
12053   effect(DEF dst, USE src);
12054 
12055   ins_cost(125);
12056   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
12057   ins_encode %{
12058     __ movl($dst$$Register, Address(rsp, $src$$disp));
12059   %}
12060   ins_pipe(ialu_reg_mem);
12061 %}
12062 
12063 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12064   match(Set dst (MoveI2F src));
12065   effect(DEF dst, USE src);
12066 
12067   ins_cost(125);
12068   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
12069   ins_encode %{
12070     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
12071   %}
12072   ins_pipe(pipe_slow);
12073 %}
12074 
12075 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12076   match(Set dst (MoveD2L src));
12077   effect(DEF dst, USE src);
12078 
12079   ins_cost(125);
12080   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12081   ins_encode %{
12082     __ movq($dst$$Register, Address(rsp, $src$$disp));
12083   %}
12084   ins_pipe(ialu_reg_mem);
12085 %}
12086 
12087 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12088   predicate(!UseXmmLoadAndClearUpper);
12089   match(Set dst (MoveL2D src));
12090   effect(DEF dst, USE src);
12091 
12092   ins_cost(125);
12093   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12094   ins_encode %{
12095     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12096   %}
12097   ins_pipe(pipe_slow);
12098 %}
12099 
12100 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12101   predicate(UseXmmLoadAndClearUpper);
12102   match(Set dst (MoveL2D src));
12103   effect(DEF dst, USE src);
12104 
12105   ins_cost(125);
12106   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12107   ins_encode %{
12108     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12109   %}
12110   ins_pipe(pipe_slow);
12111 %}
12112 
12113 
12114 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12115   match(Set dst (MoveF2I src));
12116   effect(DEF dst, USE src);
12117 
12118   ins_cost(95); // XXX
12119   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12120   ins_encode %{
12121     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12122   %}
12123   ins_pipe(pipe_slow);
12124 %}
12125 
12126 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12127   match(Set dst (MoveI2F src));
12128   effect(DEF dst, USE src);
12129 
12130   ins_cost(100);
12131   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12132   ins_encode %{
12133     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12134   %}
12135   ins_pipe( ialu_mem_reg );
12136 %}
12137 
12138 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12139   match(Set dst (MoveD2L src));
12140   effect(DEF dst, USE src);
12141 
12142   ins_cost(95); // XXX
12143   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12144   ins_encode %{
12145     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12146   %}
12147   ins_pipe(pipe_slow);
12148 %}
12149 
12150 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12151   match(Set dst (MoveL2D src));
12152   effect(DEF dst, USE src);
12153 
12154   ins_cost(100);
12155   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12156   ins_encode %{
12157     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12158   %}
12159   ins_pipe(ialu_mem_reg);
12160 %}
12161 
12162 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12163   match(Set dst (MoveF2I src));
12164   effect(DEF dst, USE src);
12165   ins_cost(85);
12166   format %{ "movd    $dst,$src\t# MoveF2I" %}
12167   ins_encode %{
12168     __ movdl($dst$$Register, $src$$XMMRegister);
12169   %}
12170   ins_pipe( pipe_slow );
12171 %}
12172 
12173 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12174   match(Set dst (MoveD2L src));
12175   effect(DEF dst, USE src);
12176   ins_cost(85);
12177   format %{ "movd    $dst,$src\t# MoveD2L" %}
12178   ins_encode %{
12179     __ movdq($dst$$Register, $src$$XMMRegister);
12180   %}
12181   ins_pipe( pipe_slow );
12182 %}
12183 
12184 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12185   match(Set dst (MoveI2F src));
12186   effect(DEF dst, USE src);
12187   ins_cost(100);
12188   format %{ "movd    $dst,$src\t# MoveI2F" %}
12189   ins_encode %{
12190     __ movdl($dst$$XMMRegister, $src$$Register);
12191   %}
12192   ins_pipe( pipe_slow );
12193 %}
12194 
12195 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12196   match(Set dst (MoveL2D src));
12197   effect(DEF dst, USE src);
12198   ins_cost(100);
12199   format %{ "movd    $dst,$src\t# MoveL2D" %}
12200   ins_encode %{
12201      __ movdq($dst$$XMMRegister, $src$$Register);
12202   %}
12203   ins_pipe( pipe_slow );
12204 %}
12205 
12206 
12207 // Fast clearing of an array
12208 // Small non-constant lenght ClearArray for non-AVX512 targets.
12209 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
12210                   Universe dummy, rFlagsReg cr)
12211 %{
12212   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
12213   match(Set dummy (ClearArray (Binary cnt base) val));
12214   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
12215 
12216   format %{ $$template
12217     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12218     $$emit$$"jg      LARGE\n\t"
12219     $$emit$$"dec     rcx\n\t"
12220     $$emit$$"js      DONE\t# Zero length\n\t"
12221     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12222     $$emit$$"dec     rcx\n\t"
12223     $$emit$$"jge     LOOP\n\t"
12224     $$emit$$"jmp     DONE\n\t"
12225     $$emit$$"# LARGE:\n\t"
12226     if (UseFastStosb) {
12227        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12228        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12229     } else if (UseXMMForObjInit) {
12230        $$emit$$"movdq   $tmp, $val\n\t"
12231        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
12232        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
12233        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12234        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12235        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12236        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
12237        $$emit$$"add     0x40,rax\n\t"
12238        $$emit$$"# L_zero_64_bytes:\n\t"
12239        $$emit$$"sub     0x8,rcx\n\t"
12240        $$emit$$"jge     L_loop\n\t"
12241        $$emit$$"add     0x4,rcx\n\t"
12242        $$emit$$"jl      L_tail\n\t"
12243        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12244        $$emit$$"add     0x20,rax\n\t"
12245        $$emit$$"sub     0x4,rcx\n\t"
12246        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12247        $$emit$$"add     0x4,rcx\n\t"
12248        $$emit$$"jle     L_end\n\t"
12249        $$emit$$"dec     rcx\n\t"
12250        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12251        $$emit$$"vmovq   xmm0,(rax)\n\t"
12252        $$emit$$"add     0x8,rax\n\t"
12253        $$emit$$"dec     rcx\n\t"
12254        $$emit$$"jge     L_sloop\n\t"
12255        $$emit$$"# L_end:\n\t"
12256     } else {
12257        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12258     }
12259     $$emit$$"# DONE"
12260   %}
12261   ins_encode %{
12262     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12263                  $tmp$$XMMRegister, false, false);
12264   %}
12265   ins_pipe(pipe_slow);
12266 %}
12267 
12268 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
12269                             Universe dummy, rFlagsReg cr)
12270 %{
12271   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
12272   match(Set dummy (ClearArray (Binary cnt base) val));
12273   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
12274 
12275   format %{ $$template
12276     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12277     $$emit$$"jg      LARGE\n\t"
12278     $$emit$$"dec     rcx\n\t"
12279     $$emit$$"js      DONE\t# Zero length\n\t"
12280     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12281     $$emit$$"dec     rcx\n\t"
12282     $$emit$$"jge     LOOP\n\t"
12283     $$emit$$"jmp     DONE\n\t"
12284     $$emit$$"# LARGE:\n\t"
12285     if (UseXMMForObjInit) {
12286        $$emit$$"movdq   $tmp, $val\n\t"
12287        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
12288        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
12289        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12290        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12291        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12292        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
12293        $$emit$$"add     0x40,rax\n\t"
12294        $$emit$$"# L_zero_64_bytes:\n\t"
12295        $$emit$$"sub     0x8,rcx\n\t"
12296        $$emit$$"jge     L_loop\n\t"
12297        $$emit$$"add     0x4,rcx\n\t"
12298        $$emit$$"jl      L_tail\n\t"
12299        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12300        $$emit$$"add     0x20,rax\n\t"
12301        $$emit$$"sub     0x4,rcx\n\t"
12302        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12303        $$emit$$"add     0x4,rcx\n\t"
12304        $$emit$$"jle     L_end\n\t"
12305        $$emit$$"dec     rcx\n\t"
12306        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12307        $$emit$$"vmovq   xmm0,(rax)\n\t"
12308        $$emit$$"add     0x8,rax\n\t"
12309        $$emit$$"dec     rcx\n\t"
12310        $$emit$$"jge     L_sloop\n\t"
12311        $$emit$$"# L_end:\n\t"
12312     } else {
12313        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12314     }
12315     $$emit$$"# DONE"
12316   %}
12317   ins_encode %{
12318     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12319                  $tmp$$XMMRegister, false, true);
12320   %}
12321   ins_pipe(pipe_slow);
12322 %}
12323 
12324 // Small non-constant length ClearArray for AVX512 targets.
12325 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
12326                        Universe dummy, rFlagsReg cr)
12327 %{
12328   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
12329   match(Set dummy (ClearArray (Binary cnt base) val));
12330   ins_cost(125);
12331   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
12332 
12333   format %{ $$template
12334     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12335     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12336     $$emit$$"jg      LARGE\n\t"
12337     $$emit$$"dec     rcx\n\t"
12338     $$emit$$"js      DONE\t# Zero length\n\t"
12339     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12340     $$emit$$"dec     rcx\n\t"
12341     $$emit$$"jge     LOOP\n\t"
12342     $$emit$$"jmp     DONE\n\t"
12343     $$emit$$"# LARGE:\n\t"
12344     if (UseFastStosb) {
12345        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12346        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12347     } else if (UseXMMForObjInit) {
12348        $$emit$$"mov     rdi,rax\n\t"
12349        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12350        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12351        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12352        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12353        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12354        $$emit$$"add     0x40,rax\n\t"
12355        $$emit$$"# L_zero_64_bytes:\n\t"
12356        $$emit$$"sub     0x8,rcx\n\t"
12357        $$emit$$"jge     L_loop\n\t"
12358        $$emit$$"add     0x4,rcx\n\t"
12359        $$emit$$"jl      L_tail\n\t"
12360        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12361        $$emit$$"add     0x20,rax\n\t"
12362        $$emit$$"sub     0x4,rcx\n\t"
12363        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12364        $$emit$$"add     0x4,rcx\n\t"
12365        $$emit$$"jle     L_end\n\t"
12366        $$emit$$"dec     rcx\n\t"
12367        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12368        $$emit$$"vmovq   xmm0,(rax)\n\t"
12369        $$emit$$"add     0x8,rax\n\t"
12370        $$emit$$"dec     rcx\n\t"
12371        $$emit$$"jge     L_sloop\n\t"
12372        $$emit$$"# L_end:\n\t"
12373     } else {
12374        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12375     }
12376     $$emit$$"# DONE"
12377   %}
12378   ins_encode %{
12379     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12380                  $tmp$$XMMRegister, false, false, $ktmp$$KRegister);
12381   %}
12382   ins_pipe(pipe_slow);
12383 %}
12384 
12385 instruct rep_stos_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
12386                                  Universe dummy, rFlagsReg cr)
12387 %{
12388   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
12389   match(Set dummy (ClearArray (Binary cnt base) val));
12390   ins_cost(125);
12391   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
12392 
12393   format %{ $$template
12394     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12395     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12396     $$emit$$"jg      LARGE\n\t"
12397     $$emit$$"dec     rcx\n\t"
12398     $$emit$$"js      DONE\t# Zero length\n\t"
12399     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12400     $$emit$$"dec     rcx\n\t"
12401     $$emit$$"jge     LOOP\n\t"
12402     $$emit$$"jmp     DONE\n\t"
12403     $$emit$$"# LARGE:\n\t"
12404     if (UseFastStosb) {
12405        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12406        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12407     } else if (UseXMMForObjInit) {
12408        $$emit$$"mov     rdi,rax\n\t"
12409        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12410        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12411        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12412        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12413        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12414        $$emit$$"add     0x40,rax\n\t"
12415        $$emit$$"# L_zero_64_bytes:\n\t"
12416        $$emit$$"sub     0x8,rcx\n\t"
12417        $$emit$$"jge     L_loop\n\t"
12418        $$emit$$"add     0x4,rcx\n\t"
12419        $$emit$$"jl      L_tail\n\t"
12420        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12421        $$emit$$"add     0x20,rax\n\t"
12422        $$emit$$"sub     0x4,rcx\n\t"
12423        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12424        $$emit$$"add     0x4,rcx\n\t"
12425        $$emit$$"jle     L_end\n\t"
12426        $$emit$$"dec     rcx\n\t"
12427        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12428        $$emit$$"vmovq   xmm0,(rax)\n\t"
12429        $$emit$$"add     0x8,rax\n\t"
12430        $$emit$$"dec     rcx\n\t"
12431        $$emit$$"jge     L_sloop\n\t"
12432        $$emit$$"# L_end:\n\t"
12433     } else {
12434        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12435     }
12436     $$emit$$"# DONE"
12437   %}
12438   ins_encode %{
12439     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12440                  $tmp$$XMMRegister, false, true, $ktmp$$KRegister);
12441   %}
12442   ins_pipe(pipe_slow);
12443 %}
12444 
12445 // Large non-constant length ClearArray for non-AVX512 targets.
12446 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
12447                         Universe dummy, rFlagsReg cr)
12448 %{
12449   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
12450   match(Set dummy (ClearArray (Binary cnt base) val));
12451   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
12452 
12453   format %{ $$template
12454     if (UseFastStosb) {
12455        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12456        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12457     } else if (UseXMMForObjInit) {
12458        $$emit$$"movdq   $tmp, $val\n\t"
12459        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
12460        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
12461        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12462        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12463        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12464        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
12465        $$emit$$"add     0x40,rax\n\t"
12466        $$emit$$"# L_zero_64_bytes:\n\t"
12467        $$emit$$"sub     0x8,rcx\n\t"
12468        $$emit$$"jge     L_loop\n\t"
12469        $$emit$$"add     0x4,rcx\n\t"
12470        $$emit$$"jl      L_tail\n\t"
12471        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12472        $$emit$$"add     0x20,rax\n\t"
12473        $$emit$$"sub     0x4,rcx\n\t"
12474        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12475        $$emit$$"add     0x4,rcx\n\t"
12476        $$emit$$"jle     L_end\n\t"
12477        $$emit$$"dec     rcx\n\t"
12478        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12479        $$emit$$"vmovq   xmm0,(rax)\n\t"
12480        $$emit$$"add     0x8,rax\n\t"
12481        $$emit$$"dec     rcx\n\t"
12482        $$emit$$"jge     L_sloop\n\t"
12483        $$emit$$"# L_end:\n\t"
12484     } else {
12485        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12486     }
12487   %}
12488   ins_encode %{
12489     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12490                  $tmp$$XMMRegister, true, false);
12491   %}
12492   ins_pipe(pipe_slow);
12493 %}
12494 
12495 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
12496                                   Universe dummy, rFlagsReg cr)
12497 %{
12498   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX <= 2));
12499   match(Set dummy (ClearArray (Binary cnt base) val));
12500   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, USE_KILL val, KILL cr);
12501 
12502   format %{ $$template
12503     if (UseXMMForObjInit) {
12504        $$emit$$"movdq   $tmp, $val\n\t"
12505        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
12506        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
12507        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12508        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12509        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12510        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
12511        $$emit$$"add     0x40,rax\n\t"
12512        $$emit$$"# L_zero_64_bytes:\n\t"
12513        $$emit$$"sub     0x8,rcx\n\t"
12514        $$emit$$"jge     L_loop\n\t"
12515        $$emit$$"add     0x4,rcx\n\t"
12516        $$emit$$"jl      L_tail\n\t"
12517        $$emit$$"vmovdqu $tmp,(rax)\n\t"
12518        $$emit$$"add     0x20,rax\n\t"
12519        $$emit$$"sub     0x4,rcx\n\t"
12520        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12521        $$emit$$"add     0x4,rcx\n\t"
12522        $$emit$$"jle     L_end\n\t"
12523        $$emit$$"dec     rcx\n\t"
12524        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12525        $$emit$$"vmovq   xmm0,(rax)\n\t"
12526        $$emit$$"add     0x8,rax\n\t"
12527        $$emit$$"dec     rcx\n\t"
12528        $$emit$$"jge     L_sloop\n\t"
12529        $$emit$$"# L_end:\n\t"
12530     } else {
12531        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12532     }
12533   %}
12534   ins_encode %{
12535     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12536                  $tmp$$XMMRegister, true, true);
12537   %}
12538   ins_pipe(pipe_slow);
12539 %}
12540 
12541 // Large non-constant length ClearArray for AVX512 targets.
12542 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
12543                              Universe dummy, rFlagsReg cr)
12544 %{
12545   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
12546   match(Set dummy (ClearArray (Binary cnt base) val));
12547   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
12548 
12549   format %{ $$template
12550     if (UseFastStosb) {
12551        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12552        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12553        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12554     } else if (UseXMMForObjInit) {
12555        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12556        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12557        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12558        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12559        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12560        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12561        $$emit$$"add     0x40,rax\n\t"
12562        $$emit$$"# L_zero_64_bytes:\n\t"
12563        $$emit$$"sub     0x8,rcx\n\t"
12564        $$emit$$"jge     L_loop\n\t"
12565        $$emit$$"add     0x4,rcx\n\t"
12566        $$emit$$"jl      L_tail\n\t"
12567        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12568        $$emit$$"add     0x20,rax\n\t"
12569        $$emit$$"sub     0x4,rcx\n\t"
12570        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12571        $$emit$$"add     0x4,rcx\n\t"
12572        $$emit$$"jle     L_end\n\t"
12573        $$emit$$"dec     rcx\n\t"
12574        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12575        $$emit$$"vmovq   xmm0,(rax)\n\t"
12576        $$emit$$"add     0x8,rax\n\t"
12577        $$emit$$"dec     rcx\n\t"
12578        $$emit$$"jge     L_sloop\n\t"
12579        $$emit$$"# L_end:\n\t"
12580     } else {
12581        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12582        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12583     }
12584   %}
12585   ins_encode %{
12586     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12587                  $tmp$$XMMRegister, true, false, $ktmp$$KRegister);
12588   %}
12589   ins_pipe(pipe_slow);
12590 %}
12591 
12592 instruct rep_stos_large_evex_word_copy(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegL val,
12593                                        Universe dummy, rFlagsReg cr)
12594 %{
12595   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only() && (UseAVX > 2));
12596   match(Set dummy (ClearArray (Binary cnt base) val));
12597   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, USE_KILL val, KILL cr);
12598 
12599   format %{ $$template
12600     if (UseFastStosb) {
12601        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12602        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12603        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12604     } else if (UseXMMForObjInit) {
12605        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12606        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12607        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12608        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12609        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12610        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12611        $$emit$$"add     0x40,rax\n\t"
12612        $$emit$$"# L_zero_64_bytes:\n\t"
12613        $$emit$$"sub     0x8,rcx\n\t"
12614        $$emit$$"jge     L_loop\n\t"
12615        $$emit$$"add     0x4,rcx\n\t"
12616        $$emit$$"jl      L_tail\n\t"
12617        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12618        $$emit$$"add     0x20,rax\n\t"
12619        $$emit$$"sub     0x4,rcx\n\t"
12620        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12621        $$emit$$"add     0x4,rcx\n\t"
12622        $$emit$$"jle     L_end\n\t"
12623        $$emit$$"dec     rcx\n\t"
12624        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12625        $$emit$$"vmovq   xmm0,(rax)\n\t"
12626        $$emit$$"add     0x8,rax\n\t"
12627        $$emit$$"dec     rcx\n\t"
12628        $$emit$$"jge     L_sloop\n\t"
12629        $$emit$$"# L_end:\n\t"
12630     } else {
12631        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12632        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12633     }
12634   %}
12635   ins_encode %{
12636     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
12637                  $tmp$$XMMRegister, true, true, $ktmp$$KRegister);
12638   %}
12639   ins_pipe(pipe_slow);
12640 %}
12641 
12642 // Small constant length ClearArray for AVX512 targets.
12643 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rax_RegL val, kReg ktmp, Universe dummy, rFlagsReg cr)
12644 %{
12645   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only() &&
12646             ((MaxVectorSize >= 32) && VM_Version::supports_avx512vl()));
12647   match(Set dummy (ClearArray (Binary cnt base) val));
12648   ins_cost(100);
12649   effect(TEMP tmp, USE_KILL val, TEMP ktmp, KILL cr);
12650   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12651   ins_encode %{
12652     __ clear_mem($base$$Register, $cnt$$constant, $val$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12653   %}
12654   ins_pipe(pipe_slow);
12655 %}
12656 
12657 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12658                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12659 %{
12660   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12661   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12662   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12663 
12664   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12665   ins_encode %{
12666     __ string_compare($str1$$Register, $str2$$Register,
12667                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12668                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12669   %}
12670   ins_pipe( pipe_slow );
12671 %}
12672 
12673 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12674                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12675 %{
12676   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12677   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12678   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12679 
12680   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12681   ins_encode %{
12682     __ string_compare($str1$$Register, $str2$$Register,
12683                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12684                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12685   %}
12686   ins_pipe( pipe_slow );
12687 %}
12688 
12689 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12690                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12691 %{
12692   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12693   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12694   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12695 
12696   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12697   ins_encode %{
12698     __ string_compare($str1$$Register, $str2$$Register,
12699                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12700                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12701   %}
12702   ins_pipe( pipe_slow );
12703 %}
12704 
12705 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12706                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12707 %{
12708   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12709   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12710   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12711 
12712   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12713   ins_encode %{
12714     __ string_compare($str1$$Register, $str2$$Register,
12715                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12716                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12717   %}
12718   ins_pipe( pipe_slow );
12719 %}
12720 
12721 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12722                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12723 %{
12724   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12725   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12726   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12727 
12728   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12729   ins_encode %{
12730     __ string_compare($str1$$Register, $str2$$Register,
12731                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12732                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12733   %}
12734   ins_pipe( pipe_slow );
12735 %}
12736 
12737 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12738                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12739 %{
12740   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12741   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12742   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12743 
12744   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12745   ins_encode %{
12746     __ string_compare($str1$$Register, $str2$$Register,
12747                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12748                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12749   %}
12750   ins_pipe( pipe_slow );
12751 %}
12752 
12753 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12754                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12755 %{
12756   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12757   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12758   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12759 
12760   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12761   ins_encode %{
12762     __ string_compare($str2$$Register, $str1$$Register,
12763                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12764                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12765   %}
12766   ins_pipe( pipe_slow );
12767 %}
12768 
12769 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12770                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12771 %{
12772   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12773   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12774   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12775 
12776   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12777   ins_encode %{
12778     __ string_compare($str2$$Register, $str1$$Register,
12779                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12780                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12781   %}
12782   ins_pipe( pipe_slow );
12783 %}
12784 
12785 // fast search of substring with known size.
12786 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12787                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12788 %{
12789   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12790   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12791   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12792 
12793   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12794   ins_encode %{
12795     int icnt2 = (int)$int_cnt2$$constant;
12796     if (icnt2 >= 16) {
12797       // IndexOf for constant substrings with size >= 16 elements
12798       // which don't need to be loaded through stack.
12799       __ string_indexofC8($str1$$Register, $str2$$Register,
12800                           $cnt1$$Register, $cnt2$$Register,
12801                           icnt2, $result$$Register,
12802                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12803     } else {
12804       // Small strings are loaded through stack if they cross page boundary.
12805       __ string_indexof($str1$$Register, $str2$$Register,
12806                         $cnt1$$Register, $cnt2$$Register,
12807                         icnt2, $result$$Register,
12808                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12809     }
12810   %}
12811   ins_pipe( pipe_slow );
12812 %}
12813 
12814 // fast search of substring with known size.
12815 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12816                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12817 %{
12818   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12819   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12820   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12821 
12822   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12823   ins_encode %{
12824     int icnt2 = (int)$int_cnt2$$constant;
12825     if (icnt2 >= 8) {
12826       // IndexOf for constant substrings with size >= 8 elements
12827       // which don't need to be loaded through stack.
12828       __ string_indexofC8($str1$$Register, $str2$$Register,
12829                           $cnt1$$Register, $cnt2$$Register,
12830                           icnt2, $result$$Register,
12831                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12832     } else {
12833       // Small strings are loaded through stack if they cross page boundary.
12834       __ string_indexof($str1$$Register, $str2$$Register,
12835                         $cnt1$$Register, $cnt2$$Register,
12836                         icnt2, $result$$Register,
12837                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12838     }
12839   %}
12840   ins_pipe( pipe_slow );
12841 %}
12842 
12843 // fast search of substring with known size.
12844 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12845                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12846 %{
12847   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12848   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12849   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12850 
12851   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12852   ins_encode %{
12853     int icnt2 = (int)$int_cnt2$$constant;
12854     if (icnt2 >= 8) {
12855       // IndexOf for constant substrings with size >= 8 elements
12856       // which don't need to be loaded through stack.
12857       __ string_indexofC8($str1$$Register, $str2$$Register,
12858                           $cnt1$$Register, $cnt2$$Register,
12859                           icnt2, $result$$Register,
12860                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12861     } else {
12862       // Small strings are loaded through stack if they cross page boundary.
12863       __ string_indexof($str1$$Register, $str2$$Register,
12864                         $cnt1$$Register, $cnt2$$Register,
12865                         icnt2, $result$$Register,
12866                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12867     }
12868   %}
12869   ins_pipe( pipe_slow );
12870 %}
12871 
12872 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12873                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12874 %{
12875   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12876   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12877   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12878 
12879   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12880   ins_encode %{
12881     __ string_indexof($str1$$Register, $str2$$Register,
12882                       $cnt1$$Register, $cnt2$$Register,
12883                       (-1), $result$$Register,
12884                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12885   %}
12886   ins_pipe( pipe_slow );
12887 %}
12888 
12889 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12890                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12891 %{
12892   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12893   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12894   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12895 
12896   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12897   ins_encode %{
12898     __ string_indexof($str1$$Register, $str2$$Register,
12899                       $cnt1$$Register, $cnt2$$Register,
12900                       (-1), $result$$Register,
12901                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12902   %}
12903   ins_pipe( pipe_slow );
12904 %}
12905 
12906 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12907                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12908 %{
12909   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12910   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12911   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12912 
12913   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12914   ins_encode %{
12915     __ string_indexof($str1$$Register, $str2$$Register,
12916                       $cnt1$$Register, $cnt2$$Register,
12917                       (-1), $result$$Register,
12918                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12919   %}
12920   ins_pipe( pipe_slow );
12921 %}
12922 
12923 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12924                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12925 %{
12926   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12927   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12928   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12929   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12930   ins_encode %{
12931     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12932                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12933   %}
12934   ins_pipe( pipe_slow );
12935 %}
12936 
12937 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12938                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12939 %{
12940   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12941   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12942   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12943   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12944   ins_encode %{
12945     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12946                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12947   %}
12948   ins_pipe( pipe_slow );
12949 %}
12950 
12951 // fast string equals
12952 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12953                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12954 %{
12955   predicate(!VM_Version::supports_avx512vlbw());
12956   match(Set result (StrEquals (Binary str1 str2) cnt));
12957   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12958 
12959   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12960   ins_encode %{
12961     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12962                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12963                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12964   %}
12965   ins_pipe( pipe_slow );
12966 %}
12967 
12968 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12969                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12970 %{
12971   predicate(VM_Version::supports_avx512vlbw());
12972   match(Set result (StrEquals (Binary str1 str2) cnt));
12973   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12974 
12975   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12976   ins_encode %{
12977     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12978                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12979                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12980   %}
12981   ins_pipe( pipe_slow );
12982 %}
12983 
12984 // fast array equals
12985 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12986                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12987 %{
12988   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12989   match(Set result (AryEq ary1 ary2));
12990   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12991 
12992   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12993   ins_encode %{
12994     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12995                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12996                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12997   %}
12998   ins_pipe( pipe_slow );
12999 %}
13000 
13001 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
13002                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
13003 %{
13004   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
13005   match(Set result (AryEq ary1 ary2));
13006   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
13007 
13008   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
13009   ins_encode %{
13010     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
13011                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
13012                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
13013   %}
13014   ins_pipe( pipe_slow );
13015 %}
13016 
13017 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
13018                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
13019 %{
13020   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
13021   match(Set result (AryEq ary1 ary2));
13022   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
13023 
13024   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
13025   ins_encode %{
13026     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
13027                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
13028                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
13029   %}
13030   ins_pipe( pipe_slow );
13031 %}
13032 
13033 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
13034                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
13035 %{
13036   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
13037   match(Set result (AryEq ary1 ary2));
13038   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
13039 
13040   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
13041   ins_encode %{
13042     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
13043                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
13044                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
13045   %}
13046   ins_pipe( pipe_slow );
13047 %}
13048 
13049 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
13050                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
13051                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
13052                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
13053                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
13054 %{
13055   predicate(UseAVX >= 2);
13056   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
13057   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
13058          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
13059          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
13060          USE basic_type, KILL cr);
13061 
13062   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
13063   ins_encode %{
13064     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
13065                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
13066                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
13067                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
13068                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
13069                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
13070                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
13071   %}
13072   ins_pipe( pipe_slow );
13073 %}
13074 
13075 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
13076                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
13077 %{
13078   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
13079   match(Set result (CountPositives ary1 len));
13080   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
13081 
13082   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
13083   ins_encode %{
13084     __ count_positives($ary1$$Register, $len$$Register,
13085                        $result$$Register, $tmp3$$Register,
13086                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
13087   %}
13088   ins_pipe( pipe_slow );
13089 %}
13090 
13091 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
13092                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
13093 %{
13094   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
13095   match(Set result (CountPositives ary1 len));
13096   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
13097 
13098   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
13099   ins_encode %{
13100     __ count_positives($ary1$$Register, $len$$Register,
13101                        $result$$Register, $tmp3$$Register,
13102                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
13103   %}
13104   ins_pipe( pipe_slow );
13105 %}
13106 
13107 // fast char[] to byte[] compression
13108 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
13109                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
13110   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
13111   match(Set result (StrCompressedCopy src (Binary dst len)));
13112   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
13113          USE_KILL len, KILL tmp5, KILL cr);
13114 
13115   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
13116   ins_encode %{
13117     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
13118                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13119                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
13120                            knoreg, knoreg);
13121   %}
13122   ins_pipe( pipe_slow );
13123 %}
13124 
13125 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
13126                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
13127   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
13128   match(Set result (StrCompressedCopy src (Binary dst len)));
13129   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
13130          USE_KILL len, KILL tmp5, KILL cr);
13131 
13132   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
13133   ins_encode %{
13134     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
13135                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13136                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
13137                            $ktmp1$$KRegister, $ktmp2$$KRegister);
13138   %}
13139   ins_pipe( pipe_slow );
13140 %}
13141 // fast byte[] to char[] inflation
13142 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
13143                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
13144   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
13145   match(Set dummy (StrInflatedCopy src (Binary dst len)));
13146   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
13147 
13148   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
13149   ins_encode %{
13150     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
13151                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
13152   %}
13153   ins_pipe( pipe_slow );
13154 %}
13155 
13156 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
13157                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
13158   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
13159   match(Set dummy (StrInflatedCopy src (Binary dst len)));
13160   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
13161 
13162   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
13163   ins_encode %{
13164     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
13165                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
13166   %}
13167   ins_pipe( pipe_slow );
13168 %}
13169 
13170 // encode char[] to byte[] in ISO_8859_1
13171 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
13172                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
13173                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
13174   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
13175   match(Set result (EncodeISOArray src (Binary dst len)));
13176   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
13177 
13178   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
13179   ins_encode %{
13180     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
13181                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13182                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
13183   %}
13184   ins_pipe( pipe_slow );
13185 %}
13186 
13187 // encode char[] to byte[] in ASCII
13188 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
13189                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
13190                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
13191   predicate(((EncodeISOArrayNode*)n)->is_ascii());
13192   match(Set result (EncodeISOArray src (Binary dst len)));
13193   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
13194 
13195   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
13196   ins_encode %{
13197     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
13198                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13199                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
13200   %}
13201   ins_pipe( pipe_slow );
13202 %}
13203 
13204 //----------Overflow Math Instructions-----------------------------------------
13205 
13206 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13207 %{
13208   match(Set cr (OverflowAddI op1 op2));
13209   effect(DEF cr, USE_KILL op1, USE op2);
13210 
13211   format %{ "addl    $op1, $op2\t# overflow check int" %}
13212 
13213   ins_encode %{
13214     __ addl($op1$$Register, $op2$$Register);
13215   %}
13216   ins_pipe(ialu_reg_reg);
13217 %}
13218 
13219 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
13220 %{
13221   match(Set cr (OverflowAddI op1 op2));
13222   effect(DEF cr, USE_KILL op1, USE op2);
13223 
13224   format %{ "addl    $op1, $op2\t# overflow check int" %}
13225 
13226   ins_encode %{
13227     __ addl($op1$$Register, $op2$$constant);
13228   %}
13229   ins_pipe(ialu_reg_reg);
13230 %}
13231 
13232 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13233 %{
13234   match(Set cr (OverflowAddL op1 op2));
13235   effect(DEF cr, USE_KILL op1, USE op2);
13236 
13237   format %{ "addq    $op1, $op2\t# overflow check long" %}
13238   ins_encode %{
13239     __ addq($op1$$Register, $op2$$Register);
13240   %}
13241   ins_pipe(ialu_reg_reg);
13242 %}
13243 
13244 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
13245 %{
13246   match(Set cr (OverflowAddL op1 op2));
13247   effect(DEF cr, USE_KILL op1, USE op2);
13248 
13249   format %{ "addq    $op1, $op2\t# overflow check long" %}
13250   ins_encode %{
13251     __ addq($op1$$Register, $op2$$constant);
13252   %}
13253   ins_pipe(ialu_reg_reg);
13254 %}
13255 
13256 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13257 %{
13258   match(Set cr (OverflowSubI op1 op2));
13259 
13260   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13261   ins_encode %{
13262     __ cmpl($op1$$Register, $op2$$Register);
13263   %}
13264   ins_pipe(ialu_reg_reg);
13265 %}
13266 
13267 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13268 %{
13269   match(Set cr (OverflowSubI op1 op2));
13270 
13271   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13272   ins_encode %{
13273     __ cmpl($op1$$Register, $op2$$constant);
13274   %}
13275   ins_pipe(ialu_reg_reg);
13276 %}
13277 
13278 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13279 %{
13280   match(Set cr (OverflowSubL op1 op2));
13281 
13282   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13283   ins_encode %{
13284     __ cmpq($op1$$Register, $op2$$Register);
13285   %}
13286   ins_pipe(ialu_reg_reg);
13287 %}
13288 
13289 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13290 %{
13291   match(Set cr (OverflowSubL op1 op2));
13292 
13293   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13294   ins_encode %{
13295     __ cmpq($op1$$Register, $op2$$constant);
13296   %}
13297   ins_pipe(ialu_reg_reg);
13298 %}
13299 
13300 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13301 %{
13302   match(Set cr (OverflowSubI zero op2));
13303   effect(DEF cr, USE_KILL op2);
13304 
13305   format %{ "negl    $op2\t# overflow check int" %}
13306   ins_encode %{
13307     __ negl($op2$$Register);
13308   %}
13309   ins_pipe(ialu_reg_reg);
13310 %}
13311 
13312 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13313 %{
13314   match(Set cr (OverflowSubL zero op2));
13315   effect(DEF cr, USE_KILL op2);
13316 
13317   format %{ "negq    $op2\t# overflow check long" %}
13318   ins_encode %{
13319     __ negq($op2$$Register);
13320   %}
13321   ins_pipe(ialu_reg_reg);
13322 %}
13323 
13324 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13325 %{
13326   match(Set cr (OverflowMulI op1 op2));
13327   effect(DEF cr, USE_KILL op1, USE op2);
13328 
13329   format %{ "imull    $op1, $op2\t# overflow check int" %}
13330   ins_encode %{
13331     __ imull($op1$$Register, $op2$$Register);
13332   %}
13333   ins_pipe(ialu_reg_reg_alu0);
13334 %}
13335 
13336 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13337 %{
13338   match(Set cr (OverflowMulI op1 op2));
13339   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13340 
13341   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13342   ins_encode %{
13343     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13344   %}
13345   ins_pipe(ialu_reg_reg_alu0);
13346 %}
13347 
13348 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13349 %{
13350   match(Set cr (OverflowMulL op1 op2));
13351   effect(DEF cr, USE_KILL op1, USE op2);
13352 
13353   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13354   ins_encode %{
13355     __ imulq($op1$$Register, $op2$$Register);
13356   %}
13357   ins_pipe(ialu_reg_reg_alu0);
13358 %}
13359 
13360 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13361 %{
13362   match(Set cr (OverflowMulL op1 op2));
13363   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13364 
13365   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13366   ins_encode %{
13367     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13368   %}
13369   ins_pipe(ialu_reg_reg_alu0);
13370 %}
13371 
13372 
13373 //----------Control Flow Instructions------------------------------------------
13374 // Signed compare Instructions
13375 
13376 // XXX more variants!!
13377 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13378 %{
13379   match(Set cr (CmpI op1 op2));
13380   effect(DEF cr, USE op1, USE op2);
13381 
13382   format %{ "cmpl    $op1, $op2" %}
13383   ins_encode %{
13384     __ cmpl($op1$$Register, $op2$$Register);
13385   %}
13386   ins_pipe(ialu_cr_reg_reg);
13387 %}
13388 
13389 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13390 %{
13391   match(Set cr (CmpI op1 op2));
13392 
13393   format %{ "cmpl    $op1, $op2" %}
13394   ins_encode %{
13395     __ cmpl($op1$$Register, $op2$$constant);
13396   %}
13397   ins_pipe(ialu_cr_reg_imm);
13398 %}
13399 
13400 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13401 %{
13402   match(Set cr (CmpI op1 (LoadI op2)));
13403 
13404   ins_cost(500); // XXX
13405   format %{ "cmpl    $op1, $op2" %}
13406   ins_encode %{
13407     __ cmpl($op1$$Register, $op2$$Address);
13408   %}
13409   ins_pipe(ialu_cr_reg_mem);
13410 %}
13411 
13412 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13413 %{
13414   match(Set cr (CmpI src zero));
13415 
13416   format %{ "testl   $src, $src" %}
13417   ins_encode %{
13418     __ testl($src$$Register, $src$$Register);
13419   %}
13420   ins_pipe(ialu_cr_reg_imm);
13421 %}
13422 
13423 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13424 %{
13425   match(Set cr (CmpI (AndI src con) zero));
13426 
13427   format %{ "testl   $src, $con" %}
13428   ins_encode %{
13429     __ testl($src$$Register, $con$$constant);
13430   %}
13431   ins_pipe(ialu_cr_reg_imm);
13432 %}
13433 
13434 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13435 %{
13436   match(Set cr (CmpI (AndI src1 src2) zero));
13437 
13438   format %{ "testl   $src1, $src2" %}
13439   ins_encode %{
13440     __ testl($src1$$Register, $src2$$Register);
13441   %}
13442   ins_pipe(ialu_cr_reg_imm);
13443 %}
13444 
13445 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13446 %{
13447   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13448 
13449   format %{ "testl   $src, $mem" %}
13450   ins_encode %{
13451     __ testl($src$$Register, $mem$$Address);
13452   %}
13453   ins_pipe(ialu_cr_reg_mem);
13454 %}
13455 
13456 // Unsigned compare Instructions; really, same as signed except they
13457 // produce an rFlagsRegU instead of rFlagsReg.
13458 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13459 %{
13460   match(Set cr (CmpU op1 op2));
13461 
13462   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13463   ins_encode %{
13464     __ cmpl($op1$$Register, $op2$$Register);
13465   %}
13466   ins_pipe(ialu_cr_reg_reg);
13467 %}
13468 
13469 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13470 %{
13471   match(Set cr (CmpU op1 op2));
13472 
13473   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13474   ins_encode %{
13475     __ cmpl($op1$$Register, $op2$$constant);
13476   %}
13477   ins_pipe(ialu_cr_reg_imm);
13478 %}
13479 
13480 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13481 %{
13482   match(Set cr (CmpU op1 (LoadI op2)));
13483 
13484   ins_cost(500); // XXX
13485   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13486   ins_encode %{
13487     __ cmpl($op1$$Register, $op2$$Address);
13488   %}
13489   ins_pipe(ialu_cr_reg_mem);
13490 %}
13491 
13492 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13493 %{
13494   match(Set cr (CmpU src zero));
13495 
13496   format %{ "testl   $src, $src\t# unsigned" %}
13497   ins_encode %{
13498     __ testl($src$$Register, $src$$Register);
13499   %}
13500   ins_pipe(ialu_cr_reg_imm);
13501 %}
13502 
13503 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13504 %{
13505   match(Set cr (CmpP op1 op2));
13506 
13507   format %{ "cmpq    $op1, $op2\t# ptr" %}
13508   ins_encode %{
13509     __ cmpq($op1$$Register, $op2$$Register);
13510   %}
13511   ins_pipe(ialu_cr_reg_reg);
13512 %}
13513 
13514 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13515 %{
13516   match(Set cr (CmpP op1 (LoadP op2)));
13517   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13518 
13519   ins_cost(500); // XXX
13520   format %{ "cmpq    $op1, $op2\t# ptr" %}
13521   ins_encode %{
13522     __ cmpq($op1$$Register, $op2$$Address);
13523   %}
13524   ins_pipe(ialu_cr_reg_mem);
13525 %}
13526 
13527 // XXX this is generalized by compP_rReg_mem???
13528 // Compare raw pointer (used in out-of-heap check).
13529 // Only works because non-oop pointers must be raw pointers
13530 // and raw pointers have no anti-dependencies.
13531 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13532 %{
13533   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13534             n->in(2)->as_Load()->barrier_data() == 0);
13535   match(Set cr (CmpP op1 (LoadP op2)));
13536 
13537   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13538   ins_encode %{
13539     __ cmpq($op1$$Register, $op2$$Address);
13540   %}
13541   ins_pipe(ialu_cr_reg_mem);
13542 %}
13543 
13544 // This will generate a signed flags result. This should be OK since
13545 // any compare to a zero should be eq/neq.
13546 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13547 %{
13548   match(Set cr (CmpP src zero));
13549 
13550   format %{ "testq   $src, $src\t# ptr" %}
13551   ins_encode %{
13552     __ testq($src$$Register, $src$$Register);
13553   %}
13554   ins_pipe(ialu_cr_reg_imm);
13555 %}
13556 
13557 // This will generate a signed flags result. This should be OK since
13558 // any compare to a zero should be eq/neq.
13559 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13560 %{
13561   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13562             n->in(1)->as_Load()->barrier_data() == 0);
13563   match(Set cr (CmpP (LoadP op) zero));
13564 
13565   ins_cost(500); // XXX
13566   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13567   ins_encode %{
13568     __ testq($op$$Address, 0xFFFFFFFF);
13569   %}
13570   ins_pipe(ialu_cr_reg_imm);
13571 %}
13572 
13573 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13574 %{
13575   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13576             n->in(1)->as_Load()->barrier_data() == 0);
13577   match(Set cr (CmpP (LoadP mem) zero));
13578 
13579   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13580   ins_encode %{
13581     __ cmpq(r12, $mem$$Address);
13582   %}
13583   ins_pipe(ialu_cr_reg_mem);
13584 %}
13585 
13586 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13587 %{
13588   match(Set cr (CmpN op1 op2));
13589 
13590   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13591   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13592   ins_pipe(ialu_cr_reg_reg);
13593 %}
13594 
13595 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13596 %{
13597   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13598   match(Set cr (CmpN src (LoadN mem)));
13599 
13600   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13601   ins_encode %{
13602     __ cmpl($src$$Register, $mem$$Address);
13603   %}
13604   ins_pipe(ialu_cr_reg_mem);
13605 %}
13606 
13607 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13608   match(Set cr (CmpN op1 op2));
13609 
13610   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13611   ins_encode %{
13612     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13613   %}
13614   ins_pipe(ialu_cr_reg_imm);
13615 %}
13616 
13617 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13618 %{
13619   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13620   match(Set cr (CmpN src (LoadN mem)));
13621 
13622   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13623   ins_encode %{
13624     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13625   %}
13626   ins_pipe(ialu_cr_reg_mem);
13627 %}
13628 
13629 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13630   match(Set cr (CmpN op1 op2));
13631 
13632   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13633   ins_encode %{
13634     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13635   %}
13636   ins_pipe(ialu_cr_reg_imm);
13637 %}
13638 
13639 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13640 %{
13641   predicate(!UseCompactObjectHeaders);
13642   match(Set cr (CmpN src (LoadNKlass mem)));
13643 
13644   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13645   ins_encode %{
13646     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13647   %}
13648   ins_pipe(ialu_cr_reg_mem);
13649 %}
13650 
13651 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13652   match(Set cr (CmpN src zero));
13653 
13654   format %{ "testl   $src, $src\t# compressed ptr" %}
13655   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13656   ins_pipe(ialu_cr_reg_imm);
13657 %}
13658 
13659 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13660 %{
13661   predicate(CompressedOops::base() != nullptr &&
13662             n->in(1)->as_Load()->barrier_data() == 0);
13663   match(Set cr (CmpN (LoadN mem) zero));
13664 
13665   ins_cost(500); // XXX
13666   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13667   ins_encode %{
13668     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13669   %}
13670   ins_pipe(ialu_cr_reg_mem);
13671 %}
13672 
13673 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13674 %{
13675   predicate(CompressedOops::base() == nullptr &&
13676             n->in(1)->as_Load()->barrier_data() == 0);
13677   match(Set cr (CmpN (LoadN mem) zero));
13678 
13679   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13680   ins_encode %{
13681     __ cmpl(r12, $mem$$Address);
13682   %}
13683   ins_pipe(ialu_cr_reg_mem);
13684 %}
13685 
13686 // Yanked all unsigned pointer compare operations.
13687 // Pointer compares are done with CmpP which is already unsigned.
13688 
13689 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13690 %{
13691   match(Set cr (CmpL op1 op2));
13692 
13693   format %{ "cmpq    $op1, $op2" %}
13694   ins_encode %{
13695     __ cmpq($op1$$Register, $op2$$Register);
13696   %}
13697   ins_pipe(ialu_cr_reg_reg);
13698 %}
13699 
13700 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13701 %{
13702   match(Set cr (CmpL op1 op2));
13703 
13704   format %{ "cmpq    $op1, $op2" %}
13705   ins_encode %{
13706     __ cmpq($op1$$Register, $op2$$constant);
13707   %}
13708   ins_pipe(ialu_cr_reg_imm);
13709 %}
13710 
13711 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13712 %{
13713   match(Set cr (CmpL op1 (LoadL op2)));
13714 
13715   format %{ "cmpq    $op1, $op2" %}
13716   ins_encode %{
13717     __ cmpq($op1$$Register, $op2$$Address);
13718   %}
13719   ins_pipe(ialu_cr_reg_mem);
13720 %}
13721 
13722 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13723 %{
13724   match(Set cr (CmpL src zero));
13725 
13726   format %{ "testq   $src, $src" %}
13727   ins_encode %{
13728     __ testq($src$$Register, $src$$Register);
13729   %}
13730   ins_pipe(ialu_cr_reg_imm);
13731 %}
13732 
13733 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13734 %{
13735   match(Set cr (CmpL (AndL src con) zero));
13736 
13737   format %{ "testq   $src, $con\t# long" %}
13738   ins_encode %{
13739     __ testq($src$$Register, $con$$constant);
13740   %}
13741   ins_pipe(ialu_cr_reg_imm);
13742 %}
13743 
13744 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13745 %{
13746   match(Set cr (CmpL (AndL src1 src2) zero));
13747 
13748   format %{ "testq   $src1, $src2\t# long" %}
13749   ins_encode %{
13750     __ testq($src1$$Register, $src2$$Register);
13751   %}
13752   ins_pipe(ialu_cr_reg_imm);
13753 %}
13754 
13755 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13756 %{
13757   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13758 
13759   format %{ "testq   $src, $mem" %}
13760   ins_encode %{
13761     __ testq($src$$Register, $mem$$Address);
13762   %}
13763   ins_pipe(ialu_cr_reg_mem);
13764 %}
13765 
13766 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13767 %{
13768   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13769 
13770   format %{ "testq   $src, $mem" %}
13771   ins_encode %{
13772     __ testq($src$$Register, $mem$$Address);
13773   %}
13774   ins_pipe(ialu_cr_reg_mem);
13775 %}
13776 
13777 // Manifest a CmpU result in an integer register.  Very painful.
13778 // This is the test to avoid.
13779 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13780 %{
13781   match(Set dst (CmpU3 src1 src2));
13782   effect(KILL flags);
13783 
13784   ins_cost(275); // XXX
13785   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13786             "movl    $dst, -1\n\t"
13787             "jb,u    done\n\t"
13788             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13789     "done:" %}
13790   ins_encode %{
13791     Label done;
13792     __ cmpl($src1$$Register, $src2$$Register);
13793     __ movl($dst$$Register, -1);
13794     __ jccb(Assembler::below, done);
13795     __ setcc(Assembler::notZero, $dst$$Register);
13796     __ bind(done);
13797   %}
13798   ins_pipe(pipe_slow);
13799 %}
13800 
13801 // Manifest a CmpL result in an integer register.  Very painful.
13802 // This is the test to avoid.
13803 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13804 %{
13805   match(Set dst (CmpL3 src1 src2));
13806   effect(KILL flags);
13807 
13808   ins_cost(275); // XXX
13809   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13810             "movl    $dst, -1\n\t"
13811             "jl,s    done\n\t"
13812             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13813     "done:" %}
13814   ins_encode %{
13815     Label done;
13816     __ cmpq($src1$$Register, $src2$$Register);
13817     __ movl($dst$$Register, -1);
13818     __ jccb(Assembler::less, done);
13819     __ setcc(Assembler::notZero, $dst$$Register);
13820     __ bind(done);
13821   %}
13822   ins_pipe(pipe_slow);
13823 %}
13824 
13825 // Manifest a CmpUL result in an integer register.  Very painful.
13826 // This is the test to avoid.
13827 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13828 %{
13829   match(Set dst (CmpUL3 src1 src2));
13830   effect(KILL flags);
13831 
13832   ins_cost(275); // XXX
13833   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13834             "movl    $dst, -1\n\t"
13835             "jb,u    done\n\t"
13836             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13837     "done:" %}
13838   ins_encode %{
13839     Label done;
13840     __ cmpq($src1$$Register, $src2$$Register);
13841     __ movl($dst$$Register, -1);
13842     __ jccb(Assembler::below, done);
13843     __ setcc(Assembler::notZero, $dst$$Register);
13844     __ bind(done);
13845   %}
13846   ins_pipe(pipe_slow);
13847 %}
13848 
13849 // Unsigned long compare Instructions; really, same as signed long except they
13850 // produce an rFlagsRegU instead of rFlagsReg.
13851 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13852 %{
13853   match(Set cr (CmpUL op1 op2));
13854 
13855   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13856   ins_encode %{
13857     __ cmpq($op1$$Register, $op2$$Register);
13858   %}
13859   ins_pipe(ialu_cr_reg_reg);
13860 %}
13861 
13862 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13863 %{
13864   match(Set cr (CmpUL op1 op2));
13865 
13866   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13867   ins_encode %{
13868     __ cmpq($op1$$Register, $op2$$constant);
13869   %}
13870   ins_pipe(ialu_cr_reg_imm);
13871 %}
13872 
13873 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13874 %{
13875   match(Set cr (CmpUL op1 (LoadL op2)));
13876 
13877   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13878   ins_encode %{
13879     __ cmpq($op1$$Register, $op2$$Address);
13880   %}
13881   ins_pipe(ialu_cr_reg_mem);
13882 %}
13883 
13884 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13885 %{
13886   match(Set cr (CmpUL src zero));
13887 
13888   format %{ "testq   $src, $src\t# unsigned" %}
13889   ins_encode %{
13890     __ testq($src$$Register, $src$$Register);
13891   %}
13892   ins_pipe(ialu_cr_reg_imm);
13893 %}
13894 
13895 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13896 %{
13897   match(Set cr (CmpI (LoadB mem) imm));
13898 
13899   ins_cost(125);
13900   format %{ "cmpb    $mem, $imm" %}
13901   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13902   ins_pipe(ialu_cr_reg_mem);
13903 %}
13904 
13905 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13906 %{
13907   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13908 
13909   ins_cost(125);
13910   format %{ "testb   $mem, $imm\t# ubyte" %}
13911   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13912   ins_pipe(ialu_cr_reg_mem);
13913 %}
13914 
13915 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13916 %{
13917   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13918 
13919   ins_cost(125);
13920   format %{ "testb   $mem, $imm\t# byte" %}
13921   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13922   ins_pipe(ialu_cr_reg_mem);
13923 %}
13924 
13925 //----------Max and Min--------------------------------------------------------
13926 // Min Instructions
13927 
13928 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13929 %{
13930   predicate(!UseAPX);
13931   effect(USE_DEF dst, USE src, USE cr);
13932 
13933   format %{ "cmovlgt $dst, $src\t# min" %}
13934   ins_encode %{
13935     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13936   %}
13937   ins_pipe(pipe_cmov_reg);
13938 %}
13939 
13940 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13941 %{
13942   predicate(UseAPX);
13943   effect(DEF dst, USE src1, USE src2, USE cr);
13944 
13945   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13946   ins_encode %{
13947     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13948   %}
13949   ins_pipe(pipe_cmov_reg);
13950 %}
13951 
13952 instruct minI_rReg(rRegI dst, rRegI src)
13953 %{
13954   predicate(!UseAPX);
13955   match(Set dst (MinI dst src));
13956 
13957   ins_cost(200);
13958   expand %{
13959     rFlagsReg cr;
13960     compI_rReg(cr, dst, src);
13961     cmovI_reg_g(dst, src, cr);
13962   %}
13963 %}
13964 
13965 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13966 %{
13967   predicate(UseAPX);
13968   match(Set dst (MinI src1 src2));
13969   effect(DEF dst, USE src1, USE src2);
13970 
13971   ins_cost(200);
13972   expand %{
13973     rFlagsReg cr;
13974     compI_rReg(cr, src1, src2);
13975     cmovI_reg_g_ndd(dst, src1, src2, cr);
13976   %}
13977 %}
13978 
13979 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13980 %{
13981   predicate(!UseAPX);
13982   effect(USE_DEF dst, USE src, USE cr);
13983 
13984   format %{ "cmovllt $dst, $src\t# max" %}
13985   ins_encode %{
13986     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13987   %}
13988   ins_pipe(pipe_cmov_reg);
13989 %}
13990 
13991 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13992 %{
13993   predicate(UseAPX);
13994   effect(DEF dst, USE src1, USE src2, USE cr);
13995 
13996   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13997   ins_encode %{
13998     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13999   %}
14000   ins_pipe(pipe_cmov_reg);
14001 %}
14002 
14003 instruct maxI_rReg(rRegI dst, rRegI src)
14004 %{
14005   predicate(!UseAPX);
14006   match(Set dst (MaxI dst src));
14007 
14008   ins_cost(200);
14009   expand %{
14010     rFlagsReg cr;
14011     compI_rReg(cr, dst, src);
14012     cmovI_reg_l(dst, src, cr);
14013   %}
14014 %}
14015 
14016 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
14017 %{
14018   predicate(UseAPX);
14019   match(Set dst (MaxI src1 src2));
14020   effect(DEF dst, USE src1, USE src2);
14021 
14022   ins_cost(200);
14023   expand %{
14024     rFlagsReg cr;
14025     compI_rReg(cr, src1, src2);
14026     cmovI_reg_l_ndd(dst, src1, src2, cr);
14027   %}
14028 %}
14029 
14030 // ============================================================================
14031 // Branch Instructions
14032 
14033 // Jump Direct - Label defines a relative address from JMP+1
14034 instruct jmpDir(label labl)
14035 %{
14036   match(Goto);
14037   effect(USE labl);
14038 
14039   ins_cost(300);
14040   format %{ "jmp     $labl" %}
14041   size(5);
14042   ins_encode %{
14043     Label* L = $labl$$label;
14044     __ jmp(*L, false); // Always long jump
14045   %}
14046   ins_pipe(pipe_jmp);
14047 %}
14048 
14049 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14050 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
14051 %{
14052   match(If cop cr);
14053   effect(USE labl);
14054 
14055   ins_cost(300);
14056   format %{ "j$cop     $labl" %}
14057   size(6);
14058   ins_encode %{
14059     Label* L = $labl$$label;
14060     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
14061   %}
14062   ins_pipe(pipe_jcc);
14063 %}
14064 
14065 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14066 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
14067 %{
14068   match(CountedLoopEnd cop cr);
14069   effect(USE labl);
14070 
14071   ins_cost(300);
14072   format %{ "j$cop     $labl\t# loop end" %}
14073   size(6);
14074   ins_encode %{
14075     Label* L = $labl$$label;
14076     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
14077   %}
14078   ins_pipe(pipe_jcc);
14079 %}
14080 
14081 // Jump Direct Conditional - using unsigned comparison
14082 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14083   match(If cop cmp);
14084   effect(USE labl);
14085 
14086   ins_cost(300);
14087   format %{ "j$cop,u   $labl" %}
14088   size(6);
14089   ins_encode %{
14090     Label* L = $labl$$label;
14091     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
14092   %}
14093   ins_pipe(pipe_jcc);
14094 %}
14095 
14096 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14097   match(If cop cmp);
14098   effect(USE labl);
14099 
14100   ins_cost(200);
14101   format %{ "j$cop,u   $labl" %}
14102   size(6);
14103   ins_encode %{
14104     Label* L = $labl$$label;
14105     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
14106   %}
14107   ins_pipe(pipe_jcc);
14108 %}
14109 
14110 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14111   match(If cop cmp);
14112   effect(USE labl);
14113 
14114   ins_cost(200);
14115   format %{ $$template
14116     if ($cop$$cmpcode == Assembler::notEqual) {
14117       $$emit$$"jp,u    $labl\n\t"
14118       $$emit$$"j$cop,u   $labl"
14119     } else {
14120       $$emit$$"jp,u    done\n\t"
14121       $$emit$$"j$cop,u   $labl\n\t"
14122       $$emit$$"done:"
14123     }
14124   %}
14125   ins_encode %{
14126     Label* l = $labl$$label;
14127     if ($cop$$cmpcode == Assembler::notEqual) {
14128       __ jcc(Assembler::parity, *l, false);
14129       __ jcc(Assembler::notEqual, *l, false);
14130     } else if ($cop$$cmpcode == Assembler::equal) {
14131       Label done;
14132       __ jccb(Assembler::parity, done);
14133       __ jcc(Assembler::equal, *l, false);
14134       __ bind(done);
14135     } else {
14136        ShouldNotReachHere();
14137     }
14138   %}
14139   ins_pipe(pipe_jcc);
14140 %}
14141 
14142 // ============================================================================
14143 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
14144 // superklass array for an instance of the superklass.  Set a hidden
14145 // internal cache on a hit (cache is checked with exposed code in
14146 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
14147 // encoding ALSO sets flags.
14148 
14149 instruct partialSubtypeCheck(rdi_RegP result,
14150                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
14151                              rFlagsReg cr)
14152 %{
14153   match(Set result (PartialSubtypeCheck sub super));
14154   predicate(!UseSecondarySupersTable);
14155   effect(KILL rcx, KILL cr);
14156 
14157   ins_cost(1100);  // slightly larger than the next version
14158   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
14159             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
14160             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
14161             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
14162             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
14163             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
14164             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
14165     "miss:\t" %}
14166 
14167   ins_encode %{
14168     Label miss;
14169     // NB: Callers may assume that, when $result is a valid register,
14170     // check_klass_subtype_slow_path_linear sets it to a nonzero
14171     // value.
14172     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
14173                                             $rcx$$Register, $result$$Register,
14174                                             nullptr, &miss,
14175                                             /*set_cond_codes:*/ true);
14176     __ xorptr($result$$Register, $result$$Register);
14177     __ bind(miss);
14178   %}
14179 
14180   ins_pipe(pipe_slow);
14181 %}
14182 
14183 // ============================================================================
14184 // Two versions of hashtable-based partialSubtypeCheck, both used when
14185 // we need to search for a super class in the secondary supers array.
14186 // The first is used when we don't know _a priori_ the class being
14187 // searched for. The second, far more common, is used when we do know:
14188 // this is used for instanceof, checkcast, and any case where C2 can
14189 // determine it by constant propagation.
14190 
14191 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
14192                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14193                                        rFlagsReg cr)
14194 %{
14195   match(Set result (PartialSubtypeCheck sub super));
14196   predicate(UseSecondarySupersTable);
14197   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14198 
14199   ins_cost(1000);
14200   format %{ "partialSubtypeCheck $result, $sub, $super" %}
14201 
14202   ins_encode %{
14203     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
14204 					 $temp3$$Register, $temp4$$Register, $result$$Register);
14205   %}
14206 
14207   ins_pipe(pipe_slow);
14208 %}
14209 
14210 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
14211                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14212                                        rFlagsReg cr)
14213 %{
14214   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
14215   predicate(UseSecondarySupersTable);
14216   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14217 
14218   ins_cost(700);  // smaller than the next version
14219   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
14220 
14221   ins_encode %{
14222     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
14223     if (InlineSecondarySupersTest) {
14224       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
14225                                        $temp3$$Register, $temp4$$Register, $result$$Register,
14226                                        super_klass_slot);
14227     } else {
14228       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
14229     }
14230   %}
14231 
14232   ins_pipe(pipe_slow);
14233 %}
14234 
14235 // ============================================================================
14236 // Branch Instructions -- short offset versions
14237 //
14238 // These instructions are used to replace jumps of a long offset (the default
14239 // match) with jumps of a shorter offset.  These instructions are all tagged
14240 // with the ins_short_branch attribute, which causes the ADLC to suppress the
14241 // match rules in general matching.  Instead, the ADLC generates a conversion
14242 // method in the MachNode which can be used to do in-place replacement of the
14243 // long variant with the shorter variant.  The compiler will determine if a
14244 // branch can be taken by the is_short_branch_offset() predicate in the machine
14245 // specific code section of the file.
14246 
14247 // Jump Direct - Label defines a relative address from JMP+1
14248 instruct jmpDir_short(label labl) %{
14249   match(Goto);
14250   effect(USE labl);
14251 
14252   ins_cost(300);
14253   format %{ "jmp,s   $labl" %}
14254   size(2);
14255   ins_encode %{
14256     Label* L = $labl$$label;
14257     __ jmpb(*L);
14258   %}
14259   ins_pipe(pipe_jmp);
14260   ins_short_branch(1);
14261 %}
14262 
14263 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14264 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
14265   match(If cop cr);
14266   effect(USE labl);
14267 
14268   ins_cost(300);
14269   format %{ "j$cop,s   $labl" %}
14270   size(2);
14271   ins_encode %{
14272     Label* L = $labl$$label;
14273     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14274   %}
14275   ins_pipe(pipe_jcc);
14276   ins_short_branch(1);
14277 %}
14278 
14279 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14280 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
14281   match(CountedLoopEnd cop cr);
14282   effect(USE labl);
14283 
14284   ins_cost(300);
14285   format %{ "j$cop,s   $labl\t# loop end" %}
14286   size(2);
14287   ins_encode %{
14288     Label* L = $labl$$label;
14289     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14290   %}
14291   ins_pipe(pipe_jcc);
14292   ins_short_branch(1);
14293 %}
14294 
14295 // Jump Direct Conditional - using unsigned comparison
14296 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14297   match(If cop cmp);
14298   effect(USE labl);
14299 
14300   ins_cost(300);
14301   format %{ "j$cop,us  $labl" %}
14302   size(2);
14303   ins_encode %{
14304     Label* L = $labl$$label;
14305     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14306   %}
14307   ins_pipe(pipe_jcc);
14308   ins_short_branch(1);
14309 %}
14310 
14311 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14312   match(If cop cmp);
14313   effect(USE labl);
14314 
14315   ins_cost(300);
14316   format %{ "j$cop,us  $labl" %}
14317   size(2);
14318   ins_encode %{
14319     Label* L = $labl$$label;
14320     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14321   %}
14322   ins_pipe(pipe_jcc);
14323   ins_short_branch(1);
14324 %}
14325 
14326 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14327   match(If cop cmp);
14328   effect(USE labl);
14329 
14330   ins_cost(300);
14331   format %{ $$template
14332     if ($cop$$cmpcode == Assembler::notEqual) {
14333       $$emit$$"jp,u,s  $labl\n\t"
14334       $$emit$$"j$cop,u,s  $labl"
14335     } else {
14336       $$emit$$"jp,u,s  done\n\t"
14337       $$emit$$"j$cop,u,s  $labl\n\t"
14338       $$emit$$"done:"
14339     }
14340   %}
14341   size(4);
14342   ins_encode %{
14343     Label* l = $labl$$label;
14344     if ($cop$$cmpcode == Assembler::notEqual) {
14345       __ jccb(Assembler::parity, *l);
14346       __ jccb(Assembler::notEqual, *l);
14347     } else if ($cop$$cmpcode == Assembler::equal) {
14348       Label done;
14349       __ jccb(Assembler::parity, done);
14350       __ jccb(Assembler::equal, *l);
14351       __ bind(done);
14352     } else {
14353        ShouldNotReachHere();
14354     }
14355   %}
14356   ins_pipe(pipe_jcc);
14357   ins_short_branch(1);
14358 %}
14359 
14360 // ============================================================================
14361 // inlined locking and unlocking
14362 
14363 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
14364   predicate(LockingMode != LM_LIGHTWEIGHT);
14365   match(Set cr (FastLock object box));
14366   effect(TEMP tmp, TEMP scr, USE_KILL box);
14367   ins_cost(300);
14368   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
14369   ins_encode %{
14370     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
14371                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
14372   %}
14373   ins_pipe(pipe_slow);
14374 %}
14375 
14376 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
14377   predicate(LockingMode != LM_LIGHTWEIGHT);
14378   match(Set cr (FastUnlock object box));
14379   effect(TEMP tmp, USE_KILL box);
14380   ins_cost(300);
14381   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
14382   ins_encode %{
14383     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
14384   %}
14385   ins_pipe(pipe_slow);
14386 %}
14387 
14388 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14389   predicate(LockingMode == LM_LIGHTWEIGHT);
14390   match(Set cr (FastLock object box));
14391   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14392   ins_cost(300);
14393   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14394   ins_encode %{
14395     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14396   %}
14397   ins_pipe(pipe_slow);
14398 %}
14399 
14400 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14401   predicate(LockingMode == LM_LIGHTWEIGHT);
14402   match(Set cr (FastUnlock object rax_reg));
14403   effect(TEMP tmp, USE_KILL rax_reg);
14404   ins_cost(300);
14405   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14406   ins_encode %{
14407     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14408   %}
14409   ins_pipe(pipe_slow);
14410 %}
14411 
14412 
14413 // ============================================================================
14414 // Safepoint Instructions
14415 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14416 %{
14417   match(SafePoint poll);
14418   effect(KILL cr, USE poll);
14419 
14420   format %{ "testl   rax, [$poll]\t"
14421             "# Safepoint: poll for GC" %}
14422   ins_cost(125);
14423   ins_encode %{
14424     __ relocate(relocInfo::poll_type);
14425     address pre_pc = __ pc();
14426     __ testl(rax, Address($poll$$Register, 0));
14427     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14428   %}
14429   ins_pipe(ialu_reg_mem);
14430 %}
14431 
14432 instruct mask_all_evexL(kReg dst, rRegL src) %{
14433   match(Set dst (MaskAll src));
14434   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14435   ins_encode %{
14436     int mask_len = Matcher::vector_length(this);
14437     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14438   %}
14439   ins_pipe( pipe_slow );
14440 %}
14441 
14442 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14443   predicate(Matcher::vector_length(n) > 32);
14444   match(Set dst (MaskAll src));
14445   effect(TEMP tmp);
14446   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14447   ins_encode %{
14448     int mask_len = Matcher::vector_length(this);
14449     __ movslq($tmp$$Register, $src$$Register);
14450     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14451   %}
14452   ins_pipe( pipe_slow );
14453 %}
14454 
14455 // ============================================================================
14456 // Procedure Call/Return Instructions
14457 // Call Java Static Instruction
14458 // Note: If this code changes, the corresponding ret_addr_offset() and
14459 //       compute_padding() functions will have to be adjusted.
14460 instruct CallStaticJavaDirect(method meth) %{
14461   match(CallStaticJava);
14462   effect(USE meth);
14463 
14464   ins_cost(300);
14465   format %{ "call,static " %}
14466   opcode(0xE8); /* E8 cd */
14467   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14468   ins_pipe(pipe_slow);
14469   ins_alignment(4);
14470 %}
14471 
14472 // Call Java Dynamic Instruction
14473 // Note: If this code changes, the corresponding ret_addr_offset() and
14474 //       compute_padding() functions will have to be adjusted.
14475 instruct CallDynamicJavaDirect(method meth)
14476 %{
14477   match(CallDynamicJava);
14478   effect(USE meth);
14479 
14480   ins_cost(300);
14481   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14482             "call,dynamic " %}
14483   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14484   ins_pipe(pipe_slow);
14485   ins_alignment(4);
14486 %}
14487 
14488 // Call Runtime Instruction
14489 instruct CallRuntimeDirect(method meth)
14490 %{
14491   match(CallRuntime);
14492   effect(USE meth);
14493 
14494   ins_cost(300);
14495   format %{ "call,runtime " %}
14496   ins_encode(clear_avx, Java_To_Runtime(meth));
14497   ins_pipe(pipe_slow);
14498 %}
14499 
14500 // Call runtime without safepoint
14501 instruct CallLeafDirect(method meth)
14502 %{
14503   match(CallLeaf);
14504   effect(USE meth);
14505 
14506   ins_cost(300);
14507   format %{ "call_leaf,runtime " %}
14508   ins_encode(clear_avx, Java_To_Runtime(meth));
14509   ins_pipe(pipe_slow);
14510 %}
14511 
14512 // Call runtime without safepoint and with vector arguments
14513 instruct CallLeafDirectVector(method meth)
14514 %{
14515   match(CallLeafVector);
14516   effect(USE meth);
14517 
14518   ins_cost(300);
14519   format %{ "call_leaf,vector " %}
14520   ins_encode(Java_To_Runtime(meth));
14521   ins_pipe(pipe_slow);
14522 %}
14523 
14524 // Call runtime without safepoint
14525 // entry point is null, target holds the address to call
14526 instruct CallLeafNoFPInDirect(rRegP target)
14527 %{
14528   predicate(n->as_Call()->entry_point() == nullptr);
14529   match(CallLeafNoFP target);
14530 
14531   ins_cost(300);
14532   format %{ "call_leaf_nofp,runtime indirect " %}
14533   ins_encode %{
14534      __ call($target$$Register);
14535   %}
14536 
14537   ins_pipe(pipe_slow);
14538 %}
14539 
14540 instruct CallLeafNoFPDirect(method meth)
14541 %{
14542   predicate(n->as_Call()->entry_point() != nullptr);
14543   match(CallLeafNoFP);
14544   effect(USE meth);
14545 
14546   ins_cost(300);
14547   format %{ "call_leaf_nofp,runtime " %}
14548   ins_encode(clear_avx, Java_To_Runtime(meth));
14549   ins_pipe(pipe_slow);
14550 %}
14551 
14552 // Return Instruction
14553 // Remove the return address & jump to it.
14554 // Notice: We always emit a nop after a ret to make sure there is room
14555 // for safepoint patching
14556 instruct Ret()
14557 %{
14558   match(Return);
14559 
14560   format %{ "ret" %}
14561   ins_encode %{
14562     __ ret(0);
14563   %}
14564   ins_pipe(pipe_jmp);
14565 %}
14566 
14567 // Tail Call; Jump from runtime stub to Java code.
14568 // Also known as an 'interprocedural jump'.
14569 // Target of jump will eventually return to caller.
14570 // TailJump below removes the return address.
14571 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14572 // emitted just above the TailCall which has reset rbp to the caller state.
14573 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14574 %{
14575   match(TailCall jump_target method_ptr);
14576 
14577   ins_cost(300);
14578   format %{ "jmp     $jump_target\t# rbx holds method" %}
14579   ins_encode %{
14580     __ jmp($jump_target$$Register);
14581   %}
14582   ins_pipe(pipe_jmp);
14583 %}
14584 
14585 // Tail Jump; remove the return address; jump to target.
14586 // TailCall above leaves the return address around.
14587 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14588 %{
14589   match(TailJump jump_target ex_oop);
14590 
14591   ins_cost(300);
14592   format %{ "popq    rdx\t# pop return address\n\t"
14593             "jmp     $jump_target" %}
14594   ins_encode %{
14595     __ popq(as_Register(RDX_enc));
14596     __ jmp($jump_target$$Register);
14597   %}
14598   ins_pipe(pipe_jmp);
14599 %}
14600 
14601 // Forward exception.
14602 instruct ForwardExceptionjmp()
14603 %{
14604   match(ForwardException);
14605 
14606   format %{ "jmp     forward_exception_stub" %}
14607   ins_encode %{
14608     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14609   %}
14610   ins_pipe(pipe_jmp);
14611 %}
14612 
14613 // Create exception oop: created by stack-crawling runtime code.
14614 // Created exception is now available to this handler, and is setup
14615 // just prior to jumping to this handler.  No code emitted.
14616 instruct CreateException(rax_RegP ex_oop)
14617 %{
14618   match(Set ex_oop (CreateEx));
14619 
14620   size(0);
14621   // use the following format syntax
14622   format %{ "# exception oop is in rax; no code emitted" %}
14623   ins_encode();
14624   ins_pipe(empty);
14625 %}
14626 
14627 // Rethrow exception:
14628 // The exception oop will come in the first argument position.
14629 // Then JUMP (not call) to the rethrow stub code.
14630 instruct RethrowException()
14631 %{
14632   match(Rethrow);
14633 
14634   // use the following format syntax
14635   format %{ "jmp     rethrow_stub" %}
14636   ins_encode %{
14637     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14638   %}
14639   ins_pipe(pipe_jmp);
14640 %}
14641 
14642 // ============================================================================
14643 // This name is KNOWN by the ADLC and cannot be changed.
14644 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14645 // for this guy.
14646 instruct tlsLoadP(r15_RegP dst) %{
14647   match(Set dst (ThreadLocal));
14648   effect(DEF dst);
14649 
14650   size(0);
14651   format %{ "# TLS is in R15" %}
14652   ins_encode( /*empty encoding*/ );
14653   ins_pipe(ialu_reg_reg);
14654 %}
14655 
14656 
14657 //----------PEEPHOLE RULES-----------------------------------------------------
14658 // These must follow all instruction definitions as they use the names
14659 // defined in the instructions definitions.
14660 //
14661 // peeppredicate ( rule_predicate );
14662 // // the predicate unless which the peephole rule will be ignored
14663 //
14664 // peepmatch ( root_instr_name [preceding_instruction]* );
14665 //
14666 // peepprocedure ( procedure_name );
14667 // // provide a procedure name to perform the optimization, the procedure should
14668 // // reside in the architecture dependent peephole file, the method has the
14669 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14670 // // with the arguments being the basic block, the current node index inside the
14671 // // block, the register allocator, the functions upon invoked return a new node
14672 // // defined in peepreplace, and the rules of the nodes appearing in the
14673 // // corresponding peepmatch, the function return true if successful, else
14674 // // return false
14675 //
14676 // peepconstraint %{
14677 // (instruction_number.operand_name relational_op instruction_number.operand_name
14678 //  [, ...] );
14679 // // instruction numbers are zero-based using left to right order in peepmatch
14680 //
14681 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14682 // // provide an instruction_number.operand_name for each operand that appears
14683 // // in the replacement instruction's match rule
14684 //
14685 // ---------VM FLAGS---------------------------------------------------------
14686 //
14687 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14688 //
14689 // Each peephole rule is given an identifying number starting with zero and
14690 // increasing by one in the order seen by the parser.  An individual peephole
14691 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14692 // on the command-line.
14693 //
14694 // ---------CURRENT LIMITATIONS----------------------------------------------
14695 //
14696 // Only transformations inside a basic block (do we need more for peephole)
14697 //
14698 // ---------EXAMPLE----------------------------------------------------------
14699 //
14700 // // pertinent parts of existing instructions in architecture description
14701 // instruct movI(rRegI dst, rRegI src)
14702 // %{
14703 //   match(Set dst (CopyI src));
14704 // %}
14705 //
14706 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14707 // %{
14708 //   match(Set dst (AddI dst src));
14709 //   effect(KILL cr);
14710 // %}
14711 //
14712 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14713 // %{
14714 //   match(Set dst (AddI dst src));
14715 // %}
14716 //
14717 // 1. Simple replacement
14718 // - Only match adjacent instructions in same basic block
14719 // - Only equality constraints
14720 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14721 // - Only one replacement instruction
14722 //
14723 // // Change (inc mov) to lea
14724 // peephole %{
14725 //   // lea should only be emitted when beneficial
14726 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14727 //   // increment preceded by register-register move
14728 //   peepmatch ( incI_rReg movI );
14729 //   // require that the destination register of the increment
14730 //   // match the destination register of the move
14731 //   peepconstraint ( 0.dst == 1.dst );
14732 //   // construct a replacement instruction that sets
14733 //   // the destination to ( move's source register + one )
14734 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14735 // %}
14736 //
14737 // 2. Procedural replacement
14738 // - More flexible finding relevent nodes
14739 // - More flexible constraints
14740 // - More flexible transformations
14741 // - May utilise architecture-dependent API more effectively
14742 // - Currently only one replacement instruction due to adlc parsing capabilities
14743 //
14744 // // Change (inc mov) to lea
14745 // peephole %{
14746 //   // lea should only be emitted when beneficial
14747 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14748 //   // the rule numbers of these nodes inside are passed into the function below
14749 //   peepmatch ( incI_rReg movI );
14750 //   // the method that takes the responsibility of transformation
14751 //   peepprocedure ( inc_mov_to_lea );
14752 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14753 //   // node is passed into the function above
14754 //   peepreplace ( leaI_rReg_immI() );
14755 // %}
14756 
14757 // These instructions is not matched by the matcher but used by the peephole
14758 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14759 %{
14760   predicate(false);
14761   match(Set dst (AddI src1 src2));
14762   format %{ "leal    $dst, [$src1 + $src2]" %}
14763   ins_encode %{
14764     Register dst = $dst$$Register;
14765     Register src1 = $src1$$Register;
14766     Register src2 = $src2$$Register;
14767     if (src1 != rbp && src1 != r13) {
14768       __ leal(dst, Address(src1, src2, Address::times_1));
14769     } else {
14770       assert(src2 != rbp && src2 != r13, "");
14771       __ leal(dst, Address(src2, src1, Address::times_1));
14772     }
14773   %}
14774   ins_pipe(ialu_reg_reg);
14775 %}
14776 
14777 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14778 %{
14779   predicate(false);
14780   match(Set dst (AddI src1 src2));
14781   format %{ "leal    $dst, [$src1 + $src2]" %}
14782   ins_encode %{
14783     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14784   %}
14785   ins_pipe(ialu_reg_reg);
14786 %}
14787 
14788 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14789 %{
14790   predicate(false);
14791   match(Set dst (LShiftI src shift));
14792   format %{ "leal    $dst, [$src << $shift]" %}
14793   ins_encode %{
14794     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14795     Register src = $src$$Register;
14796     if (scale == Address::times_2 && src != rbp && src != r13) {
14797       __ leal($dst$$Register, Address(src, src, Address::times_1));
14798     } else {
14799       __ leal($dst$$Register, Address(noreg, src, scale));
14800     }
14801   %}
14802   ins_pipe(ialu_reg_reg);
14803 %}
14804 
14805 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14806 %{
14807   predicate(false);
14808   match(Set dst (AddL src1 src2));
14809   format %{ "leaq    $dst, [$src1 + $src2]" %}
14810   ins_encode %{
14811     Register dst = $dst$$Register;
14812     Register src1 = $src1$$Register;
14813     Register src2 = $src2$$Register;
14814     if (src1 != rbp && src1 != r13) {
14815       __ leaq(dst, Address(src1, src2, Address::times_1));
14816     } else {
14817       assert(src2 != rbp && src2 != r13, "");
14818       __ leaq(dst, Address(src2, src1, Address::times_1));
14819     }
14820   %}
14821   ins_pipe(ialu_reg_reg);
14822 %}
14823 
14824 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14825 %{
14826   predicate(false);
14827   match(Set dst (AddL src1 src2));
14828   format %{ "leaq    $dst, [$src1 + $src2]" %}
14829   ins_encode %{
14830     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14831   %}
14832   ins_pipe(ialu_reg_reg);
14833 %}
14834 
14835 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14836 %{
14837   predicate(false);
14838   match(Set dst (LShiftL src shift));
14839   format %{ "leaq    $dst, [$src << $shift]" %}
14840   ins_encode %{
14841     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14842     Register src = $src$$Register;
14843     if (scale == Address::times_2 && src != rbp && src != r13) {
14844       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14845     } else {
14846       __ leaq($dst$$Register, Address(noreg, src, scale));
14847     }
14848   %}
14849   ins_pipe(ialu_reg_reg);
14850 %}
14851 
14852 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14853 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14854 // processors with at least partial ALU support for lea
14855 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14856 // beneficial for processors with full ALU support
14857 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14858 
14859 peephole
14860 %{
14861   peeppredicate(VM_Version::supports_fast_2op_lea());
14862   peepmatch (addI_rReg);
14863   peepprocedure (lea_coalesce_reg);
14864   peepreplace (leaI_rReg_rReg_peep());
14865 %}
14866 
14867 peephole
14868 %{
14869   peeppredicate(VM_Version::supports_fast_2op_lea());
14870   peepmatch (addI_rReg_imm);
14871   peepprocedure (lea_coalesce_imm);
14872   peepreplace (leaI_rReg_immI_peep());
14873 %}
14874 
14875 peephole
14876 %{
14877   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14878                 VM_Version::is_intel_cascade_lake());
14879   peepmatch (incI_rReg);
14880   peepprocedure (lea_coalesce_imm);
14881   peepreplace (leaI_rReg_immI_peep());
14882 %}
14883 
14884 peephole
14885 %{
14886   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14887                 VM_Version::is_intel_cascade_lake());
14888   peepmatch (decI_rReg);
14889   peepprocedure (lea_coalesce_imm);
14890   peepreplace (leaI_rReg_immI_peep());
14891 %}
14892 
14893 peephole
14894 %{
14895   peeppredicate(VM_Version::supports_fast_2op_lea());
14896   peepmatch (salI_rReg_immI2);
14897   peepprocedure (lea_coalesce_imm);
14898   peepreplace (leaI_rReg_immI2_peep());
14899 %}
14900 
14901 peephole
14902 %{
14903   peeppredicate(VM_Version::supports_fast_2op_lea());
14904   peepmatch (addL_rReg);
14905   peepprocedure (lea_coalesce_reg);
14906   peepreplace (leaL_rReg_rReg_peep());
14907 %}
14908 
14909 peephole
14910 %{
14911   peeppredicate(VM_Version::supports_fast_2op_lea());
14912   peepmatch (addL_rReg_imm);
14913   peepprocedure (lea_coalesce_imm);
14914   peepreplace (leaL_rReg_immL32_peep());
14915 %}
14916 
14917 peephole
14918 %{
14919   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14920                 VM_Version::is_intel_cascade_lake());
14921   peepmatch (incL_rReg);
14922   peepprocedure (lea_coalesce_imm);
14923   peepreplace (leaL_rReg_immL32_peep());
14924 %}
14925 
14926 peephole
14927 %{
14928   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14929                 VM_Version::is_intel_cascade_lake());
14930   peepmatch (decL_rReg);
14931   peepprocedure (lea_coalesce_imm);
14932   peepreplace (leaL_rReg_immL32_peep());
14933 %}
14934 
14935 peephole
14936 %{
14937   peeppredicate(VM_Version::supports_fast_2op_lea());
14938   peepmatch (salL_rReg_immI2);
14939   peepprocedure (lea_coalesce_imm);
14940   peepreplace (leaL_rReg_immI2_peep());
14941 %}
14942 
14943 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14944 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14945 
14946 //int variant
14947 peephole
14948 %{
14949   peepmatch (testI_reg);
14950   peepprocedure (test_may_remove);
14951 %}
14952 
14953 //long variant
14954 peephole
14955 %{
14956   peepmatch (testL_reg);
14957   peepprocedure (test_may_remove);
14958 %}
14959 
14960 
14961 //----------SMARTSPILL RULES---------------------------------------------------
14962 // These must follow all instruction definitions as they use the names
14963 // defined in the instructions definitions.