1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 
  442 extern RegMask _ANY_REG_mask;
  443 extern RegMask _PTR_REG_mask;
  444 extern RegMask _PTR_REG_NO_RBP_mask;
  445 extern RegMask _PTR_NO_RAX_REG_mask;
  446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  447 extern RegMask _LONG_REG_mask;
  448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  449 extern RegMask _LONG_NO_RCX_REG_mask;
  450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  451 extern RegMask _INT_REG_mask;
  452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  453 extern RegMask _INT_NO_RCX_REG_mask;
  454 extern RegMask _INT_NO_RBP_R13_REG_mask;
  455 extern RegMask _FLOAT_REG_mask;
  456 
  457 extern RegMask _STACK_OR_PTR_REG_mask;
  458 extern RegMask _STACK_OR_LONG_REG_mask;
  459 extern RegMask _STACK_OR_INT_REG_mask;
  460 
  461 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  462 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  463 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  464 
  465 %}
  466 
  467 source %{
  468 #define   RELOC_IMM64    Assembler::imm_operand
  469 #define   RELOC_DISP32   Assembler::disp32_operand
  470 
  471 #define __ masm->
  472 
  473 RegMask _ANY_REG_mask;
  474 RegMask _PTR_REG_mask;
  475 RegMask _PTR_REG_NO_RBP_mask;
  476 RegMask _PTR_NO_RAX_REG_mask;
  477 RegMask _PTR_NO_RAX_RBX_REG_mask;
  478 RegMask _LONG_REG_mask;
  479 RegMask _LONG_NO_RAX_RDX_REG_mask;
  480 RegMask _LONG_NO_RCX_REG_mask;
  481 RegMask _LONG_NO_RBP_R13_REG_mask;
  482 RegMask _INT_REG_mask;
  483 RegMask _INT_NO_RAX_RDX_REG_mask;
  484 RegMask _INT_NO_RCX_REG_mask;
  485 RegMask _INT_NO_RBP_R13_REG_mask;
  486 RegMask _FLOAT_REG_mask;
  487 RegMask _STACK_OR_PTR_REG_mask;
  488 RegMask _STACK_OR_LONG_REG_mask;
  489 RegMask _STACK_OR_INT_REG_mask;
  490 
  491 static bool need_r12_heapbase() {
  492   return UseCompressedOops;
  493 }
  494 
  495 void reg_mask_init() {
  496   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  497 
  498   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  499   // We derive a number of subsets from it.
  500   _ANY_REG_mask = _ALL_REG_mask;
  501 
  502   if (PreserveFramePointer) {
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  504     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  505   }
  506   if (need_r12_heapbase()) {
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  508     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  509   }
  510 
  511   _PTR_REG_mask = _ANY_REG_mask;
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  515   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  516   if (!UseAPX) {
  517     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  519       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  520     }
  521   }
  522 
  523   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  524   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  525 
  526   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  528   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  529 
  530   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  532   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  533 
  534   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  536   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  537 
  538 
  539   _LONG_REG_mask = _PTR_REG_mask;
  540   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  541   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  542 
  543   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  547   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  548 
  549   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  551   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  552 
  553   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  557   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  558 
  559   _INT_REG_mask = _ALL_INT_REG_mask;
  560   if (!UseAPX) {
  561     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  562       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  563     }
  564   }
  565 
  566   if (PreserveFramePointer) {
  567     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  568   }
  569   if (need_r12_heapbase()) {
  570     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  571   }
  572 
  573   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  574   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  575 
  576   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  578   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  579 
  580   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  581   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  582 
  583   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  585   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  586 
  587   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  588   // from the float_reg_legacy/float_reg_evex register class.
  589   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  590 }
  591 
  592 static bool generate_vzeroupper(Compile* C) {
  593   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  594 }
  595 
  596 static int clear_avx_size() {
  597   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  598 }
  599 
  600 // !!!!! Special hack to get all types of calls to specify the byte offset
  601 //       from the start of the call to the point where the return address
  602 //       will point.
  603 int MachCallStaticJavaNode::ret_addr_offset()
  604 {
  605   int offset = 5; // 5 bytes from start of call to where return address points
  606   offset += clear_avx_size();
  607   return offset;
  608 }
  609 
  610 int MachCallDynamicJavaNode::ret_addr_offset()
  611 {
  612   int offset = 15; // 15 bytes from start of call to where return address points
  613   offset += clear_avx_size();
  614   return offset;
  615 }
  616 
  617 int MachCallRuntimeNode::ret_addr_offset() {
  618   int offset = 13; // movq r10,#addr; callq (r10)
  619   if (this->ideal_Opcode() != Op_CallLeafVector) {
  620     offset += clear_avx_size();
  621   }
  622   return offset;
  623 }
  624 //
  625 // Compute padding required for nodes which need alignment
  626 //
  627 
  628 // The address of the call instruction needs to be 4-byte aligned to
  629 // ensure that it does not span a cache line so that it can be patched.
  630 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  631 {
  632   current_offset += clear_avx_size(); // skip vzeroupper
  633   current_offset += 1; // skip call opcode byte
  634   return align_up(current_offset, alignment_required()) - current_offset;
  635 }
  636 
  637 // The address of the call instruction needs to be 4-byte aligned to
  638 // ensure that it does not span a cache line so that it can be patched.
  639 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  640 {
  641   current_offset += clear_avx_size(); // skip vzeroupper
  642   current_offset += 11; // skip movq instruction + call opcode byte
  643   return align_up(current_offset, alignment_required()) - current_offset;
  644 }
  645 
  646 // This could be in MacroAssembler but it's fairly C2 specific
  647 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  648   Label exit;
  649   __ jccb(Assembler::noParity, exit);
  650   __ pushf();
  651   //
  652   // comiss/ucomiss instructions set ZF,PF,CF flags and
  653   // zero OF,AF,SF for NaN values.
  654   // Fixup flags by zeroing ZF,PF so that compare of NaN
  655   // values returns 'less than' result (CF is set).
  656   // Leave the rest of flags unchanged.
  657   //
  658   //    7 6 5 4 3 2 1 0
  659   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  660   //    0 0 1 0 1 0 1 1   (0x2B)
  661   //
  662   __ andq(Address(rsp, 0), 0xffffff2b);
  663   __ popf();
  664   __ bind(exit);
  665 }
  666 
  667 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  668   Label done;
  669   __ movl(dst, -1);
  670   __ jcc(Assembler::parity, done);
  671   __ jcc(Assembler::below, done);
  672   __ setcc(Assembler::notEqual, dst);
  673   __ bind(done);
  674 }
  675 
  676 // Math.min()    # Math.max()
  677 // --------------------------
  678 // ucomis[s/d]   #
  679 // ja   -> b     # a
  680 // jp   -> NaN   # NaN
  681 // jb   -> a     # b
  682 // je            #
  683 // |-jz -> a | b # a & b
  684 // |    -> a     #
  685 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  686                             XMMRegister a, XMMRegister b,
  687                             XMMRegister xmmt, Register rt,
  688                             bool min, bool single) {
  689 
  690   Label nan, zero, below, above, done;
  691 
  692   if (single)
  693     __ ucomiss(a, b);
  694   else
  695     __ ucomisd(a, b);
  696 
  697   if (dst->encoding() != (min ? b : a)->encoding())
  698     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  699   else
  700     __ jccb(Assembler::above, done);
  701 
  702   __ jccb(Assembler::parity, nan);  // PF=1
  703   __ jccb(Assembler::below, below); // CF=1
  704 
  705   // equal
  706   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  707   if (single) {
  708     __ ucomiss(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movflt(dst, a);
  712     __ jmp(done);
  713   }
  714   else {
  715     __ ucomisd(a, xmmt);
  716     __ jccb(Assembler::equal, zero);
  717 
  718     __ movdbl(dst, a);
  719     __ jmp(done);
  720   }
  721 
  722   __ bind(zero);
  723   if (min)
  724     __ vpor(dst, a, b, Assembler::AVX_128bit);
  725   else
  726     __ vpand(dst, a, b, Assembler::AVX_128bit);
  727 
  728   __ jmp(done);
  729 
  730   __ bind(above);
  731   if (single)
  732     __ movflt(dst, min ? b : a);
  733   else
  734     __ movdbl(dst, min ? b : a);
  735 
  736   __ jmp(done);
  737 
  738   __ bind(nan);
  739   if (single) {
  740     __ movl(rt, 0x7fc00000); // Float.NaN
  741     __ movdl(dst, rt);
  742   }
  743   else {
  744     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  745     __ movdq(dst, rt);
  746   }
  747   __ jmp(done);
  748 
  749   __ bind(below);
  750   if (single)
  751     __ movflt(dst, min ? a : b);
  752   else
  753     __ movdbl(dst, min ? a : b);
  754 
  755   __ bind(done);
  756 }
  757 
  758 //=============================================================================
  759 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  760 
  761 int ConstantTable::calculate_table_base_offset() const {
  762   return 0;  // absolute addressing, no offset
  763 }
  764 
  765 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  766 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  767   ShouldNotReachHere();
  768 }
  769 
  770 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  771   // Empty encoding
  772 }
  773 
  774 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  775   return 0;
  776 }
  777 
  778 #ifndef PRODUCT
  779 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  780   st->print("# MachConstantBaseNode (empty encoding)");
  781 }
  782 #endif
  783 
  784 
  785 //=============================================================================
  786 #ifndef PRODUCT
  787 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  788   Compile* C = ra_->C;
  789 
  790   int framesize = C->output()->frame_size_in_bytes();
  791   int bangsize = C->output()->bang_size_in_bytes();
  792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  793   // Remove wordSize for return addr which is already pushed.
  794   framesize -= wordSize;
  795 
  796   if (C->output()->need_stack_bang(bangsize)) {
  797     framesize -= wordSize;
  798     st->print("# stack bang (%d bytes)", bangsize);
  799     st->print("\n\t");
  800     st->print("pushq   rbp\t# Save rbp");
  801     if (PreserveFramePointer) {
  802         st->print("\n\t");
  803         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  804     }
  805     if (framesize) {
  806       st->print("\n\t");
  807       st->print("subq    rsp, #%d\t# Create frame",framesize);
  808     }
  809   } else {
  810     st->print("subq    rsp, #%d\t# Create frame",framesize);
  811     st->print("\n\t");
  812     framesize -= wordSize;
  813     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  814     if (PreserveFramePointer) {
  815       st->print("\n\t");
  816       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  817       if (framesize > 0) {
  818         st->print("\n\t");
  819         st->print("addq    rbp, #%d", framesize);
  820       }
  821     }
  822   }
  823 
  824   if (VerifyStackAtCalls) {
  825     st->print("\n\t");
  826     framesize -= wordSize;
  827     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  828 #ifdef ASSERT
  829     st->print("\n\t");
  830     st->print("# stack alignment check");
  831 #endif
  832   }
  833   if (C->stub_function() != nullptr) {
  834     st->print("\n\t");
  835     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  836     st->print("\n\t");
  837     st->print("je      fast_entry\t");
  838     st->print("\n\t");
  839     st->print("call    #nmethod_entry_barrier_stub\t");
  840     st->print("\n\tfast_entry:");
  841   }
  842   st->cr();
  843 }
  844 #endif
  845 
  846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  847   Compile* C = ra_->C;
  848 
  849   int framesize = C->output()->frame_size_in_bytes();
  850   int bangsize = C->output()->bang_size_in_bytes();
  851 
  852   if (C->clinit_barrier_on_entry()) {
  853     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  854     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  855 
  856     Label L_skip_barrier;
  857     Register klass = rscratch1;
  858 
  859     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  860     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  861 
  862     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  863 
  864     __ bind(L_skip_barrier);
  865   }
  866 
  867   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  868 
  869   C->output()->set_frame_complete(__ offset());
  870 
  871   if (C->has_mach_constant_base_node()) {
  872     // NOTE: We set the table base offset here because users might be
  873     // emitted before MachConstantBaseNode.
  874     ConstantTable& constant_table = C->output()->constant_table();
  875     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  876   }
  877 }
  878 
  879 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  880 {
  881   return MachNode::size(ra_); // too many variables; just compute it
  882                               // the hard way
  883 }
  884 
  885 int MachPrologNode::reloc() const
  886 {
  887   return 0; // a large enough number
  888 }
  889 
  890 //=============================================================================
  891 #ifndef PRODUCT
  892 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  893 {
  894   Compile* C = ra_->C;
  895   if (generate_vzeroupper(C)) {
  896     st->print("vzeroupper");
  897     st->cr(); st->print("\t");
  898   }
  899 
  900   int framesize = C->output()->frame_size_in_bytes();
  901   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  902   // Remove word for return adr already pushed
  903   // and RBP
  904   framesize -= 2*wordSize;
  905 
  906   if (framesize) {
  907     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  908     st->print("\t");
  909   }
  910 
  911   st->print_cr("popq    rbp");
  912   if (do_polling() && C->is_method_compilation()) {
  913     st->print("\t");
  914     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  915                  "ja      #safepoint_stub\t"
  916                  "# Safepoint: poll for GC");
  917   }
  918 }
  919 #endif
  920 
  921 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  922 {
  923   Compile* C = ra_->C;
  924 
  925   if (generate_vzeroupper(C)) {
  926     // Clear upper bits of YMM registers when current compiled code uses
  927     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  928     __ vzeroupper();
  929   }
  930 
  931   int framesize = C->output()->frame_size_in_bytes();
  932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  933   // Remove word for return adr already pushed
  934   // and RBP
  935   framesize -= 2*wordSize;
  936 
  937   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  938 
  939   if (framesize) {
  940     __ addq(rsp, framesize);
  941   }
  942 
  943   __ popq(rbp);
  944 
  945   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  946     __ reserved_stack_check();
  947   }
  948 
  949   if (do_polling() && C->is_method_compilation()) {
  950     Label dummy_label;
  951     Label* code_stub = &dummy_label;
  952     if (!C->output()->in_scratch_emit_size()) {
  953       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  954       C->output()->add_stub(stub);
  955       code_stub = &stub->entry();
  956     }
  957     __ relocate(relocInfo::poll_return_type);
  958     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  959   }
  960 }
  961 
  962 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  963 {
  964   return MachNode::size(ra_); // too many variables; just compute it
  965                               // the hard way
  966 }
  967 
  968 int MachEpilogNode::reloc() const
  969 {
  970   return 2; // a large enough number
  971 }
  972 
  973 const Pipeline* MachEpilogNode::pipeline() const
  974 {
  975   return MachNode::pipeline_class();
  976 }
  977 
  978 //=============================================================================
  979 
  980 enum RC {
  981   rc_bad,
  982   rc_int,
  983   rc_kreg,
  984   rc_float,
  985   rc_stack
  986 };
  987 
  988 static enum RC rc_class(OptoReg::Name reg)
  989 {
  990   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  991 
  992   if (OptoReg::is_stack(reg)) return rc_stack;
  993 
  994   VMReg r = OptoReg::as_VMReg(reg);
  995 
  996   if (r->is_Register()) return rc_int;
  997 
  998   if (r->is_KRegister()) return rc_kreg;
  999 
 1000   assert(r->is_XMMRegister(), "must be");
 1001   return rc_float;
 1002 }
 1003 
 1004 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1005 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1006                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1007 
 1008 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1009                      int stack_offset, int reg, uint ireg, outputStream* st);
 1010 
 1011 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1012                                       int dst_offset, uint ireg, outputStream* st) {
 1013   if (masm) {
 1014     switch (ireg) {
 1015     case Op_VecS:
 1016       __ movq(Address(rsp, -8), rax);
 1017       __ movl(rax, Address(rsp, src_offset));
 1018       __ movl(Address(rsp, dst_offset), rax);
 1019       __ movq(rax, Address(rsp, -8));
 1020       break;
 1021     case Op_VecD:
 1022       __ pushq(Address(rsp, src_offset));
 1023       __ popq (Address(rsp, dst_offset));
 1024       break;
 1025     case Op_VecX:
 1026       __ pushq(Address(rsp, src_offset));
 1027       __ popq (Address(rsp, dst_offset));
 1028       __ pushq(Address(rsp, src_offset+8));
 1029       __ popq (Address(rsp, dst_offset+8));
 1030       break;
 1031     case Op_VecY:
 1032       __ vmovdqu(Address(rsp, -32), xmm0);
 1033       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1034       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1035       __ vmovdqu(xmm0, Address(rsp, -32));
 1036       break;
 1037     case Op_VecZ:
 1038       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1039       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1040       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1041       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #ifndef PRODUCT
 1047   } else {
 1048     switch (ireg) {
 1049     case Op_VecS:
 1050       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                 "movl    rax, [rsp + #%d]\n\t"
 1052                 "movl    [rsp + #%d], rax\n\t"
 1053                 "movq    rax, [rsp - #8]",
 1054                 src_offset, dst_offset);
 1055       break;
 1056     case Op_VecD:
 1057       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1058                 "popq    [rsp + #%d]",
 1059                 src_offset, dst_offset);
 1060       break;
 1061      case Op_VecX:
 1062       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1063                 "popq    [rsp + #%d]\n\t"
 1064                 "pushq   [rsp + #%d]\n\t"
 1065                 "popq    [rsp + #%d]",
 1066                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1067       break;
 1068     case Op_VecY:
 1069       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1070                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1071                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1072                 "vmovdqu xmm0, [rsp - #32]",
 1073                 src_offset, dst_offset);
 1074       break;
 1075     case Op_VecZ:
 1076       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1077                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1078                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1079                 "vmovdqu xmm0, [rsp - #64]",
 1080                 src_offset, dst_offset);
 1081       break;
 1082     default:
 1083       ShouldNotReachHere();
 1084     }
 1085 #endif
 1086   }
 1087 }
 1088 
 1089 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1090                                        PhaseRegAlloc* ra_,
 1091                                        bool do_size,
 1092                                        outputStream* st) const {
 1093   assert(masm != nullptr || st  != nullptr, "sanity");
 1094   // Get registers to move
 1095   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1096   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1097   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1098   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1099 
 1100   enum RC src_second_rc = rc_class(src_second);
 1101   enum RC src_first_rc = rc_class(src_first);
 1102   enum RC dst_second_rc = rc_class(dst_second);
 1103   enum RC dst_first_rc = rc_class(dst_first);
 1104 
 1105   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1106          "must move at least 1 register" );
 1107 
 1108   if (src_first == dst_first && src_second == dst_second) {
 1109     // Self copy, no move
 1110     return 0;
 1111   }
 1112   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1113     uint ireg = ideal_reg();
 1114     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1115     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1116     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1117       // mem -> mem
 1118       int src_offset = ra_->reg2offset(src_first);
 1119       int dst_offset = ra_->reg2offset(dst_first);
 1120       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1121     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1122       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1123     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1124       int stack_offset = ra_->reg2offset(dst_first);
 1125       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1126     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1127       int stack_offset = ra_->reg2offset(src_first);
 1128       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1129     } else {
 1130       ShouldNotReachHere();
 1131     }
 1132     return 0;
 1133   }
 1134   if (src_first_rc == rc_stack) {
 1135     // mem ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // mem -> mem
 1138       assert(src_second != dst_first, "overlap");
 1139       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1140           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1141         // 64-bit
 1142         int src_offset = ra_->reg2offset(src_first);
 1143         int dst_offset = ra_->reg2offset(dst_first);
 1144         if (masm) {
 1145           __ pushq(Address(rsp, src_offset));
 1146           __ popq (Address(rsp, dst_offset));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1150                     "popq    [rsp + #%d]",
 1151                      src_offset, dst_offset);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         // No pushl/popl, so:
 1159         int src_offset = ra_->reg2offset(src_first);
 1160         int dst_offset = ra_->reg2offset(dst_first);
 1161         if (masm) {
 1162           __ movq(Address(rsp, -8), rax);
 1163           __ movl(rax, Address(rsp, src_offset));
 1164           __ movl(Address(rsp, dst_offset), rax);
 1165           __ movq(rax, Address(rsp, -8));
 1166 #ifndef PRODUCT
 1167         } else {
 1168           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1169                     "movl    rax, [rsp + #%d]\n\t"
 1170                     "movl    [rsp + #%d], rax\n\t"
 1171                     "movq    rax, [rsp - #8]",
 1172                      src_offset, dst_offset);
 1173 #endif
 1174         }
 1175       }
 1176       return 0;
 1177     } else if (dst_first_rc == rc_int) {
 1178       // mem -> gpr
 1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1181         // 64-bit
 1182         int offset = ra_->reg2offset(src_first);
 1183         if (masm) {
 1184           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1185 #ifndef PRODUCT
 1186         } else {
 1187           st->print("movq    %s, [rsp + #%d]\t# spill",
 1188                      Matcher::regName[dst_first],
 1189                      offset);
 1190 #endif
 1191         }
 1192       } else {
 1193         // 32-bit
 1194         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1195         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1196         int offset = ra_->reg2offset(src_first);
 1197         if (masm) {
 1198           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1199 #ifndef PRODUCT
 1200         } else {
 1201           st->print("movl    %s, [rsp + #%d]\t# spill",
 1202                      Matcher::regName[dst_first],
 1203                      offset);
 1204 #endif
 1205         }
 1206       }
 1207       return 0;
 1208     } else if (dst_first_rc == rc_float) {
 1209       // mem-> xmm
 1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1212         // 64-bit
 1213         int offset = ra_->reg2offset(src_first);
 1214         if (masm) {
 1215           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1216 #ifndef PRODUCT
 1217         } else {
 1218           st->print("%s  %s, [rsp + #%d]\t# spill",
 1219                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1220                      Matcher::regName[dst_first],
 1221                      offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         int offset = ra_->reg2offset(src_first);
 1229         if (masm) {
 1230           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("movss   %s, [rsp + #%d]\t# spill",
 1234                      Matcher::regName[dst_first],
 1235                      offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_kreg) {
 1241       // mem -> kreg
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (masm) {
 1247           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1248 #ifndef PRODUCT
 1249         } else {
 1250           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1251                      Matcher::regName[dst_first],
 1252                      offset);
 1253 #endif
 1254         }
 1255       }
 1256       return 0;
 1257     }
 1258   } else if (src_first_rc == rc_int) {
 1259     // gpr ->
 1260     if (dst_first_rc == rc_stack) {
 1261       // gpr -> mem
 1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1264         // 64-bit
 1265         int offset = ra_->reg2offset(dst_first);
 1266         if (masm) {
 1267           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1268 #ifndef PRODUCT
 1269         } else {
 1270           st->print("movq    [rsp + #%d], %s\t# spill",
 1271                      offset,
 1272                      Matcher::regName[src_first]);
 1273 #endif
 1274         }
 1275       } else {
 1276         // 32-bit
 1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1279         int offset = ra_->reg2offset(dst_first);
 1280         if (masm) {
 1281           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("movl    [rsp + #%d], %s\t# spill",
 1285                      offset,
 1286                      Matcher::regName[src_first]);
 1287 #endif
 1288         }
 1289       }
 1290       return 0;
 1291     } else if (dst_first_rc == rc_int) {
 1292       // gpr -> gpr
 1293       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1294           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1295         // 64-bit
 1296         if (masm) {
 1297           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1298                   as_Register(Matcher::_regEncode[src_first]));
 1299 #ifndef PRODUCT
 1300         } else {
 1301           st->print("movq    %s, %s\t# spill",
 1302                      Matcher::regName[dst_first],
 1303                      Matcher::regName[src_first]);
 1304 #endif
 1305         }
 1306         return 0;
 1307       } else {
 1308         // 32-bit
 1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1311         if (masm) {
 1312           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1313                   as_Register(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("movl    %s, %s\t# spill",
 1317                      Matcher::regName[dst_first],
 1318                      Matcher::regName[src_first]);
 1319 #endif
 1320         }
 1321         return 0;
 1322       }
 1323     } else if (dst_first_rc == rc_float) {
 1324       // gpr -> xmm
 1325       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1326           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1327         // 64-bit
 1328         if (masm) {
 1329           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1330 #ifndef PRODUCT
 1331         } else {
 1332           st->print("movdq   %s, %s\t# spill",
 1333                      Matcher::regName[dst_first],
 1334                      Matcher::regName[src_first]);
 1335 #endif
 1336         }
 1337       } else {
 1338         // 32-bit
 1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1341         if (masm) {
 1342           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1343 #ifndef PRODUCT
 1344         } else {
 1345           st->print("movdl   %s, %s\t# spill",
 1346                      Matcher::regName[dst_first],
 1347                      Matcher::regName[src_first]);
 1348 #endif
 1349         }
 1350       }
 1351       return 0;
 1352     } else if (dst_first_rc == rc_kreg) {
 1353       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1354           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1355         // 64-bit
 1356         if (masm) {
 1357           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1358   #ifndef PRODUCT
 1359         } else {
 1360            st->print("kmovq   %s, %s\t# spill",
 1361                        Matcher::regName[dst_first],
 1362                        Matcher::regName[src_first]);
 1363   #endif
 1364         }
 1365       }
 1366       Unimplemented();
 1367       return 0;
 1368     }
 1369   } else if (src_first_rc == rc_float) {
 1370     // xmm ->
 1371     if (dst_first_rc == rc_stack) {
 1372       // xmm -> mem
 1373       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1374           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1375         // 64-bit
 1376         int offset = ra_->reg2offset(dst_first);
 1377         if (masm) {
 1378           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1379 #ifndef PRODUCT
 1380         } else {
 1381           st->print("movsd   [rsp + #%d], %s\t# spill",
 1382                      offset,
 1383                      Matcher::regName[src_first]);
 1384 #endif
 1385         }
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         int offset = ra_->reg2offset(dst_first);
 1391         if (masm) {
 1392           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1393 #ifndef PRODUCT
 1394         } else {
 1395           st->print("movss   [rsp + #%d], %s\t# spill",
 1396                      offset,
 1397                      Matcher::regName[src_first]);
 1398 #endif
 1399         }
 1400       }
 1401       return 0;
 1402     } else if (dst_first_rc == rc_int) {
 1403       // xmm -> gpr
 1404       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1405           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1406         // 64-bit
 1407         if (masm) {
 1408           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       } else {
 1417         // 32-bit
 1418         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1419         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1420         if (masm) {
 1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1422 #ifndef PRODUCT
 1423         } else {
 1424           st->print("movdl   %s, %s\t# spill",
 1425                      Matcher::regName[dst_first],
 1426                      Matcher::regName[src_first]);
 1427 #endif
 1428         }
 1429       }
 1430       return 0;
 1431     } else if (dst_first_rc == rc_float) {
 1432       // xmm -> xmm
 1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1435         // 64-bit
 1436         if (masm) {
 1437           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1438 #ifndef PRODUCT
 1439         } else {
 1440           st->print("%s  %s, %s\t# spill",
 1441                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       } else {
 1447         // 32-bit
 1448         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1449         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1450         if (masm) {
 1451           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1452 #ifndef PRODUCT
 1453         } else {
 1454           st->print("%s  %s, %s\t# spill",
 1455                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1456                      Matcher::regName[dst_first],
 1457                      Matcher::regName[src_first]);
 1458 #endif
 1459         }
 1460       }
 1461       return 0;
 1462     } else if (dst_first_rc == rc_kreg) {
 1463       assert(false, "Illegal spilling");
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_kreg) {
 1467     if (dst_first_rc == rc_stack) {
 1468       // mem -> kreg
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         int offset = ra_->reg2offset(dst_first);
 1473         if (masm) {
 1474           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1478                      offset,
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       }
 1483       return 0;
 1484     } else if (dst_first_rc == rc_int) {
 1485       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1486           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1487         // 64-bit
 1488         if (masm) {
 1489           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1490 #ifndef PRODUCT
 1491         } else {
 1492          st->print("kmovq   %s, %s\t# spill",
 1493                      Matcher::regName[dst_first],
 1494                      Matcher::regName[src_first]);
 1495 #endif
 1496         }
 1497       }
 1498       Unimplemented();
 1499       return 0;
 1500     } else if (dst_first_rc == rc_kreg) {
 1501       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1502           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1503         // 64-bit
 1504         if (masm) {
 1505           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508          st->print("kmovq   %s, %s\t# spill",
 1509                      Matcher::regName[dst_first],
 1510                      Matcher::regName[src_first]);
 1511 #endif
 1512         }
 1513       }
 1514       return 0;
 1515     } else if (dst_first_rc == rc_float) {
 1516       assert(false, "Illegal spill");
 1517       return 0;
 1518     }
 1519   }
 1520 
 1521   assert(0," foo ");
 1522   Unimplemented();
 1523   return 0;
 1524 }
 1525 
 1526 #ifndef PRODUCT
 1527 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1528   implementation(nullptr, ra_, false, st);
 1529 }
 1530 #endif
 1531 
 1532 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1533   implementation(masm, ra_, false, nullptr);
 1534 }
 1535 
 1536 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1537   return MachNode::size(ra_);
 1538 }
 1539 
 1540 //=============================================================================
 1541 #ifndef PRODUCT
 1542 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1543 {
 1544   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1545   int reg = ra_->get_reg_first(this);
 1546   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1547             Matcher::regName[reg], offset);
 1548 }
 1549 #endif
 1550 
 1551 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1552 {
 1553   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1554   int reg = ra_->get_encode(this);
 1555 
 1556   __ lea(as_Register(reg), Address(rsp, offset));
 1557 }
 1558 
 1559 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1560 {
 1561   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1562   if (ra_->get_encode(this) > 15) {
 1563     return (offset < 0x80) ? 6 : 9; // REX2
 1564   } else {
 1565     return (offset < 0x80) ? 5 : 8; // REX
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1590 {
 1591   return MachNode::size(ra_); // too many variables; just compute it
 1592                               // the hard way
 1593 }
 1594 
 1595 
 1596 //=============================================================================
 1597 
 1598 bool Matcher::supports_vector_calling_convention(void) {
 1599   return EnableVectorSupport;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     DEBUG_ONLY(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     DEBUG_ONLY(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     // No relocation needed
 1863     __ mov64(r10, (int64_t) $meth$$method);
 1864     __ call(r10);
 1865     __ post_call_nop();
 1866   %}
 1867 
 1868   enc_class Java_Static_Call(method meth)
 1869   %{
 1870     // JAVA STATIC CALL
 1871     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1872     // determine who we intended to call.
 1873     if (!_method) {
 1874       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1875     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1876       // The NOP here is purely to ensure that eliding a call to
 1877       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1878       __ addr_nop_5();
 1879       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1880     } else {
 1881       int method_index = resolved_method_index(masm);
 1882       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1883                                                   : static_call_Relocation::spec(method_index);
 1884       address mark = __ pc();
 1885       int call_offset = __ offset();
 1886       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1887       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1888         // Calls of the same statically bound method can share
 1889         // a stub to the interpreter.
 1890         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1891       } else {
 1892         // Emit stubs for static call.
 1893         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1894         __ clear_inst_mark();
 1895         if (stub == nullptr) {
 1896           ciEnv::current()->record_failure("CodeCache is full");
 1897           return;
 1898         }
 1899       }
 1900     }
 1901     __ post_call_nop();
 1902   %}
 1903 
 1904   enc_class Java_Dynamic_Call(method meth) %{
 1905     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1906     __ post_call_nop();
 1907   %}
 1908 
 1909 %}
 1910 
 1911 
 1912 
 1913 //----------FRAME--------------------------------------------------------------
 1914 // Definition of frame structure and management information.
 1915 //
 1916 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1917 //                             |   (to get allocators register number
 1918 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1919 //  r   CALLER     |        |
 1920 //  o     |        +--------+      pad to even-align allocators stack-slot
 1921 //  w     V        |  pad0  |        numbers; owned by CALLER
 1922 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1923 //  h     ^        |   in   |  5
 1924 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1925 //  |     |        |        |  3
 1926 //  |     |        +--------+
 1927 //  V     |        | old out|      Empty on Intel, window on Sparc
 1928 //        |    old |preserve|      Must be even aligned.
 1929 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1930 //        |        |   in   |  3   area for Intel ret address
 1931 //     Owned by    |preserve|      Empty on Sparc.
 1932 //       SELF      +--------+
 1933 //        |        |  pad2  |  2   pad to align old SP
 1934 //        |        +--------+  1
 1935 //        |        | locks  |  0
 1936 //        |        +--------+----> OptoReg::stack0(), even aligned
 1937 //        |        |  pad1  | 11   pad to align new SP
 1938 //        |        +--------+
 1939 //        |        |        | 10
 1940 //        |        | spills |  9   spills
 1941 //        V        |        |  8   (pad0 slot for callee)
 1942 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1943 //        ^        |  out   |  7
 1944 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1945 //     Owned by    +--------+
 1946 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1947 //        |    new |preserve|      Must be even-aligned.
 1948 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1949 //        |        |        |
 1950 //
 1951 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1952 //         known from SELF's arguments and the Java calling convention.
 1953 //         Region 6-7 is determined per call site.
 1954 // Note 2: If the calling convention leaves holes in the incoming argument
 1955 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1956 //         are owned by the CALLEE.  Holes should not be necessary in the
 1957 //         incoming area, as the Java calling convention is completely under
 1958 //         the control of the AD file.  Doubles can be sorted and packed to
 1959 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1960 //         varargs C calling conventions.
 1961 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1962 //         even aligned with pad0 as needed.
 1963 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1964 //         region 6-11 is even aligned; it may be padded out more so that
 1965 //         the region from SP to FP meets the minimum stack alignment.
 1966 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1967 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1968 //         SP meets the minimum alignment.
 1969 
 1970 frame
 1971 %{
 1972   // These three registers define part of the calling convention
 1973   // between compiled code and the interpreter.
 1974   inline_cache_reg(RAX);                // Inline Cache Register
 1975 
 1976   // Optional: name the operand used by cisc-spilling to access
 1977   // [stack_pointer + offset]
 1978   cisc_spilling_operand_name(indOffset32);
 1979 
 1980   // Number of stack slots consumed by locking an object
 1981   sync_stack_slots(2);
 1982 
 1983   // Compiled code's Frame Pointer
 1984   frame_pointer(RSP);
 1985 
 1986   // Interpreter stores its frame pointer in a register which is
 1987   // stored to the stack by I2CAdaptors.
 1988   // I2CAdaptors convert from interpreted java to compiled java.
 1989   interpreter_frame_pointer(RBP);
 1990 
 1991   // Stack alignment requirement
 1992   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1993 
 1994   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1995   // for calls to C.  Supports the var-args backing area for register parms.
 1996   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1997 
 1998   // The after-PROLOG location of the return address.  Location of
 1999   // return address specifies a type (REG or STACK) and a number
 2000   // representing the register number (i.e. - use a register name) or
 2001   // stack slot.
 2002   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2003   // Otherwise, it is above the locks and verification slot and alignment word
 2004   return_addr(STACK - 2 +
 2005               align_up((Compile::current()->in_preserve_stack_slots() +
 2006                         Compile::current()->fixed_slots()),
 2007                        stack_alignment_in_slots()));
 2008 
 2009   // Location of compiled Java return values.  Same as C for now.
 2010   return_value
 2011   %{
 2012     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2013            "only return normal values");
 2014 
 2015     static const int lo[Op_RegL + 1] = {
 2016       0,
 2017       0,
 2018       RAX_num,  // Op_RegN
 2019       RAX_num,  // Op_RegI
 2020       RAX_num,  // Op_RegP
 2021       XMM0_num, // Op_RegF
 2022       XMM0_num, // Op_RegD
 2023       RAX_num   // Op_RegL
 2024     };
 2025     static const int hi[Op_RegL + 1] = {
 2026       0,
 2027       0,
 2028       OptoReg::Bad, // Op_RegN
 2029       OptoReg::Bad, // Op_RegI
 2030       RAX_H_num,    // Op_RegP
 2031       OptoReg::Bad, // Op_RegF
 2032       XMM0b_num,    // Op_RegD
 2033       RAX_H_num     // Op_RegL
 2034     };
 2035     // Excluded flags and vector registers.
 2036     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2037     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2038   %}
 2039 %}
 2040 
 2041 //----------ATTRIBUTES---------------------------------------------------------
 2042 //----------Operand Attributes-------------------------------------------------
 2043 op_attrib op_cost(0);        // Required cost attribute
 2044 
 2045 //----------Instruction Attributes---------------------------------------------
 2046 ins_attrib ins_cost(100);       // Required cost attribute
 2047 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2048 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2049                                 // a non-matching short branch variant
 2050                                 // of some long branch?
 2051 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2052                                 // be a power of 2) specifies the
 2053                                 // alignment that some part of the
 2054                                 // instruction (not necessarily the
 2055                                 // start) requires.  If > 1, a
 2056                                 // compute_padding() function must be
 2057                                 // provided for the instruction
 2058 
 2059 //----------OPERANDS-----------------------------------------------------------
 2060 // Operand definitions must precede instruction definitions for correct parsing
 2061 // in the ADLC because operands constitute user defined types which are used in
 2062 // instruction definitions.
 2063 
 2064 //----------Simple Operands----------------------------------------------------
 2065 // Immediate Operands
 2066 // Integer Immediate
 2067 operand immI()
 2068 %{
 2069   match(ConI);
 2070 
 2071   op_cost(10);
 2072   format %{ %}
 2073   interface(CONST_INTER);
 2074 %}
 2075 
 2076 // Constant for test vs zero
 2077 operand immI_0()
 2078 %{
 2079   predicate(n->get_int() == 0);
 2080   match(ConI);
 2081 
 2082   op_cost(0);
 2083   format %{ %}
 2084   interface(CONST_INTER);
 2085 %}
 2086 
 2087 // Constant for increment
 2088 operand immI_1()
 2089 %{
 2090   predicate(n->get_int() == 1);
 2091   match(ConI);
 2092 
 2093   op_cost(0);
 2094   format %{ %}
 2095   interface(CONST_INTER);
 2096 %}
 2097 
 2098 // Constant for decrement
 2099 operand immI_M1()
 2100 %{
 2101   predicate(n->get_int() == -1);
 2102   match(ConI);
 2103 
 2104   op_cost(0);
 2105   format %{ %}
 2106   interface(CONST_INTER);
 2107 %}
 2108 
 2109 operand immI_2()
 2110 %{
 2111   predicate(n->get_int() == 2);
 2112   match(ConI);
 2113 
 2114   op_cost(0);
 2115   format %{ %}
 2116   interface(CONST_INTER);
 2117 %}
 2118 
 2119 operand immI_4()
 2120 %{
 2121   predicate(n->get_int() == 4);
 2122   match(ConI);
 2123 
 2124   op_cost(0);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 operand immI_8()
 2130 %{
 2131   predicate(n->get_int() == 8);
 2132   match(ConI);
 2133 
 2134   op_cost(0);
 2135   format %{ %}
 2136   interface(CONST_INTER);
 2137 %}
 2138 
 2139 // Valid scale values for addressing modes
 2140 operand immI2()
 2141 %{
 2142   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2143   match(ConI);
 2144 
 2145   format %{ %}
 2146   interface(CONST_INTER);
 2147 %}
 2148 
 2149 operand immU7()
 2150 %{
 2151   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2152   match(ConI);
 2153 
 2154   op_cost(5);
 2155   format %{ %}
 2156   interface(CONST_INTER);
 2157 %}
 2158 
 2159 operand immI8()
 2160 %{
 2161   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2162   match(ConI);
 2163 
 2164   op_cost(5);
 2165   format %{ %}
 2166   interface(CONST_INTER);
 2167 %}
 2168 
 2169 operand immU8()
 2170 %{
 2171   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2172   match(ConI);
 2173 
 2174   op_cost(5);
 2175   format %{ %}
 2176   interface(CONST_INTER);
 2177 %}
 2178 
 2179 operand immI16()
 2180 %{
 2181   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2182   match(ConI);
 2183 
 2184   op_cost(10);
 2185   format %{ %}
 2186   interface(CONST_INTER);
 2187 %}
 2188 
 2189 // Int Immediate non-negative
 2190 operand immU31()
 2191 %{
 2192   predicate(n->get_int() >= 0);
 2193   match(ConI);
 2194 
 2195   op_cost(0);
 2196   format %{ %}
 2197   interface(CONST_INTER);
 2198 %}
 2199 
 2200 // Pointer Immediate
 2201 operand immP()
 2202 %{
 2203   match(ConP);
 2204 
 2205   op_cost(10);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 // Null Pointer Immediate
 2211 operand immP0()
 2212 %{
 2213   predicate(n->get_ptr() == 0);
 2214   match(ConP);
 2215 
 2216   op_cost(5);
 2217   format %{ %}
 2218   interface(CONST_INTER);
 2219 %}
 2220 
 2221 // Pointer Immediate
 2222 operand immN() %{
 2223   match(ConN);
 2224 
 2225   op_cost(10);
 2226   format %{ %}
 2227   interface(CONST_INTER);
 2228 %}
 2229 
 2230 operand immNKlass() %{
 2231   match(ConNKlass);
 2232 
 2233   op_cost(10);
 2234   format %{ %}
 2235   interface(CONST_INTER);
 2236 %}
 2237 
 2238 // Null Pointer Immediate
 2239 operand immN0() %{
 2240   predicate(n->get_narrowcon() == 0);
 2241   match(ConN);
 2242 
 2243   op_cost(5);
 2244   format %{ %}
 2245   interface(CONST_INTER);
 2246 %}
 2247 
 2248 operand immP31()
 2249 %{
 2250   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2251             && (n->get_ptr() >> 31) == 0);
 2252   match(ConP);
 2253 
 2254   op_cost(5);
 2255   format %{ %}
 2256   interface(CONST_INTER);
 2257 %}
 2258 
 2259 
 2260 // Long Immediate
 2261 operand immL()
 2262 %{
 2263   match(ConL);
 2264 
 2265   op_cost(20);
 2266   format %{ %}
 2267   interface(CONST_INTER);
 2268 %}
 2269 
 2270 // Long Immediate 8-bit
 2271 operand immL8()
 2272 %{
 2273   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2274   match(ConL);
 2275 
 2276   op_cost(5);
 2277   format %{ %}
 2278   interface(CONST_INTER);
 2279 %}
 2280 
 2281 // Long Immediate 32-bit unsigned
 2282 operand immUL32()
 2283 %{
 2284   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2285   match(ConL);
 2286 
 2287   op_cost(10);
 2288   format %{ %}
 2289   interface(CONST_INTER);
 2290 %}
 2291 
 2292 // Long Immediate 32-bit signed
 2293 operand immL32()
 2294 %{
 2295   predicate(n->get_long() == (int) (n->get_long()));
 2296   match(ConL);
 2297 
 2298   op_cost(15);
 2299   format %{ %}
 2300   interface(CONST_INTER);
 2301 %}
 2302 
 2303 operand immL_Pow2()
 2304 %{
 2305   predicate(is_power_of_2((julong)n->get_long()));
 2306   match(ConL);
 2307 
 2308   op_cost(15);
 2309   format %{ %}
 2310   interface(CONST_INTER);
 2311 %}
 2312 
 2313 operand immL_NotPow2()
 2314 %{
 2315   predicate(is_power_of_2((julong)~n->get_long()));
 2316   match(ConL);
 2317 
 2318   op_cost(15);
 2319   format %{ %}
 2320   interface(CONST_INTER);
 2321 %}
 2322 
 2323 // Long Immediate zero
 2324 operand immL0()
 2325 %{
 2326   predicate(n->get_long() == 0L);
 2327   match(ConL);
 2328 
 2329   op_cost(10);
 2330   format %{ %}
 2331   interface(CONST_INTER);
 2332 %}
 2333 
 2334 // Constant for increment
 2335 operand immL1()
 2336 %{
 2337   predicate(n->get_long() == 1);
 2338   match(ConL);
 2339 
 2340   format %{ %}
 2341   interface(CONST_INTER);
 2342 %}
 2343 
 2344 // Constant for decrement
 2345 operand immL_M1()
 2346 %{
 2347   predicate(n->get_long() == -1);
 2348   match(ConL);
 2349 
 2350   format %{ %}
 2351   interface(CONST_INTER);
 2352 %}
 2353 
 2354 // Long Immediate: low 32-bit mask
 2355 operand immL_32bits()
 2356 %{
 2357   predicate(n->get_long() == 0xFFFFFFFFL);
 2358   match(ConL);
 2359   op_cost(20);
 2360 
 2361   format %{ %}
 2362   interface(CONST_INTER);
 2363 %}
 2364 
 2365 // Int Immediate: 2^n-1, positive
 2366 operand immI_Pow2M1()
 2367 %{
 2368   predicate((n->get_int() > 0)
 2369             && is_power_of_2((juint)n->get_int() + 1));
 2370   match(ConI);
 2371 
 2372   op_cost(20);
 2373   format %{ %}
 2374   interface(CONST_INTER);
 2375 %}
 2376 
 2377 // Float Immediate zero
 2378 operand immF0()
 2379 %{
 2380   predicate(jint_cast(n->getf()) == 0);
 2381   match(ConF);
 2382 
 2383   op_cost(5);
 2384   format %{ %}
 2385   interface(CONST_INTER);
 2386 %}
 2387 
 2388 // Float Immediate
 2389 operand immF()
 2390 %{
 2391   match(ConF);
 2392 
 2393   op_cost(15);
 2394   format %{ %}
 2395   interface(CONST_INTER);
 2396 %}
 2397 
 2398 // Half Float Immediate
 2399 operand immH()
 2400 %{
 2401   match(ConH);
 2402 
 2403   op_cost(15);
 2404   format %{ %}
 2405   interface(CONST_INTER);
 2406 %}
 2407 
 2408 // Double Immediate zero
 2409 operand immD0()
 2410 %{
 2411   predicate(jlong_cast(n->getd()) == 0);
 2412   match(ConD);
 2413 
 2414   op_cost(5);
 2415   format %{ %}
 2416   interface(CONST_INTER);
 2417 %}
 2418 
 2419 // Double Immediate
 2420 operand immD()
 2421 %{
 2422   match(ConD);
 2423 
 2424   op_cost(15);
 2425   format %{ %}
 2426   interface(CONST_INTER);
 2427 %}
 2428 
 2429 // Immediates for special shifts (sign extend)
 2430 
 2431 // Constants for increment
 2432 operand immI_16()
 2433 %{
 2434   predicate(n->get_int() == 16);
 2435   match(ConI);
 2436 
 2437   format %{ %}
 2438   interface(CONST_INTER);
 2439 %}
 2440 
 2441 operand immI_24()
 2442 %{
 2443   predicate(n->get_int() == 24);
 2444   match(ConI);
 2445 
 2446   format %{ %}
 2447   interface(CONST_INTER);
 2448 %}
 2449 
 2450 // Constant for byte-wide masking
 2451 operand immI_255()
 2452 %{
 2453   predicate(n->get_int() == 255);
 2454   match(ConI);
 2455 
 2456   format %{ %}
 2457   interface(CONST_INTER);
 2458 %}
 2459 
 2460 // Constant for short-wide masking
 2461 operand immI_65535()
 2462 %{
 2463   predicate(n->get_int() == 65535);
 2464   match(ConI);
 2465 
 2466   format %{ %}
 2467   interface(CONST_INTER);
 2468 %}
 2469 
 2470 // Constant for byte-wide masking
 2471 operand immL_255()
 2472 %{
 2473   predicate(n->get_long() == 255);
 2474   match(ConL);
 2475 
 2476   format %{ %}
 2477   interface(CONST_INTER);
 2478 %}
 2479 
 2480 // Constant for short-wide masking
 2481 operand immL_65535()
 2482 %{
 2483   predicate(n->get_long() == 65535);
 2484   match(ConL);
 2485 
 2486   format %{ %}
 2487   interface(CONST_INTER);
 2488 %}
 2489 
 2490 operand kReg()
 2491 %{
 2492   constraint(ALLOC_IN_RC(vectmask_reg));
 2493   match(RegVectMask);
 2494   format %{%}
 2495   interface(REG_INTER);
 2496 %}
 2497 
 2498 // Register Operands
 2499 // Integer Register
 2500 operand rRegI()
 2501 %{
 2502   constraint(ALLOC_IN_RC(int_reg));
 2503   match(RegI);
 2504 
 2505   match(rax_RegI);
 2506   match(rbx_RegI);
 2507   match(rcx_RegI);
 2508   match(rdx_RegI);
 2509   match(rdi_RegI);
 2510 
 2511   format %{ %}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 // Special Registers
 2516 operand rax_RegI()
 2517 %{
 2518   constraint(ALLOC_IN_RC(int_rax_reg));
 2519   match(RegI);
 2520   match(rRegI);
 2521 
 2522   format %{ "RAX" %}
 2523   interface(REG_INTER);
 2524 %}
 2525 
 2526 // Special Registers
 2527 operand rbx_RegI()
 2528 %{
 2529   constraint(ALLOC_IN_RC(int_rbx_reg));
 2530   match(RegI);
 2531   match(rRegI);
 2532 
 2533   format %{ "RBX" %}
 2534   interface(REG_INTER);
 2535 %}
 2536 
 2537 operand rcx_RegI()
 2538 %{
 2539   constraint(ALLOC_IN_RC(int_rcx_reg));
 2540   match(RegI);
 2541   match(rRegI);
 2542 
 2543   format %{ "RCX" %}
 2544   interface(REG_INTER);
 2545 %}
 2546 
 2547 operand rdx_RegI()
 2548 %{
 2549   constraint(ALLOC_IN_RC(int_rdx_reg));
 2550   match(RegI);
 2551   match(rRegI);
 2552 
 2553   format %{ "RDX" %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 operand rdi_RegI()
 2558 %{
 2559   constraint(ALLOC_IN_RC(int_rdi_reg));
 2560   match(RegI);
 2561   match(rRegI);
 2562 
 2563   format %{ "RDI" %}
 2564   interface(REG_INTER);
 2565 %}
 2566 
 2567 operand no_rax_rdx_RegI()
 2568 %{
 2569   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2570   match(RegI);
 2571   match(rbx_RegI);
 2572   match(rcx_RegI);
 2573   match(rdi_RegI);
 2574 
 2575   format %{ %}
 2576   interface(REG_INTER);
 2577 %}
 2578 
 2579 operand no_rbp_r13_RegI()
 2580 %{
 2581   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2582   match(RegI);
 2583   match(rRegI);
 2584   match(rax_RegI);
 2585   match(rbx_RegI);
 2586   match(rcx_RegI);
 2587   match(rdx_RegI);
 2588   match(rdi_RegI);
 2589 
 2590   format %{ %}
 2591   interface(REG_INTER);
 2592 %}
 2593 
 2594 // Pointer Register
 2595 operand any_RegP()
 2596 %{
 2597   constraint(ALLOC_IN_RC(any_reg));
 2598   match(RegP);
 2599   match(rax_RegP);
 2600   match(rbx_RegP);
 2601   match(rdi_RegP);
 2602   match(rsi_RegP);
 2603   match(rbp_RegP);
 2604   match(r15_RegP);
 2605   match(rRegP);
 2606 
 2607   format %{ %}
 2608   interface(REG_INTER);
 2609 %}
 2610 
 2611 operand rRegP()
 2612 %{
 2613   constraint(ALLOC_IN_RC(ptr_reg));
 2614   match(RegP);
 2615   match(rax_RegP);
 2616   match(rbx_RegP);
 2617   match(rdi_RegP);
 2618   match(rsi_RegP);
 2619   match(rbp_RegP);  // See Q&A below about
 2620   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2621 
 2622   format %{ %}
 2623   interface(REG_INTER);
 2624 %}
 2625 
 2626 operand rRegN() %{
 2627   constraint(ALLOC_IN_RC(int_reg));
 2628   match(RegN);
 2629 
 2630   format %{ %}
 2631   interface(REG_INTER);
 2632 %}
 2633 
 2634 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2635 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2636 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2637 // The output of an instruction is controlled by the allocator, which respects
 2638 // register class masks, not match rules.  Unless an instruction mentions
 2639 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2640 // by the allocator as an input.
 2641 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2642 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2643 // result, RBP is not included in the output of the instruction either.
 2644 
 2645 // This operand is not allowed to use RBP even if
 2646 // RBP is not used to hold the frame pointer.
 2647 operand no_rbp_RegP()
 2648 %{
 2649   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2650   match(RegP);
 2651   match(rbx_RegP);
 2652   match(rsi_RegP);
 2653   match(rdi_RegP);
 2654 
 2655   format %{ %}
 2656   interface(REG_INTER);
 2657 %}
 2658 
 2659 // Special Registers
 2660 // Return a pointer value
 2661 operand rax_RegP()
 2662 %{
 2663   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2664   match(RegP);
 2665   match(rRegP);
 2666 
 2667   format %{ %}
 2668   interface(REG_INTER);
 2669 %}
 2670 
 2671 // Special Registers
 2672 // Return a compressed pointer value
 2673 operand rax_RegN()
 2674 %{
 2675   constraint(ALLOC_IN_RC(int_rax_reg));
 2676   match(RegN);
 2677   match(rRegN);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 // Used in AtomicAdd
 2684 operand rbx_RegP()
 2685 %{
 2686   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2687   match(RegP);
 2688   match(rRegP);
 2689 
 2690   format %{ %}
 2691   interface(REG_INTER);
 2692 %}
 2693 
 2694 operand rsi_RegP()
 2695 %{
 2696   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2697   match(RegP);
 2698   match(rRegP);
 2699 
 2700   format %{ %}
 2701   interface(REG_INTER);
 2702 %}
 2703 
 2704 operand rbp_RegP()
 2705 %{
 2706   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2707   match(RegP);
 2708   match(rRegP);
 2709 
 2710   format %{ %}
 2711   interface(REG_INTER);
 2712 %}
 2713 
 2714 // Used in rep stosq
 2715 operand rdi_RegP()
 2716 %{
 2717   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2718   match(RegP);
 2719   match(rRegP);
 2720 
 2721   format %{ %}
 2722   interface(REG_INTER);
 2723 %}
 2724 
 2725 operand r15_RegP()
 2726 %{
 2727   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2728   match(RegP);
 2729   match(rRegP);
 2730 
 2731   format %{ %}
 2732   interface(REG_INTER);
 2733 %}
 2734 
 2735 operand rRegL()
 2736 %{
 2737   constraint(ALLOC_IN_RC(long_reg));
 2738   match(RegL);
 2739   match(rax_RegL);
 2740   match(rdx_RegL);
 2741 
 2742   format %{ %}
 2743   interface(REG_INTER);
 2744 %}
 2745 
 2746 // Special Registers
 2747 operand no_rax_rdx_RegL()
 2748 %{
 2749   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2750   match(RegL);
 2751   match(rRegL);
 2752 
 2753   format %{ %}
 2754   interface(REG_INTER);
 2755 %}
 2756 
 2757 operand rax_RegL()
 2758 %{
 2759   constraint(ALLOC_IN_RC(long_rax_reg));
 2760   match(RegL);
 2761   match(rRegL);
 2762 
 2763   format %{ "RAX" %}
 2764   interface(REG_INTER);
 2765 %}
 2766 
 2767 operand rcx_RegL()
 2768 %{
 2769   constraint(ALLOC_IN_RC(long_rcx_reg));
 2770   match(RegL);
 2771   match(rRegL);
 2772 
 2773   format %{ %}
 2774   interface(REG_INTER);
 2775 %}
 2776 
 2777 operand rdx_RegL()
 2778 %{
 2779   constraint(ALLOC_IN_RC(long_rdx_reg));
 2780   match(RegL);
 2781   match(rRegL);
 2782 
 2783   format %{ %}
 2784   interface(REG_INTER);
 2785 %}
 2786 
 2787 operand r11_RegL()
 2788 %{
 2789   constraint(ALLOC_IN_RC(long_r11_reg));
 2790   match(RegL);
 2791   match(rRegL);
 2792 
 2793   format %{ %}
 2794   interface(REG_INTER);
 2795 %}
 2796 
 2797 operand no_rbp_r13_RegL()
 2798 %{
 2799   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2800   match(RegL);
 2801   match(rRegL);
 2802   match(rax_RegL);
 2803   match(rcx_RegL);
 2804   match(rdx_RegL);
 2805 
 2806   format %{ %}
 2807   interface(REG_INTER);
 2808 %}
 2809 
 2810 // Flags register, used as output of compare instructions
 2811 operand rFlagsReg()
 2812 %{
 2813   constraint(ALLOC_IN_RC(int_flags));
 2814   match(RegFlags);
 2815 
 2816   format %{ "RFLAGS" %}
 2817   interface(REG_INTER);
 2818 %}
 2819 
 2820 // Flags register, used as output of FLOATING POINT compare instructions
 2821 operand rFlagsRegU()
 2822 %{
 2823   constraint(ALLOC_IN_RC(int_flags));
 2824   match(RegFlags);
 2825 
 2826   format %{ "RFLAGS_U" %}
 2827   interface(REG_INTER);
 2828 %}
 2829 
 2830 operand rFlagsRegUCF() %{
 2831   constraint(ALLOC_IN_RC(int_flags));
 2832   match(RegFlags);
 2833   predicate(false);
 2834 
 2835   format %{ "RFLAGS_U_CF" %}
 2836   interface(REG_INTER);
 2837 %}
 2838 
 2839 // Float register operands
 2840 operand regF() %{
 2841    constraint(ALLOC_IN_RC(float_reg));
 2842    match(RegF);
 2843 
 2844    format %{ %}
 2845    interface(REG_INTER);
 2846 %}
 2847 
 2848 // Float register operands
 2849 operand legRegF() %{
 2850    constraint(ALLOC_IN_RC(float_reg_legacy));
 2851    match(RegF);
 2852 
 2853    format %{ %}
 2854    interface(REG_INTER);
 2855 %}
 2856 
 2857 // Float register operands
 2858 operand vlRegF() %{
 2859    constraint(ALLOC_IN_RC(float_reg_vl));
 2860    match(RegF);
 2861 
 2862    format %{ %}
 2863    interface(REG_INTER);
 2864 %}
 2865 
 2866 // Double register operands
 2867 operand regD() %{
 2868    constraint(ALLOC_IN_RC(double_reg));
 2869    match(RegD);
 2870 
 2871    format %{ %}
 2872    interface(REG_INTER);
 2873 %}
 2874 
 2875 // Double register operands
 2876 operand legRegD() %{
 2877    constraint(ALLOC_IN_RC(double_reg_legacy));
 2878    match(RegD);
 2879 
 2880    format %{ %}
 2881    interface(REG_INTER);
 2882 %}
 2883 
 2884 // Double register operands
 2885 operand vlRegD() %{
 2886    constraint(ALLOC_IN_RC(double_reg_vl));
 2887    match(RegD);
 2888 
 2889    format %{ %}
 2890    interface(REG_INTER);
 2891 %}
 2892 
 2893 //----------Memory Operands----------------------------------------------------
 2894 // Direct Memory Operand
 2895 // operand direct(immP addr)
 2896 // %{
 2897 //   match(addr);
 2898 
 2899 //   format %{ "[$addr]" %}
 2900 //   interface(MEMORY_INTER) %{
 2901 //     base(0xFFFFFFFF);
 2902 //     index(0x4);
 2903 //     scale(0x0);
 2904 //     disp($addr);
 2905 //   %}
 2906 // %}
 2907 
 2908 // Indirect Memory Operand
 2909 operand indirect(any_RegP reg)
 2910 %{
 2911   constraint(ALLOC_IN_RC(ptr_reg));
 2912   match(reg);
 2913 
 2914   format %{ "[$reg]" %}
 2915   interface(MEMORY_INTER) %{
 2916     base($reg);
 2917     index(0x4);
 2918     scale(0x0);
 2919     disp(0x0);
 2920   %}
 2921 %}
 2922 
 2923 // Indirect Memory Plus Short Offset Operand
 2924 operand indOffset8(any_RegP reg, immL8 off)
 2925 %{
 2926   constraint(ALLOC_IN_RC(ptr_reg));
 2927   match(AddP reg off);
 2928 
 2929   format %{ "[$reg + $off (8-bit)]" %}
 2930   interface(MEMORY_INTER) %{
 2931     base($reg);
 2932     index(0x4);
 2933     scale(0x0);
 2934     disp($off);
 2935   %}
 2936 %}
 2937 
 2938 // Indirect Memory Plus Long Offset Operand
 2939 operand indOffset32(any_RegP reg, immL32 off)
 2940 %{
 2941   constraint(ALLOC_IN_RC(ptr_reg));
 2942   match(AddP reg off);
 2943 
 2944   format %{ "[$reg + $off (32-bit)]" %}
 2945   interface(MEMORY_INTER) %{
 2946     base($reg);
 2947     index(0x4);
 2948     scale(0x0);
 2949     disp($off);
 2950   %}
 2951 %}
 2952 
 2953 // Indirect Memory Plus Index Register Plus Offset Operand
 2954 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2955 %{
 2956   constraint(ALLOC_IN_RC(ptr_reg));
 2957   match(AddP (AddP reg lreg) off);
 2958 
 2959   op_cost(10);
 2960   format %{"[$reg + $off + $lreg]" %}
 2961   interface(MEMORY_INTER) %{
 2962     base($reg);
 2963     index($lreg);
 2964     scale(0x0);
 2965     disp($off);
 2966   %}
 2967 %}
 2968 
 2969 // Indirect Memory Plus Index Register Plus Offset Operand
 2970 operand indIndex(any_RegP reg, rRegL lreg)
 2971 %{
 2972   constraint(ALLOC_IN_RC(ptr_reg));
 2973   match(AddP reg lreg);
 2974 
 2975   op_cost(10);
 2976   format %{"[$reg + $lreg]" %}
 2977   interface(MEMORY_INTER) %{
 2978     base($reg);
 2979     index($lreg);
 2980     scale(0x0);
 2981     disp(0x0);
 2982   %}
 2983 %}
 2984 
 2985 // Indirect Memory Times Scale Plus Index Register
 2986 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2987 %{
 2988   constraint(ALLOC_IN_RC(ptr_reg));
 2989   match(AddP reg (LShiftL lreg scale));
 2990 
 2991   op_cost(10);
 2992   format %{"[$reg + $lreg << $scale]" %}
 2993   interface(MEMORY_INTER) %{
 2994     base($reg);
 2995     index($lreg);
 2996     scale($scale);
 2997     disp(0x0);
 2998   %}
 2999 %}
 3000 
 3001 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3002 %{
 3003   constraint(ALLOC_IN_RC(ptr_reg));
 3004   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3005   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3006 
 3007   op_cost(10);
 3008   format %{"[$reg + pos $idx << $scale]" %}
 3009   interface(MEMORY_INTER) %{
 3010     base($reg);
 3011     index($idx);
 3012     scale($scale);
 3013     disp(0x0);
 3014   %}
 3015 %}
 3016 
 3017 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3018 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3019 %{
 3020   constraint(ALLOC_IN_RC(ptr_reg));
 3021   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3022 
 3023   op_cost(10);
 3024   format %{"[$reg + $off + $lreg << $scale]" %}
 3025   interface(MEMORY_INTER) %{
 3026     base($reg);
 3027     index($lreg);
 3028     scale($scale);
 3029     disp($off);
 3030   %}
 3031 %}
 3032 
 3033 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3034 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3035 %{
 3036   constraint(ALLOC_IN_RC(ptr_reg));
 3037   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3038   match(AddP (AddP reg (ConvI2L idx)) off);
 3039 
 3040   op_cost(10);
 3041   format %{"[$reg + $off + $idx]" %}
 3042   interface(MEMORY_INTER) %{
 3043     base($reg);
 3044     index($idx);
 3045     scale(0x0);
 3046     disp($off);
 3047   %}
 3048 %}
 3049 
 3050 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3051 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3052 %{
 3053   constraint(ALLOC_IN_RC(ptr_reg));
 3054   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3055   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3056 
 3057   op_cost(10);
 3058   format %{"[$reg + $off + $idx << $scale]" %}
 3059   interface(MEMORY_INTER) %{
 3060     base($reg);
 3061     index($idx);
 3062     scale($scale);
 3063     disp($off);
 3064   %}
 3065 %}
 3066 
 3067 // Indirect Narrow Oop Plus Offset Operand
 3068 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3069 // we can't free r12 even with CompressedOops::base() == nullptr.
 3070 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3071   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3072   constraint(ALLOC_IN_RC(ptr_reg));
 3073   match(AddP (DecodeN reg) off);
 3074 
 3075   op_cost(10);
 3076   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3077   interface(MEMORY_INTER) %{
 3078     base(0xc); // R12
 3079     index($reg);
 3080     scale(0x3);
 3081     disp($off);
 3082   %}
 3083 %}
 3084 
 3085 // Indirect Memory Operand
 3086 operand indirectNarrow(rRegN reg)
 3087 %{
 3088   predicate(CompressedOops::shift() == 0);
 3089   constraint(ALLOC_IN_RC(ptr_reg));
 3090   match(DecodeN reg);
 3091 
 3092   format %{ "[$reg]" %}
 3093   interface(MEMORY_INTER) %{
 3094     base($reg);
 3095     index(0x4);
 3096     scale(0x0);
 3097     disp(0x0);
 3098   %}
 3099 %}
 3100 
 3101 // Indirect Memory Plus Short Offset Operand
 3102 operand indOffset8Narrow(rRegN reg, immL8 off)
 3103 %{
 3104   predicate(CompressedOops::shift() == 0);
 3105   constraint(ALLOC_IN_RC(ptr_reg));
 3106   match(AddP (DecodeN reg) off);
 3107 
 3108   format %{ "[$reg + $off (8-bit)]" %}
 3109   interface(MEMORY_INTER) %{
 3110     base($reg);
 3111     index(0x4);
 3112     scale(0x0);
 3113     disp($off);
 3114   %}
 3115 %}
 3116 
 3117 // Indirect Memory Plus Long Offset Operand
 3118 operand indOffset32Narrow(rRegN reg, immL32 off)
 3119 %{
 3120   predicate(CompressedOops::shift() == 0);
 3121   constraint(ALLOC_IN_RC(ptr_reg));
 3122   match(AddP (DecodeN reg) off);
 3123 
 3124   format %{ "[$reg + $off (32-bit)]" %}
 3125   interface(MEMORY_INTER) %{
 3126     base($reg);
 3127     index(0x4);
 3128     scale(0x0);
 3129     disp($off);
 3130   %}
 3131 %}
 3132 
 3133 // Indirect Memory Plus Index Register Plus Offset Operand
 3134 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3135 %{
 3136   predicate(CompressedOops::shift() == 0);
 3137   constraint(ALLOC_IN_RC(ptr_reg));
 3138   match(AddP (AddP (DecodeN reg) lreg) off);
 3139 
 3140   op_cost(10);
 3141   format %{"[$reg + $off + $lreg]" %}
 3142   interface(MEMORY_INTER) %{
 3143     base($reg);
 3144     index($lreg);
 3145     scale(0x0);
 3146     disp($off);
 3147   %}
 3148 %}
 3149 
 3150 // Indirect Memory Plus Index Register Plus Offset Operand
 3151 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3152 %{
 3153   predicate(CompressedOops::shift() == 0);
 3154   constraint(ALLOC_IN_RC(ptr_reg));
 3155   match(AddP (DecodeN reg) lreg);
 3156 
 3157   op_cost(10);
 3158   format %{"[$reg + $lreg]" %}
 3159   interface(MEMORY_INTER) %{
 3160     base($reg);
 3161     index($lreg);
 3162     scale(0x0);
 3163     disp(0x0);
 3164   %}
 3165 %}
 3166 
 3167 // Indirect Memory Times Scale Plus Index Register
 3168 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3169 %{
 3170   predicate(CompressedOops::shift() == 0);
 3171   constraint(ALLOC_IN_RC(ptr_reg));
 3172   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3173 
 3174   op_cost(10);
 3175   format %{"[$reg + $lreg << $scale]" %}
 3176   interface(MEMORY_INTER) %{
 3177     base($reg);
 3178     index($lreg);
 3179     scale($scale);
 3180     disp(0x0);
 3181   %}
 3182 %}
 3183 
 3184 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3185 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3186 %{
 3187   predicate(CompressedOops::shift() == 0);
 3188   constraint(ALLOC_IN_RC(ptr_reg));
 3189   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3190 
 3191   op_cost(10);
 3192   format %{"[$reg + $off + $lreg << $scale]" %}
 3193   interface(MEMORY_INTER) %{
 3194     base($reg);
 3195     index($lreg);
 3196     scale($scale);
 3197     disp($off);
 3198   %}
 3199 %}
 3200 
 3201 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3202 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3203 %{
 3204   constraint(ALLOC_IN_RC(ptr_reg));
 3205   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3206   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3207 
 3208   op_cost(10);
 3209   format %{"[$reg + $off + $idx]" %}
 3210   interface(MEMORY_INTER) %{
 3211     base($reg);
 3212     index($idx);
 3213     scale(0x0);
 3214     disp($off);
 3215   %}
 3216 %}
 3217 
 3218 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3219 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3220 %{
 3221   constraint(ALLOC_IN_RC(ptr_reg));
 3222   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3223   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3224 
 3225   op_cost(10);
 3226   format %{"[$reg + $off + $idx << $scale]" %}
 3227   interface(MEMORY_INTER) %{
 3228     base($reg);
 3229     index($idx);
 3230     scale($scale);
 3231     disp($off);
 3232   %}
 3233 %}
 3234 
 3235 //----------Special Memory Operands--------------------------------------------
 3236 // Stack Slot Operand - This operand is used for loading and storing temporary
 3237 //                      values on the stack where a match requires a value to
 3238 //                      flow through memory.
 3239 operand stackSlotP(sRegP reg)
 3240 %{
 3241   constraint(ALLOC_IN_RC(stack_slots));
 3242   // No match rule because this operand is only generated in matching
 3243 
 3244   format %{ "[$reg]" %}
 3245   interface(MEMORY_INTER) %{
 3246     base(0x4);   // RSP
 3247     index(0x4);  // No Index
 3248     scale(0x0);  // No Scale
 3249     disp($reg);  // Stack Offset
 3250   %}
 3251 %}
 3252 
 3253 operand stackSlotI(sRegI reg)
 3254 %{
 3255   constraint(ALLOC_IN_RC(stack_slots));
 3256   // No match rule because this operand is only generated in matching
 3257 
 3258   format %{ "[$reg]" %}
 3259   interface(MEMORY_INTER) %{
 3260     base(0x4);   // RSP
 3261     index(0x4);  // No Index
 3262     scale(0x0);  // No Scale
 3263     disp($reg);  // Stack Offset
 3264   %}
 3265 %}
 3266 
 3267 operand stackSlotF(sRegF reg)
 3268 %{
 3269   constraint(ALLOC_IN_RC(stack_slots));
 3270   // No match rule because this operand is only generated in matching
 3271 
 3272   format %{ "[$reg]" %}
 3273   interface(MEMORY_INTER) %{
 3274     base(0x4);   // RSP
 3275     index(0x4);  // No Index
 3276     scale(0x0);  // No Scale
 3277     disp($reg);  // Stack Offset
 3278   %}
 3279 %}
 3280 
 3281 operand stackSlotD(sRegD reg)
 3282 %{
 3283   constraint(ALLOC_IN_RC(stack_slots));
 3284   // No match rule because this operand is only generated in matching
 3285 
 3286   format %{ "[$reg]" %}
 3287   interface(MEMORY_INTER) %{
 3288     base(0x4);   // RSP
 3289     index(0x4);  // No Index
 3290     scale(0x0);  // No Scale
 3291     disp($reg);  // Stack Offset
 3292   %}
 3293 %}
 3294 operand stackSlotL(sRegL reg)
 3295 %{
 3296   constraint(ALLOC_IN_RC(stack_slots));
 3297   // No match rule because this operand is only generated in matching
 3298 
 3299   format %{ "[$reg]" %}
 3300   interface(MEMORY_INTER) %{
 3301     base(0x4);   // RSP
 3302     index(0x4);  // No Index
 3303     scale(0x0);  // No Scale
 3304     disp($reg);  // Stack Offset
 3305   %}
 3306 %}
 3307 
 3308 //----------Conditional Branch Operands----------------------------------------
 3309 // Comparison Op  - This is the operation of the comparison, and is limited to
 3310 //                  the following set of codes:
 3311 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3312 //
 3313 // Other attributes of the comparison, such as unsignedness, are specified
 3314 // by the comparison instruction that sets a condition code flags register.
 3315 // That result is represented by a flags operand whose subtype is appropriate
 3316 // to the unsignedness (etc.) of the comparison.
 3317 //
 3318 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3319 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3320 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3321 
 3322 // Comparison Code
 3323 operand cmpOp()
 3324 %{
 3325   match(Bool);
 3326 
 3327   format %{ "" %}
 3328   interface(COND_INTER) %{
 3329     equal(0x4, "e");
 3330     not_equal(0x5, "ne");
 3331     less(0xC, "l");
 3332     greater_equal(0xD, "ge");
 3333     less_equal(0xE, "le");
 3334     greater(0xF, "g");
 3335     overflow(0x0, "o");
 3336     no_overflow(0x1, "no");
 3337   %}
 3338 %}
 3339 
 3340 // Comparison Code, unsigned compare.  Used by FP also, with
 3341 // C2 (unordered) turned into GT or LT already.  The other bits
 3342 // C0 and C3 are turned into Carry & Zero flags.
 3343 operand cmpOpU()
 3344 %{
 3345   match(Bool);
 3346 
 3347   format %{ "" %}
 3348   interface(COND_INTER) %{
 3349     equal(0x4, "e");
 3350     not_equal(0x5, "ne");
 3351     less(0x2, "b");
 3352     greater_equal(0x3, "ae");
 3353     less_equal(0x6, "be");
 3354     greater(0x7, "a");
 3355     overflow(0x0, "o");
 3356     no_overflow(0x1, "no");
 3357   %}
 3358 %}
 3359 
 3360 
 3361 // Floating comparisons that don't require any fixup for the unordered case,
 3362 // If both inputs of the comparison are the same, ZF is always set so we
 3363 // don't need to use cmpOpUCF2 for eq/ne
 3364 operand cmpOpUCF() %{
 3365   match(Bool);
 3366   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3367             n->as_Bool()->_test._test == BoolTest::ge ||
 3368             n->as_Bool()->_test._test == BoolTest::le ||
 3369             n->as_Bool()->_test._test == BoolTest::gt ||
 3370             n->in(1)->in(1) == n->in(1)->in(2));
 3371   format %{ "" %}
 3372   interface(COND_INTER) %{
 3373     equal(0xb, "np");
 3374     not_equal(0xa, "p");
 3375     less(0x2, "b");
 3376     greater_equal(0x3, "ae");
 3377     less_equal(0x6, "be");
 3378     greater(0x7, "a");
 3379     overflow(0x0, "o");
 3380     no_overflow(0x1, "no");
 3381   %}
 3382 %}
 3383 
 3384 
 3385 // Floating comparisons that can be fixed up with extra conditional jumps
 3386 operand cmpOpUCF2() %{
 3387   match(Bool);
 3388   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3389              n->as_Bool()->_test._test == BoolTest::eq) &&
 3390             n->in(1)->in(1) != n->in(1)->in(2));
 3391   format %{ "" %}
 3392   interface(COND_INTER) %{
 3393     equal(0x4, "e");
 3394     not_equal(0x5, "ne");
 3395     less(0x2, "b");
 3396     greater_equal(0x3, "ae");
 3397     less_equal(0x6, "be");
 3398     greater(0x7, "a");
 3399     overflow(0x0, "o");
 3400     no_overflow(0x1, "no");
 3401   %}
 3402 %}
 3403 
 3404 //----------OPERAND CLASSES----------------------------------------------------
 3405 // Operand Classes are groups of operands that are used as to simplify
 3406 // instruction definitions by not requiring the AD writer to specify separate
 3407 // instructions for every form of operand when the instruction accepts
 3408 // multiple operand types with the same basic encoding and format.  The classic
 3409 // case of this is memory operands.
 3410 
 3411 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3412                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3413                indCompressedOopOffset,
 3414                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3415                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3416                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3417 
 3418 //----------PIPELINE-----------------------------------------------------------
 3419 // Rules which define the behavior of the target architectures pipeline.
 3420 pipeline %{
 3421 
 3422 //----------ATTRIBUTES---------------------------------------------------------
 3423 attributes %{
 3424   variable_size_instructions;        // Fixed size instructions
 3425   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3426   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3427   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3428   instruction_fetch_units = 1;       // of 16 bytes
 3429 
 3430   // List of nop instructions
 3431   nops( MachNop );
 3432 %}
 3433 
 3434 //----------RESOURCES----------------------------------------------------------
 3435 // Resources are the functional units available to the machine
 3436 
 3437 // Generic P2/P3 pipeline
 3438 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3439 // 3 instructions decoded per cycle.
 3440 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3441 // 3 ALU op, only ALU0 handles mul instructions.
 3442 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3443            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3444            BR, FPU,
 3445            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3446 
 3447 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3448 // Pipeline Description specifies the stages in the machine's pipeline
 3449 
 3450 // Generic P2/P3 pipeline
 3451 pipe_desc(S0, S1, S2, S3, S4, S5);
 3452 
 3453 //----------PIPELINE CLASSES---------------------------------------------------
 3454 // Pipeline Classes describe the stages in which input and output are
 3455 // referenced by the hardware pipeline.
 3456 
 3457 // Naming convention: ialu or fpu
 3458 // Then: _reg
 3459 // Then: _reg if there is a 2nd register
 3460 // Then: _long if it's a pair of instructions implementing a long
 3461 // Then: _fat if it requires the big decoder
 3462 //   Or: _mem if it requires the big decoder and a memory unit.
 3463 
 3464 // Integer ALU reg operation
 3465 pipe_class ialu_reg(rRegI dst)
 3466 %{
 3467     single_instruction;
 3468     dst    : S4(write);
 3469     dst    : S3(read);
 3470     DECODE : S0;        // any decoder
 3471     ALU    : S3;        // any alu
 3472 %}
 3473 
 3474 // Long ALU reg operation
 3475 pipe_class ialu_reg_long(rRegL dst)
 3476 %{
 3477     instruction_count(2);
 3478     dst    : S4(write);
 3479     dst    : S3(read);
 3480     DECODE : S0(2);     // any 2 decoders
 3481     ALU    : S3(2);     // both alus
 3482 %}
 3483 
 3484 // Integer ALU reg operation using big decoder
 3485 pipe_class ialu_reg_fat(rRegI dst)
 3486 %{
 3487     single_instruction;
 3488     dst    : S4(write);
 3489     dst    : S3(read);
 3490     D0     : S0;        // big decoder only
 3491     ALU    : S3;        // any alu
 3492 %}
 3493 
 3494 // Integer ALU reg-reg operation
 3495 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3496 %{
 3497     single_instruction;
 3498     dst    : S4(write);
 3499     src    : S3(read);
 3500     DECODE : S0;        // any decoder
 3501     ALU    : S3;        // any alu
 3502 %}
 3503 
 3504 // Integer ALU reg-reg operation
 3505 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3506 %{
 3507     single_instruction;
 3508     dst    : S4(write);
 3509     src    : S3(read);
 3510     D0     : S0;        // big decoder only
 3511     ALU    : S3;        // any alu
 3512 %}
 3513 
 3514 // Integer ALU reg-mem operation
 3515 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3516 %{
 3517     single_instruction;
 3518     dst    : S5(write);
 3519     mem    : S3(read);
 3520     D0     : S0;        // big decoder only
 3521     ALU    : S4;        // any alu
 3522     MEM    : S3;        // any mem
 3523 %}
 3524 
 3525 // Integer mem operation (prefetch)
 3526 pipe_class ialu_mem(memory mem)
 3527 %{
 3528     single_instruction;
 3529     mem    : S3(read);
 3530     D0     : S0;        // big decoder only
 3531     MEM    : S3;        // any mem
 3532 %}
 3533 
 3534 // Integer Store to Memory
 3535 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3536 %{
 3537     single_instruction;
 3538     mem    : S3(read);
 3539     src    : S5(read);
 3540     D0     : S0;        // big decoder only
 3541     ALU    : S4;        // any alu
 3542     MEM    : S3;
 3543 %}
 3544 
 3545 // // Long Store to Memory
 3546 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3547 // %{
 3548 //     instruction_count(2);
 3549 //     mem    : S3(read);
 3550 //     src    : S5(read);
 3551 //     D0     : S0(2);          // big decoder only; twice
 3552 //     ALU    : S4(2);     // any 2 alus
 3553 //     MEM    : S3(2);  // Both mems
 3554 // %}
 3555 
 3556 // Integer Store to Memory
 3557 pipe_class ialu_mem_imm(memory mem)
 3558 %{
 3559     single_instruction;
 3560     mem    : S3(read);
 3561     D0     : S0;        // big decoder only
 3562     ALU    : S4;        // any alu
 3563     MEM    : S3;
 3564 %}
 3565 
 3566 // Integer ALU0 reg-reg operation
 3567 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3568 %{
 3569     single_instruction;
 3570     dst    : S4(write);
 3571     src    : S3(read);
 3572     D0     : S0;        // Big decoder only
 3573     ALU0   : S3;        // only alu0
 3574 %}
 3575 
 3576 // Integer ALU0 reg-mem operation
 3577 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3578 %{
 3579     single_instruction;
 3580     dst    : S5(write);
 3581     mem    : S3(read);
 3582     D0     : S0;        // big decoder only
 3583     ALU0   : S4;        // ALU0 only
 3584     MEM    : S3;        // any mem
 3585 %}
 3586 
 3587 // Integer ALU reg-reg operation
 3588 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3589 %{
 3590     single_instruction;
 3591     cr     : S4(write);
 3592     src1   : S3(read);
 3593     src2   : S3(read);
 3594     DECODE : S0;        // any decoder
 3595     ALU    : S3;        // any alu
 3596 %}
 3597 
 3598 // Integer ALU reg-imm operation
 3599 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3600 %{
 3601     single_instruction;
 3602     cr     : S4(write);
 3603     src1   : S3(read);
 3604     DECODE : S0;        // any decoder
 3605     ALU    : S3;        // any alu
 3606 %}
 3607 
 3608 // Integer ALU reg-mem operation
 3609 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3610 %{
 3611     single_instruction;
 3612     cr     : S4(write);
 3613     src1   : S3(read);
 3614     src2   : S3(read);
 3615     D0     : S0;        // big decoder only
 3616     ALU    : S4;        // any alu
 3617     MEM    : S3;
 3618 %}
 3619 
 3620 // Conditional move reg-reg
 3621 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3622 %{
 3623     instruction_count(4);
 3624     y      : S4(read);
 3625     q      : S3(read);
 3626     p      : S3(read);
 3627     DECODE : S0(4);     // any decoder
 3628 %}
 3629 
 3630 // Conditional move reg-reg
 3631 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3632 %{
 3633     single_instruction;
 3634     dst    : S4(write);
 3635     src    : S3(read);
 3636     cr     : S3(read);
 3637     DECODE : S0;        // any decoder
 3638 %}
 3639 
 3640 // Conditional move reg-mem
 3641 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3642 %{
 3643     single_instruction;
 3644     dst    : S4(write);
 3645     src    : S3(read);
 3646     cr     : S3(read);
 3647     DECODE : S0;        // any decoder
 3648     MEM    : S3;
 3649 %}
 3650 
 3651 // Conditional move reg-reg long
 3652 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3653 %{
 3654     single_instruction;
 3655     dst    : S4(write);
 3656     src    : S3(read);
 3657     cr     : S3(read);
 3658     DECODE : S0(2);     // any 2 decoders
 3659 %}
 3660 
 3661 // Float reg-reg operation
 3662 pipe_class fpu_reg(regD dst)
 3663 %{
 3664     instruction_count(2);
 3665     dst    : S3(read);
 3666     DECODE : S0(2);     // any 2 decoders
 3667     FPU    : S3;
 3668 %}
 3669 
 3670 // Float reg-reg operation
 3671 pipe_class fpu_reg_reg(regD dst, regD src)
 3672 %{
 3673     instruction_count(2);
 3674     dst    : S4(write);
 3675     src    : S3(read);
 3676     DECODE : S0(2);     // any 2 decoders
 3677     FPU    : S3;
 3678 %}
 3679 
 3680 // Float reg-reg operation
 3681 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3682 %{
 3683     instruction_count(3);
 3684     dst    : S4(write);
 3685     src1   : S3(read);
 3686     src2   : S3(read);
 3687     DECODE : S0(3);     // any 3 decoders
 3688     FPU    : S3(2);
 3689 %}
 3690 
 3691 // Float reg-reg operation
 3692 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3693 %{
 3694     instruction_count(4);
 3695     dst    : S4(write);
 3696     src1   : S3(read);
 3697     src2   : S3(read);
 3698     src3   : S3(read);
 3699     DECODE : S0(4);     // any 3 decoders
 3700     FPU    : S3(2);
 3701 %}
 3702 
 3703 // Float reg-reg operation
 3704 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3705 %{
 3706     instruction_count(4);
 3707     dst    : S4(write);
 3708     src1   : S3(read);
 3709     src2   : S3(read);
 3710     src3   : S3(read);
 3711     DECODE : S1(3);     // any 3 decoders
 3712     D0     : S0;        // Big decoder only
 3713     FPU    : S3(2);
 3714     MEM    : S3;
 3715 %}
 3716 
 3717 // Float reg-mem operation
 3718 pipe_class fpu_reg_mem(regD dst, memory mem)
 3719 %{
 3720     instruction_count(2);
 3721     dst    : S5(write);
 3722     mem    : S3(read);
 3723     D0     : S0;        // big decoder only
 3724     DECODE : S1;        // any decoder for FPU POP
 3725     FPU    : S4;
 3726     MEM    : S3;        // any mem
 3727 %}
 3728 
 3729 // Float reg-mem operation
 3730 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3731 %{
 3732     instruction_count(3);
 3733     dst    : S5(write);
 3734     src1   : S3(read);
 3735     mem    : S3(read);
 3736     D0     : S0;        // big decoder only
 3737     DECODE : S1(2);     // any decoder for FPU POP
 3738     FPU    : S4;
 3739     MEM    : S3;        // any mem
 3740 %}
 3741 
 3742 // Float mem-reg operation
 3743 pipe_class fpu_mem_reg(memory mem, regD src)
 3744 %{
 3745     instruction_count(2);
 3746     src    : S5(read);
 3747     mem    : S3(read);
 3748     DECODE : S0;        // any decoder for FPU PUSH
 3749     D0     : S1;        // big decoder only
 3750     FPU    : S4;
 3751     MEM    : S3;        // any mem
 3752 %}
 3753 
 3754 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3755 %{
 3756     instruction_count(3);
 3757     src1   : S3(read);
 3758     src2   : S3(read);
 3759     mem    : S3(read);
 3760     DECODE : S0(2);     // any decoder for FPU PUSH
 3761     D0     : S1;        // big decoder only
 3762     FPU    : S4;
 3763     MEM    : S3;        // any mem
 3764 %}
 3765 
 3766 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3767 %{
 3768     instruction_count(3);
 3769     src1   : S3(read);
 3770     src2   : S3(read);
 3771     mem    : S4(read);
 3772     DECODE : S0;        // any decoder for FPU PUSH
 3773     D0     : S0(2);     // big decoder only
 3774     FPU    : S4;
 3775     MEM    : S3(2);     // any mem
 3776 %}
 3777 
 3778 pipe_class fpu_mem_mem(memory dst, memory src1)
 3779 %{
 3780     instruction_count(2);
 3781     src1   : S3(read);
 3782     dst    : S4(read);
 3783     D0     : S0(2);     // big decoder only
 3784     MEM    : S3(2);     // any mem
 3785 %}
 3786 
 3787 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3788 %{
 3789     instruction_count(3);
 3790     src1   : S3(read);
 3791     src2   : S3(read);
 3792     dst    : S4(read);
 3793     D0     : S0(3);     // big decoder only
 3794     FPU    : S4;
 3795     MEM    : S3(3);     // any mem
 3796 %}
 3797 
 3798 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3799 %{
 3800     instruction_count(3);
 3801     src1   : S4(read);
 3802     mem    : S4(read);
 3803     DECODE : S0;        // any decoder for FPU PUSH
 3804     D0     : S0(2);     // big decoder only
 3805     FPU    : S4;
 3806     MEM    : S3(2);     // any mem
 3807 %}
 3808 
 3809 // Float load constant
 3810 pipe_class fpu_reg_con(regD dst)
 3811 %{
 3812     instruction_count(2);
 3813     dst    : S5(write);
 3814     D0     : S0;        // big decoder only for the load
 3815     DECODE : S1;        // any decoder for FPU POP
 3816     FPU    : S4;
 3817     MEM    : S3;        // any mem
 3818 %}
 3819 
 3820 // Float load constant
 3821 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3822 %{
 3823     instruction_count(3);
 3824     dst    : S5(write);
 3825     src    : S3(read);
 3826     D0     : S0;        // big decoder only for the load
 3827     DECODE : S1(2);     // any decoder for FPU POP
 3828     FPU    : S4;
 3829     MEM    : S3;        // any mem
 3830 %}
 3831 
 3832 // UnConditional branch
 3833 pipe_class pipe_jmp(label labl)
 3834 %{
 3835     single_instruction;
 3836     BR   : S3;
 3837 %}
 3838 
 3839 // Conditional branch
 3840 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3841 %{
 3842     single_instruction;
 3843     cr    : S1(read);
 3844     BR    : S3;
 3845 %}
 3846 
 3847 // Allocation idiom
 3848 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3849 %{
 3850     instruction_count(1); force_serialization;
 3851     fixed_latency(6);
 3852     heap_ptr : S3(read);
 3853     DECODE   : S0(3);
 3854     D0       : S2;
 3855     MEM      : S3;
 3856     ALU      : S3(2);
 3857     dst      : S5(write);
 3858     BR       : S5;
 3859 %}
 3860 
 3861 // Generic big/slow expanded idiom
 3862 pipe_class pipe_slow()
 3863 %{
 3864     instruction_count(10); multiple_bundles; force_serialization;
 3865     fixed_latency(100);
 3866     D0  : S0(2);
 3867     MEM : S3(2);
 3868 %}
 3869 
 3870 // The real do-nothing guy
 3871 pipe_class empty()
 3872 %{
 3873     instruction_count(0);
 3874 %}
 3875 
 3876 // Define the class for the Nop node
 3877 define
 3878 %{
 3879    MachNop = empty;
 3880 %}
 3881 
 3882 %}
 3883 
 3884 //----------INSTRUCTIONS-------------------------------------------------------
 3885 //
 3886 // match      -- States which machine-independent subtree may be replaced
 3887 //               by this instruction.
 3888 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3889 //               selection to identify a minimum cost tree of machine
 3890 //               instructions that matches a tree of machine-independent
 3891 //               instructions.
 3892 // format     -- A string providing the disassembly for this instruction.
 3893 //               The value of an instruction's operand may be inserted
 3894 //               by referring to it with a '$' prefix.
 3895 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3896 //               to within an encode class as $primary, $secondary, and $tertiary
 3897 //               rrspectively.  The primary opcode is commonly used to
 3898 //               indicate the type of machine instruction, while secondary
 3899 //               and tertiary are often used for prefix options or addressing
 3900 //               modes.
 3901 // ins_encode -- A list of encode classes with parameters. The encode class
 3902 //               name must have been defined in an 'enc_class' specification
 3903 //               in the encode section of the architecture description.
 3904 
 3905 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3906 // Load Float
 3907 instruct MoveF2VL(vlRegF dst, regF src) %{
 3908   match(Set dst src);
 3909   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3910   ins_encode %{
 3911     ShouldNotReachHere();
 3912   %}
 3913   ins_pipe( fpu_reg_reg );
 3914 %}
 3915 
 3916 // Load Float
 3917 instruct MoveF2LEG(legRegF dst, regF src) %{
 3918   match(Set dst src);
 3919   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3920   ins_encode %{
 3921     ShouldNotReachHere();
 3922   %}
 3923   ins_pipe( fpu_reg_reg );
 3924 %}
 3925 
 3926 // Load Float
 3927 instruct MoveVL2F(regF dst, vlRegF src) %{
 3928   match(Set dst src);
 3929   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3930   ins_encode %{
 3931     ShouldNotReachHere();
 3932   %}
 3933   ins_pipe( fpu_reg_reg );
 3934 %}
 3935 
 3936 // Load Float
 3937 instruct MoveLEG2F(regF dst, legRegF src) %{
 3938   match(Set dst src);
 3939   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3940   ins_encode %{
 3941     ShouldNotReachHere();
 3942   %}
 3943   ins_pipe( fpu_reg_reg );
 3944 %}
 3945 
 3946 // Load Double
 3947 instruct MoveD2VL(vlRegD dst, regD src) %{
 3948   match(Set dst src);
 3949   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3950   ins_encode %{
 3951     ShouldNotReachHere();
 3952   %}
 3953   ins_pipe( fpu_reg_reg );
 3954 %}
 3955 
 3956 // Load Double
 3957 instruct MoveD2LEG(legRegD dst, regD src) %{
 3958   match(Set dst src);
 3959   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3960   ins_encode %{
 3961     ShouldNotReachHere();
 3962   %}
 3963   ins_pipe( fpu_reg_reg );
 3964 %}
 3965 
 3966 // Load Double
 3967 instruct MoveVL2D(regD dst, vlRegD src) %{
 3968   match(Set dst src);
 3969   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3970   ins_encode %{
 3971     ShouldNotReachHere();
 3972   %}
 3973   ins_pipe( fpu_reg_reg );
 3974 %}
 3975 
 3976 // Load Double
 3977 instruct MoveLEG2D(regD dst, legRegD src) %{
 3978   match(Set dst src);
 3979   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3980   ins_encode %{
 3981     ShouldNotReachHere();
 3982   %}
 3983   ins_pipe( fpu_reg_reg );
 3984 %}
 3985 
 3986 //----------Load/Store/Move Instructions---------------------------------------
 3987 //----------Load Instructions--------------------------------------------------
 3988 
 3989 // Load Byte (8 bit signed)
 3990 instruct loadB(rRegI dst, memory mem)
 3991 %{
 3992   match(Set dst (LoadB mem));
 3993 
 3994   ins_cost(125);
 3995   format %{ "movsbl  $dst, $mem\t# byte" %}
 3996 
 3997   ins_encode %{
 3998     __ movsbl($dst$$Register, $mem$$Address);
 3999   %}
 4000 
 4001   ins_pipe(ialu_reg_mem);
 4002 %}
 4003 
 4004 // Load Byte (8 bit signed) into Long Register
 4005 instruct loadB2L(rRegL dst, memory mem)
 4006 %{
 4007   match(Set dst (ConvI2L (LoadB mem)));
 4008 
 4009   ins_cost(125);
 4010   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4011 
 4012   ins_encode %{
 4013     __ movsbq($dst$$Register, $mem$$Address);
 4014   %}
 4015 
 4016   ins_pipe(ialu_reg_mem);
 4017 %}
 4018 
 4019 // Load Unsigned Byte (8 bit UNsigned)
 4020 instruct loadUB(rRegI dst, memory mem)
 4021 %{
 4022   match(Set dst (LoadUB mem));
 4023 
 4024   ins_cost(125);
 4025   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4026 
 4027   ins_encode %{
 4028     __ movzbl($dst$$Register, $mem$$Address);
 4029   %}
 4030 
 4031   ins_pipe(ialu_reg_mem);
 4032 %}
 4033 
 4034 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4035 instruct loadUB2L(rRegL dst, memory mem)
 4036 %{
 4037   match(Set dst (ConvI2L (LoadUB mem)));
 4038 
 4039   ins_cost(125);
 4040   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4041 
 4042   ins_encode %{
 4043     __ movzbq($dst$$Register, $mem$$Address);
 4044   %}
 4045 
 4046   ins_pipe(ialu_reg_mem);
 4047 %}
 4048 
 4049 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4050 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4051   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4052   effect(KILL cr);
 4053 
 4054   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4055             "andl    $dst, right_n_bits($mask, 8)" %}
 4056   ins_encode %{
 4057     Register Rdst = $dst$$Register;
 4058     __ movzbq(Rdst, $mem$$Address);
 4059     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4060   %}
 4061   ins_pipe(ialu_reg_mem);
 4062 %}
 4063 
 4064 // Load Short (16 bit signed)
 4065 instruct loadS(rRegI dst, memory mem)
 4066 %{
 4067   match(Set dst (LoadS mem));
 4068 
 4069   ins_cost(125);
 4070   format %{ "movswl $dst, $mem\t# short" %}
 4071 
 4072   ins_encode %{
 4073     __ movswl($dst$$Register, $mem$$Address);
 4074   %}
 4075 
 4076   ins_pipe(ialu_reg_mem);
 4077 %}
 4078 
 4079 // Load Short (16 bit signed) to Byte (8 bit signed)
 4080 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4081   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4082 
 4083   ins_cost(125);
 4084   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4085   ins_encode %{
 4086     __ movsbl($dst$$Register, $mem$$Address);
 4087   %}
 4088   ins_pipe(ialu_reg_mem);
 4089 %}
 4090 
 4091 // Load Short (16 bit signed) into Long Register
 4092 instruct loadS2L(rRegL dst, memory mem)
 4093 %{
 4094   match(Set dst (ConvI2L (LoadS mem)));
 4095 
 4096   ins_cost(125);
 4097   format %{ "movswq $dst, $mem\t# short -> long" %}
 4098 
 4099   ins_encode %{
 4100     __ movswq($dst$$Register, $mem$$Address);
 4101   %}
 4102 
 4103   ins_pipe(ialu_reg_mem);
 4104 %}
 4105 
 4106 // Load Unsigned Short/Char (16 bit UNsigned)
 4107 instruct loadUS(rRegI dst, memory mem)
 4108 %{
 4109   match(Set dst (LoadUS mem));
 4110 
 4111   ins_cost(125);
 4112   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4113 
 4114   ins_encode %{
 4115     __ movzwl($dst$$Register, $mem$$Address);
 4116   %}
 4117 
 4118   ins_pipe(ialu_reg_mem);
 4119 %}
 4120 
 4121 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4122 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4123   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4124 
 4125   ins_cost(125);
 4126   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4127   ins_encode %{
 4128     __ movsbl($dst$$Register, $mem$$Address);
 4129   %}
 4130   ins_pipe(ialu_reg_mem);
 4131 %}
 4132 
 4133 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4134 instruct loadUS2L(rRegL dst, memory mem)
 4135 %{
 4136   match(Set dst (ConvI2L (LoadUS mem)));
 4137 
 4138   ins_cost(125);
 4139   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4140 
 4141   ins_encode %{
 4142     __ movzwq($dst$$Register, $mem$$Address);
 4143   %}
 4144 
 4145   ins_pipe(ialu_reg_mem);
 4146 %}
 4147 
 4148 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4149 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4150   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4151 
 4152   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4153   ins_encode %{
 4154     __ movzbq($dst$$Register, $mem$$Address);
 4155   %}
 4156   ins_pipe(ialu_reg_mem);
 4157 %}
 4158 
 4159 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4160 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4161   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4162   effect(KILL cr);
 4163 
 4164   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4165             "andl    $dst, right_n_bits($mask, 16)" %}
 4166   ins_encode %{
 4167     Register Rdst = $dst$$Register;
 4168     __ movzwq(Rdst, $mem$$Address);
 4169     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4170   %}
 4171   ins_pipe(ialu_reg_mem);
 4172 %}
 4173 
 4174 // Load Integer
 4175 instruct loadI(rRegI dst, memory mem)
 4176 %{
 4177   match(Set dst (LoadI mem));
 4178 
 4179   ins_cost(125);
 4180   format %{ "movl    $dst, $mem\t# int" %}
 4181 
 4182   ins_encode %{
 4183     __ movl($dst$$Register, $mem$$Address);
 4184   %}
 4185 
 4186   ins_pipe(ialu_reg_mem);
 4187 %}
 4188 
 4189 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4190 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4191   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4192 
 4193   ins_cost(125);
 4194   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4195   ins_encode %{
 4196     __ movsbl($dst$$Register, $mem$$Address);
 4197   %}
 4198   ins_pipe(ialu_reg_mem);
 4199 %}
 4200 
 4201 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4202 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4203   match(Set dst (AndI (LoadI mem) mask));
 4204 
 4205   ins_cost(125);
 4206   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4207   ins_encode %{
 4208     __ movzbl($dst$$Register, $mem$$Address);
 4209   %}
 4210   ins_pipe(ialu_reg_mem);
 4211 %}
 4212 
 4213 // Load Integer (32 bit signed) to Short (16 bit signed)
 4214 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4215   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4216 
 4217   ins_cost(125);
 4218   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4219   ins_encode %{
 4220     __ movswl($dst$$Register, $mem$$Address);
 4221   %}
 4222   ins_pipe(ialu_reg_mem);
 4223 %}
 4224 
 4225 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4226 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4227   match(Set dst (AndI (LoadI mem) mask));
 4228 
 4229   ins_cost(125);
 4230   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4231   ins_encode %{
 4232     __ movzwl($dst$$Register, $mem$$Address);
 4233   %}
 4234   ins_pipe(ialu_reg_mem);
 4235 %}
 4236 
 4237 // Load Integer into Long Register
 4238 instruct loadI2L(rRegL dst, memory mem)
 4239 %{
 4240   match(Set dst (ConvI2L (LoadI mem)));
 4241 
 4242   ins_cost(125);
 4243   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4244 
 4245   ins_encode %{
 4246     __ movslq($dst$$Register, $mem$$Address);
 4247   %}
 4248 
 4249   ins_pipe(ialu_reg_mem);
 4250 %}
 4251 
 4252 // Load Integer with mask 0xFF into Long Register
 4253 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4254   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4255 
 4256   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4257   ins_encode %{
 4258     __ movzbq($dst$$Register, $mem$$Address);
 4259   %}
 4260   ins_pipe(ialu_reg_mem);
 4261 %}
 4262 
 4263 // Load Integer with mask 0xFFFF into Long Register
 4264 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4265   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4266 
 4267   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4268   ins_encode %{
 4269     __ movzwq($dst$$Register, $mem$$Address);
 4270   %}
 4271   ins_pipe(ialu_reg_mem);
 4272 %}
 4273 
 4274 // Load Integer with a 31-bit mask into Long Register
 4275 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4276   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4277   effect(KILL cr);
 4278 
 4279   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4280             "andl    $dst, $mask" %}
 4281   ins_encode %{
 4282     Register Rdst = $dst$$Register;
 4283     __ movl(Rdst, $mem$$Address);
 4284     __ andl(Rdst, $mask$$constant);
 4285   %}
 4286   ins_pipe(ialu_reg_mem);
 4287 %}
 4288 
 4289 // Load Unsigned Integer into Long Register
 4290 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4291 %{
 4292   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4293 
 4294   ins_cost(125);
 4295   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4296 
 4297   ins_encode %{
 4298     __ movl($dst$$Register, $mem$$Address);
 4299   %}
 4300 
 4301   ins_pipe(ialu_reg_mem);
 4302 %}
 4303 
 4304 // Load Long
 4305 instruct loadL(rRegL dst, memory mem)
 4306 %{
 4307   match(Set dst (LoadL mem));
 4308 
 4309   ins_cost(125);
 4310   format %{ "movq    $dst, $mem\t# long" %}
 4311 
 4312   ins_encode %{
 4313     __ movq($dst$$Register, $mem$$Address);
 4314   %}
 4315 
 4316   ins_pipe(ialu_reg_mem); // XXX
 4317 %}
 4318 
 4319 // Load Range
 4320 instruct loadRange(rRegI dst, memory mem)
 4321 %{
 4322   match(Set dst (LoadRange mem));
 4323 
 4324   ins_cost(125); // XXX
 4325   format %{ "movl    $dst, $mem\t# range" %}
 4326   ins_encode %{
 4327     __ movl($dst$$Register, $mem$$Address);
 4328   %}
 4329   ins_pipe(ialu_reg_mem);
 4330 %}
 4331 
 4332 // Load Pointer
 4333 instruct loadP(rRegP dst, memory mem)
 4334 %{
 4335   match(Set dst (LoadP mem));
 4336   predicate(n->as_Load()->barrier_data() == 0);
 4337 
 4338   ins_cost(125); // XXX
 4339   format %{ "movq    $dst, $mem\t# ptr" %}
 4340   ins_encode %{
 4341     __ movq($dst$$Register, $mem$$Address);
 4342   %}
 4343   ins_pipe(ialu_reg_mem); // XXX
 4344 %}
 4345 
 4346 // Load Compressed Pointer
 4347 instruct loadN(rRegN dst, memory mem)
 4348 %{
 4349    predicate(n->as_Load()->barrier_data() == 0);
 4350    match(Set dst (LoadN mem));
 4351 
 4352    ins_cost(125); // XXX
 4353    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4354    ins_encode %{
 4355      __ movl($dst$$Register, $mem$$Address);
 4356    %}
 4357    ins_pipe(ialu_reg_mem); // XXX
 4358 %}
 4359 
 4360 
 4361 // Load Klass Pointer
 4362 instruct loadKlass(rRegP dst, memory mem)
 4363 %{
 4364   match(Set dst (LoadKlass mem));
 4365 
 4366   ins_cost(125); // XXX
 4367   format %{ "movq    $dst, $mem\t# class" %}
 4368   ins_encode %{
 4369     __ movq($dst$$Register, $mem$$Address);
 4370   %}
 4371   ins_pipe(ialu_reg_mem); // XXX
 4372 %}
 4373 
 4374 // Load narrow Klass Pointer
 4375 instruct loadNKlass(rRegN dst, memory mem)
 4376 %{
 4377   predicate(!UseCompactObjectHeaders);
 4378   match(Set dst (LoadNKlass mem));
 4379 
 4380   ins_cost(125); // XXX
 4381   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4382   ins_encode %{
 4383     __ movl($dst$$Register, $mem$$Address);
 4384   %}
 4385   ins_pipe(ialu_reg_mem); // XXX
 4386 %}
 4387 
 4388 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4389 %{
 4390   predicate(UseCompactObjectHeaders);
 4391   match(Set dst (LoadNKlass mem));
 4392   effect(KILL cr);
 4393   ins_cost(125);
 4394   format %{
 4395     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4396     "shrl    $dst, markWord::klass_shift_at_offset"
 4397   %}
 4398   ins_encode %{
 4399     if (UseAPX) {
 4400       __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
 4401     }
 4402     else {
 4403       __ movl($dst$$Register, $mem$$Address);
 4404       __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4405     }
 4406   %}
 4407   ins_pipe(ialu_reg_mem);
 4408 %}
 4409 
 4410 // Load Float
 4411 instruct loadF(regF dst, memory mem)
 4412 %{
 4413   match(Set dst (LoadF mem));
 4414 
 4415   ins_cost(145); // XXX
 4416   format %{ "movss   $dst, $mem\t# float" %}
 4417   ins_encode %{
 4418     __ movflt($dst$$XMMRegister, $mem$$Address);
 4419   %}
 4420   ins_pipe(pipe_slow); // XXX
 4421 %}
 4422 
 4423 // Load Double
 4424 instruct loadD_partial(regD dst, memory mem)
 4425 %{
 4426   predicate(!UseXmmLoadAndClearUpper);
 4427   match(Set dst (LoadD mem));
 4428 
 4429   ins_cost(145); // XXX
 4430   format %{ "movlpd  $dst, $mem\t# double" %}
 4431   ins_encode %{
 4432     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4433   %}
 4434   ins_pipe(pipe_slow); // XXX
 4435 %}
 4436 
 4437 instruct loadD(regD dst, memory mem)
 4438 %{
 4439   predicate(UseXmmLoadAndClearUpper);
 4440   match(Set dst (LoadD mem));
 4441 
 4442   ins_cost(145); // XXX
 4443   format %{ "movsd   $dst, $mem\t# double" %}
 4444   ins_encode %{
 4445     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4446   %}
 4447   ins_pipe(pipe_slow); // XXX
 4448 %}
 4449 
 4450 // max = java.lang.Math.max(float a, float b)
 4451 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4452   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4453   match(Set dst (MaxF a b));
 4454   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4455   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4456   ins_encode %{
 4457     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4458   %}
 4459   ins_pipe( pipe_slow );
 4460 %}
 4461 
 4462 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4463   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4464   match(Set dst (MaxF a b));
 4465   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4466 
 4467   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4468   ins_encode %{
 4469     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4470                     false /*min*/, true /*single*/);
 4471   %}
 4472   ins_pipe( pipe_slow );
 4473 %}
 4474 
 4475 // max = java.lang.Math.max(double a, double b)
 4476 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4477   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4478   match(Set dst (MaxD a b));
 4479   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4480   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4481   ins_encode %{
 4482     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4483   %}
 4484   ins_pipe( pipe_slow );
 4485 %}
 4486 
 4487 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4488   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4489   match(Set dst (MaxD a b));
 4490   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4491 
 4492   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4493   ins_encode %{
 4494     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4495                     false /*min*/, false /*single*/);
 4496   %}
 4497   ins_pipe( pipe_slow );
 4498 %}
 4499 
 4500 // min = java.lang.Math.min(float a, float b)
 4501 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4502   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4503   match(Set dst (MinF a b));
 4504   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4505   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4506   ins_encode %{
 4507     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4508   %}
 4509   ins_pipe( pipe_slow );
 4510 %}
 4511 
 4512 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4513   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4514   match(Set dst (MinF a b));
 4515   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4516 
 4517   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4518   ins_encode %{
 4519     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4520                     true /*min*/, true /*single*/);
 4521   %}
 4522   ins_pipe( pipe_slow );
 4523 %}
 4524 
 4525 // min = java.lang.Math.min(double a, double b)
 4526 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4527   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4528   match(Set dst (MinD a b));
 4529   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4530     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4531   ins_encode %{
 4532     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4533   %}
 4534   ins_pipe( pipe_slow );
 4535 %}
 4536 
 4537 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4538   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4539   match(Set dst (MinD a b));
 4540   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4541 
 4542   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4543   ins_encode %{
 4544     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4545                     true /*min*/, false /*single*/);
 4546   %}
 4547   ins_pipe( pipe_slow );
 4548 %}
 4549 
 4550 // Load Effective Address
 4551 instruct leaP8(rRegP dst, indOffset8 mem)
 4552 %{
 4553   match(Set dst mem);
 4554 
 4555   ins_cost(110); // XXX
 4556   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4557   ins_encode %{
 4558     __ leaq($dst$$Register, $mem$$Address);
 4559   %}
 4560   ins_pipe(ialu_reg_reg_fat);
 4561 %}
 4562 
 4563 instruct leaP32(rRegP dst, indOffset32 mem)
 4564 %{
 4565   match(Set dst mem);
 4566 
 4567   ins_cost(110);
 4568   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4569   ins_encode %{
 4570     __ leaq($dst$$Register, $mem$$Address);
 4571   %}
 4572   ins_pipe(ialu_reg_reg_fat);
 4573 %}
 4574 
 4575 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4576 %{
 4577   match(Set dst mem);
 4578 
 4579   ins_cost(110);
 4580   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4581   ins_encode %{
 4582     __ leaq($dst$$Register, $mem$$Address);
 4583   %}
 4584   ins_pipe(ialu_reg_reg_fat);
 4585 %}
 4586 
 4587 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4588 %{
 4589   match(Set dst mem);
 4590 
 4591   ins_cost(110);
 4592   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4593   ins_encode %{
 4594     __ leaq($dst$$Register, $mem$$Address);
 4595   %}
 4596   ins_pipe(ialu_reg_reg_fat);
 4597 %}
 4598 
 4599 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4600 %{
 4601   match(Set dst mem);
 4602 
 4603   ins_cost(110);
 4604   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4605   ins_encode %{
 4606     __ leaq($dst$$Register, $mem$$Address);
 4607   %}
 4608   ins_pipe(ialu_reg_reg_fat);
 4609 %}
 4610 
 4611 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4612 %{
 4613   match(Set dst mem);
 4614 
 4615   ins_cost(110);
 4616   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4617   ins_encode %{
 4618     __ leaq($dst$$Register, $mem$$Address);
 4619   %}
 4620   ins_pipe(ialu_reg_reg_fat);
 4621 %}
 4622 
 4623 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4624 %{
 4625   match(Set dst mem);
 4626 
 4627   ins_cost(110);
 4628   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4629   ins_encode %{
 4630     __ leaq($dst$$Register, $mem$$Address);
 4631   %}
 4632   ins_pipe(ialu_reg_reg_fat);
 4633 %}
 4634 
 4635 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4636 %{
 4637   match(Set dst mem);
 4638 
 4639   ins_cost(110);
 4640   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4641   ins_encode %{
 4642     __ leaq($dst$$Register, $mem$$Address);
 4643   %}
 4644   ins_pipe(ialu_reg_reg_fat);
 4645 %}
 4646 
 4647 // Load Effective Address which uses Narrow (32-bits) oop
 4648 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4649 %{
 4650   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4651   match(Set dst mem);
 4652 
 4653   ins_cost(110);
 4654   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4655   ins_encode %{
 4656     __ leaq($dst$$Register, $mem$$Address);
 4657   %}
 4658   ins_pipe(ialu_reg_reg_fat);
 4659 %}
 4660 
 4661 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4662 %{
 4663   predicate(CompressedOops::shift() == 0);
 4664   match(Set dst mem);
 4665 
 4666   ins_cost(110); // XXX
 4667   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4668   ins_encode %{
 4669     __ leaq($dst$$Register, $mem$$Address);
 4670   %}
 4671   ins_pipe(ialu_reg_reg_fat);
 4672 %}
 4673 
 4674 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4675 %{
 4676   predicate(CompressedOops::shift() == 0);
 4677   match(Set dst mem);
 4678 
 4679   ins_cost(110);
 4680   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4681   ins_encode %{
 4682     __ leaq($dst$$Register, $mem$$Address);
 4683   %}
 4684   ins_pipe(ialu_reg_reg_fat);
 4685 %}
 4686 
 4687 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4688 %{
 4689   predicate(CompressedOops::shift() == 0);
 4690   match(Set dst mem);
 4691 
 4692   ins_cost(110);
 4693   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4694   ins_encode %{
 4695     __ leaq($dst$$Register, $mem$$Address);
 4696   %}
 4697   ins_pipe(ialu_reg_reg_fat);
 4698 %}
 4699 
 4700 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4701 %{
 4702   predicate(CompressedOops::shift() == 0);
 4703   match(Set dst mem);
 4704 
 4705   ins_cost(110);
 4706   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4707   ins_encode %{
 4708     __ leaq($dst$$Register, $mem$$Address);
 4709   %}
 4710   ins_pipe(ialu_reg_reg_fat);
 4711 %}
 4712 
 4713 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4714 %{
 4715   predicate(CompressedOops::shift() == 0);
 4716   match(Set dst mem);
 4717 
 4718   ins_cost(110);
 4719   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4720   ins_encode %{
 4721     __ leaq($dst$$Register, $mem$$Address);
 4722   %}
 4723   ins_pipe(ialu_reg_reg_fat);
 4724 %}
 4725 
 4726 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4727 %{
 4728   predicate(CompressedOops::shift() == 0);
 4729   match(Set dst mem);
 4730 
 4731   ins_cost(110);
 4732   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4733   ins_encode %{
 4734     __ leaq($dst$$Register, $mem$$Address);
 4735   %}
 4736   ins_pipe(ialu_reg_reg_fat);
 4737 %}
 4738 
 4739 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4740 %{
 4741   predicate(CompressedOops::shift() == 0);
 4742   match(Set dst mem);
 4743 
 4744   ins_cost(110);
 4745   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4746   ins_encode %{
 4747     __ leaq($dst$$Register, $mem$$Address);
 4748   %}
 4749   ins_pipe(ialu_reg_reg_fat);
 4750 %}
 4751 
 4752 instruct loadConI(rRegI dst, immI src)
 4753 %{
 4754   match(Set dst src);
 4755 
 4756   format %{ "movl    $dst, $src\t# int" %}
 4757   ins_encode %{
 4758     __ movl($dst$$Register, $src$$constant);
 4759   %}
 4760   ins_pipe(ialu_reg_fat); // XXX
 4761 %}
 4762 
 4763 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4764 %{
 4765   match(Set dst src);
 4766   effect(KILL cr);
 4767 
 4768   ins_cost(50);
 4769   format %{ "xorl    $dst, $dst\t# int" %}
 4770   ins_encode %{
 4771     __ xorl($dst$$Register, $dst$$Register);
 4772   %}
 4773   ins_pipe(ialu_reg);
 4774 %}
 4775 
 4776 instruct loadConL(rRegL dst, immL src)
 4777 %{
 4778   match(Set dst src);
 4779 
 4780   ins_cost(150);
 4781   format %{ "movq    $dst, $src\t# long" %}
 4782   ins_encode %{
 4783     __ mov64($dst$$Register, $src$$constant);
 4784   %}
 4785   ins_pipe(ialu_reg);
 4786 %}
 4787 
 4788 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4789 %{
 4790   match(Set dst src);
 4791   effect(KILL cr);
 4792 
 4793   ins_cost(50);
 4794   format %{ "xorl    $dst, $dst\t# long" %}
 4795   ins_encode %{
 4796     __ xorl($dst$$Register, $dst$$Register);
 4797   %}
 4798   ins_pipe(ialu_reg); // XXX
 4799 %}
 4800 
 4801 instruct loadConUL32(rRegL dst, immUL32 src)
 4802 %{
 4803   match(Set dst src);
 4804 
 4805   ins_cost(60);
 4806   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4807   ins_encode %{
 4808     __ movl($dst$$Register, $src$$constant);
 4809   %}
 4810   ins_pipe(ialu_reg);
 4811 %}
 4812 
 4813 instruct loadConL32(rRegL dst, immL32 src)
 4814 %{
 4815   match(Set dst src);
 4816 
 4817   ins_cost(70);
 4818   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4819   ins_encode %{
 4820     __ movq($dst$$Register, $src$$constant);
 4821   %}
 4822   ins_pipe(ialu_reg);
 4823 %}
 4824 
 4825 instruct loadConP(rRegP dst, immP con) %{
 4826   match(Set dst con);
 4827 
 4828   format %{ "movq    $dst, $con\t# ptr" %}
 4829   ins_encode %{
 4830     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4831   %}
 4832   ins_pipe(ialu_reg_fat); // XXX
 4833 %}
 4834 
 4835 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4836 %{
 4837   match(Set dst src);
 4838   effect(KILL cr);
 4839 
 4840   ins_cost(50);
 4841   format %{ "xorl    $dst, $dst\t# ptr" %}
 4842   ins_encode %{
 4843     __ xorl($dst$$Register, $dst$$Register);
 4844   %}
 4845   ins_pipe(ialu_reg);
 4846 %}
 4847 
 4848 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4849 %{
 4850   match(Set dst src);
 4851   effect(KILL cr);
 4852 
 4853   ins_cost(60);
 4854   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4855   ins_encode %{
 4856     __ movl($dst$$Register, $src$$constant);
 4857   %}
 4858   ins_pipe(ialu_reg);
 4859 %}
 4860 
 4861 instruct loadConF(regF dst, immF con) %{
 4862   match(Set dst con);
 4863   ins_cost(125);
 4864   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4865   ins_encode %{
 4866     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4867   %}
 4868   ins_pipe(pipe_slow);
 4869 %}
 4870 
 4871 instruct loadConH(regF dst, immH con) %{
 4872   match(Set dst con);
 4873   ins_cost(125);
 4874   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4875   ins_encode %{
 4876     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4877   %}
 4878   ins_pipe(pipe_slow);
 4879 %}
 4880 
 4881 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4882   match(Set dst src);
 4883   effect(KILL cr);
 4884   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4885   ins_encode %{
 4886     __ xorq($dst$$Register, $dst$$Register);
 4887   %}
 4888   ins_pipe(ialu_reg);
 4889 %}
 4890 
 4891 instruct loadConN(rRegN dst, immN src) %{
 4892   match(Set dst src);
 4893 
 4894   ins_cost(125);
 4895   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4896   ins_encode %{
 4897     address con = (address)$src$$constant;
 4898     if (con == nullptr) {
 4899       ShouldNotReachHere();
 4900     } else {
 4901       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4902     }
 4903   %}
 4904   ins_pipe(ialu_reg_fat); // XXX
 4905 %}
 4906 
 4907 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4908   match(Set dst src);
 4909 
 4910   ins_cost(125);
 4911   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4912   ins_encode %{
 4913     address con = (address)$src$$constant;
 4914     if (con == nullptr) {
 4915       ShouldNotReachHere();
 4916     } else {
 4917       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4918     }
 4919   %}
 4920   ins_pipe(ialu_reg_fat); // XXX
 4921 %}
 4922 
 4923 instruct loadConF0(regF dst, immF0 src)
 4924 %{
 4925   match(Set dst src);
 4926   ins_cost(100);
 4927 
 4928   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4929   ins_encode %{
 4930     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4931   %}
 4932   ins_pipe(pipe_slow);
 4933 %}
 4934 
 4935 // Use the same format since predicate() can not be used here.
 4936 instruct loadConD(regD dst, immD con) %{
 4937   match(Set dst con);
 4938   ins_cost(125);
 4939   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4940   ins_encode %{
 4941     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4942   %}
 4943   ins_pipe(pipe_slow);
 4944 %}
 4945 
 4946 instruct loadConD0(regD dst, immD0 src)
 4947 %{
 4948   match(Set dst src);
 4949   ins_cost(100);
 4950 
 4951   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4952   ins_encode %{
 4953     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4954   %}
 4955   ins_pipe(pipe_slow);
 4956 %}
 4957 
 4958 instruct loadSSI(rRegI dst, stackSlotI src)
 4959 %{
 4960   match(Set dst src);
 4961 
 4962   ins_cost(125);
 4963   format %{ "movl    $dst, $src\t# int stk" %}
 4964   ins_encode %{
 4965     __ movl($dst$$Register, $src$$Address);
 4966   %}
 4967   ins_pipe(ialu_reg_mem);
 4968 %}
 4969 
 4970 instruct loadSSL(rRegL dst, stackSlotL src)
 4971 %{
 4972   match(Set dst src);
 4973 
 4974   ins_cost(125);
 4975   format %{ "movq    $dst, $src\t# long stk" %}
 4976   ins_encode %{
 4977     __ movq($dst$$Register, $src$$Address);
 4978   %}
 4979   ins_pipe(ialu_reg_mem);
 4980 %}
 4981 
 4982 instruct loadSSP(rRegP dst, stackSlotP src)
 4983 %{
 4984   match(Set dst src);
 4985 
 4986   ins_cost(125);
 4987   format %{ "movq    $dst, $src\t# ptr stk" %}
 4988   ins_encode %{
 4989     __ movq($dst$$Register, $src$$Address);
 4990   %}
 4991   ins_pipe(ialu_reg_mem);
 4992 %}
 4993 
 4994 instruct loadSSF(regF dst, stackSlotF src)
 4995 %{
 4996   match(Set dst src);
 4997 
 4998   ins_cost(125);
 4999   format %{ "movss   $dst, $src\t# float stk" %}
 5000   ins_encode %{
 5001     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5002   %}
 5003   ins_pipe(pipe_slow); // XXX
 5004 %}
 5005 
 5006 // Use the same format since predicate() can not be used here.
 5007 instruct loadSSD(regD dst, stackSlotD src)
 5008 %{
 5009   match(Set dst src);
 5010 
 5011   ins_cost(125);
 5012   format %{ "movsd   $dst, $src\t# double stk" %}
 5013   ins_encode  %{
 5014     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5015   %}
 5016   ins_pipe(pipe_slow); // XXX
 5017 %}
 5018 
 5019 // Prefetch instructions for allocation.
 5020 // Must be safe to execute with invalid address (cannot fault).
 5021 
 5022 instruct prefetchAlloc( memory mem ) %{
 5023   predicate(AllocatePrefetchInstr==3);
 5024   match(PrefetchAllocation mem);
 5025   ins_cost(125);
 5026 
 5027   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5028   ins_encode %{
 5029     __ prefetchw($mem$$Address);
 5030   %}
 5031   ins_pipe(ialu_mem);
 5032 %}
 5033 
 5034 instruct prefetchAllocNTA( memory mem ) %{
 5035   predicate(AllocatePrefetchInstr==0);
 5036   match(PrefetchAllocation mem);
 5037   ins_cost(125);
 5038 
 5039   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5040   ins_encode %{
 5041     __ prefetchnta($mem$$Address);
 5042   %}
 5043   ins_pipe(ialu_mem);
 5044 %}
 5045 
 5046 instruct prefetchAllocT0( memory mem ) %{
 5047   predicate(AllocatePrefetchInstr==1);
 5048   match(PrefetchAllocation mem);
 5049   ins_cost(125);
 5050 
 5051   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5052   ins_encode %{
 5053     __ prefetcht0($mem$$Address);
 5054   %}
 5055   ins_pipe(ialu_mem);
 5056 %}
 5057 
 5058 instruct prefetchAllocT2( memory mem ) %{
 5059   predicate(AllocatePrefetchInstr==2);
 5060   match(PrefetchAllocation mem);
 5061   ins_cost(125);
 5062 
 5063   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5064   ins_encode %{
 5065     __ prefetcht2($mem$$Address);
 5066   %}
 5067   ins_pipe(ialu_mem);
 5068 %}
 5069 
 5070 //----------Store Instructions-------------------------------------------------
 5071 
 5072 // Store Byte
 5073 instruct storeB(memory mem, rRegI src)
 5074 %{
 5075   match(Set mem (StoreB mem src));
 5076 
 5077   ins_cost(125); // XXX
 5078   format %{ "movb    $mem, $src\t# byte" %}
 5079   ins_encode %{
 5080     __ movb($mem$$Address, $src$$Register);
 5081   %}
 5082   ins_pipe(ialu_mem_reg);
 5083 %}
 5084 
 5085 // Store Char/Short
 5086 instruct storeC(memory mem, rRegI src)
 5087 %{
 5088   match(Set mem (StoreC mem src));
 5089 
 5090   ins_cost(125); // XXX
 5091   format %{ "movw    $mem, $src\t# char/short" %}
 5092   ins_encode %{
 5093     __ movw($mem$$Address, $src$$Register);
 5094   %}
 5095   ins_pipe(ialu_mem_reg);
 5096 %}
 5097 
 5098 // Store Integer
 5099 instruct storeI(memory mem, rRegI src)
 5100 %{
 5101   match(Set mem (StoreI mem src));
 5102 
 5103   ins_cost(125); // XXX
 5104   format %{ "movl    $mem, $src\t# int" %}
 5105   ins_encode %{
 5106     __ movl($mem$$Address, $src$$Register);
 5107   %}
 5108   ins_pipe(ialu_mem_reg);
 5109 %}
 5110 
 5111 // Store Long
 5112 instruct storeL(memory mem, rRegL src)
 5113 %{
 5114   match(Set mem (StoreL mem src));
 5115 
 5116   ins_cost(125); // XXX
 5117   format %{ "movq    $mem, $src\t# long" %}
 5118   ins_encode %{
 5119     __ movq($mem$$Address, $src$$Register);
 5120   %}
 5121   ins_pipe(ialu_mem_reg); // XXX
 5122 %}
 5123 
 5124 // Store Pointer
 5125 instruct storeP(memory mem, any_RegP src)
 5126 %{
 5127   predicate(n->as_Store()->barrier_data() == 0);
 5128   match(Set mem (StoreP mem src));
 5129 
 5130   ins_cost(125); // XXX
 5131   format %{ "movq    $mem, $src\t# ptr" %}
 5132   ins_encode %{
 5133     __ movq($mem$$Address, $src$$Register);
 5134   %}
 5135   ins_pipe(ialu_mem_reg);
 5136 %}
 5137 
 5138 instruct storeImmP0(memory mem, immP0 zero)
 5139 %{
 5140   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5141   match(Set mem (StoreP mem zero));
 5142 
 5143   ins_cost(125); // XXX
 5144   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5145   ins_encode %{
 5146     __ movq($mem$$Address, r12);
 5147   %}
 5148   ins_pipe(ialu_mem_reg);
 5149 %}
 5150 
 5151 // Store Null Pointer, mark word, or other simple pointer constant.
 5152 instruct storeImmP(memory mem, immP31 src)
 5153 %{
 5154   predicate(n->as_Store()->barrier_data() == 0);
 5155   match(Set mem (StoreP mem src));
 5156 
 5157   ins_cost(150); // XXX
 5158   format %{ "movq    $mem, $src\t# ptr" %}
 5159   ins_encode %{
 5160     __ movq($mem$$Address, $src$$constant);
 5161   %}
 5162   ins_pipe(ialu_mem_imm);
 5163 %}
 5164 
 5165 // Store Compressed Pointer
 5166 instruct storeN(memory mem, rRegN src)
 5167 %{
 5168   predicate(n->as_Store()->barrier_data() == 0);
 5169   match(Set mem (StoreN mem src));
 5170 
 5171   ins_cost(125); // XXX
 5172   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5173   ins_encode %{
 5174     __ movl($mem$$Address, $src$$Register);
 5175   %}
 5176   ins_pipe(ialu_mem_reg);
 5177 %}
 5178 
 5179 instruct storeNKlass(memory mem, rRegN src)
 5180 %{
 5181   match(Set mem (StoreNKlass mem src));
 5182 
 5183   ins_cost(125); // XXX
 5184   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5185   ins_encode %{
 5186     __ movl($mem$$Address, $src$$Register);
 5187   %}
 5188   ins_pipe(ialu_mem_reg);
 5189 %}
 5190 
 5191 instruct storeImmN0(memory mem, immN0 zero)
 5192 %{
 5193   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5194   match(Set mem (StoreN mem zero));
 5195 
 5196   ins_cost(125); // XXX
 5197   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5198   ins_encode %{
 5199     __ movl($mem$$Address, r12);
 5200   %}
 5201   ins_pipe(ialu_mem_reg);
 5202 %}
 5203 
 5204 instruct storeImmN(memory mem, immN src)
 5205 %{
 5206   predicate(n->as_Store()->barrier_data() == 0);
 5207   match(Set mem (StoreN mem src));
 5208 
 5209   ins_cost(150); // XXX
 5210   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5211   ins_encode %{
 5212     address con = (address)$src$$constant;
 5213     if (con == nullptr) {
 5214       __ movl($mem$$Address, 0);
 5215     } else {
 5216       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5217     }
 5218   %}
 5219   ins_pipe(ialu_mem_imm);
 5220 %}
 5221 
 5222 instruct storeImmNKlass(memory mem, immNKlass src)
 5223 %{
 5224   match(Set mem (StoreNKlass mem src));
 5225 
 5226   ins_cost(150); // XXX
 5227   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5228   ins_encode %{
 5229     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5230   %}
 5231   ins_pipe(ialu_mem_imm);
 5232 %}
 5233 
 5234 // Store Integer Immediate
 5235 instruct storeImmI0(memory mem, immI_0 zero)
 5236 %{
 5237   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5238   match(Set mem (StoreI mem zero));
 5239 
 5240   ins_cost(125); // XXX
 5241   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5242   ins_encode %{
 5243     __ movl($mem$$Address, r12);
 5244   %}
 5245   ins_pipe(ialu_mem_reg);
 5246 %}
 5247 
 5248 instruct storeImmI(memory mem, immI src)
 5249 %{
 5250   match(Set mem (StoreI mem src));
 5251 
 5252   ins_cost(150);
 5253   format %{ "movl    $mem, $src\t# int" %}
 5254   ins_encode %{
 5255     __ movl($mem$$Address, $src$$constant);
 5256   %}
 5257   ins_pipe(ialu_mem_imm);
 5258 %}
 5259 
 5260 // Store Long Immediate
 5261 instruct storeImmL0(memory mem, immL0 zero)
 5262 %{
 5263   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5264   match(Set mem (StoreL mem zero));
 5265 
 5266   ins_cost(125); // XXX
 5267   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5268   ins_encode %{
 5269     __ movq($mem$$Address, r12);
 5270   %}
 5271   ins_pipe(ialu_mem_reg);
 5272 %}
 5273 
 5274 instruct storeImmL(memory mem, immL32 src)
 5275 %{
 5276   match(Set mem (StoreL mem src));
 5277 
 5278   ins_cost(150);
 5279   format %{ "movq    $mem, $src\t# long" %}
 5280   ins_encode %{
 5281     __ movq($mem$$Address, $src$$constant);
 5282   %}
 5283   ins_pipe(ialu_mem_imm);
 5284 %}
 5285 
 5286 // Store Short/Char Immediate
 5287 instruct storeImmC0(memory mem, immI_0 zero)
 5288 %{
 5289   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5290   match(Set mem (StoreC mem zero));
 5291 
 5292   ins_cost(125); // XXX
 5293   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5294   ins_encode %{
 5295     __ movw($mem$$Address, r12);
 5296   %}
 5297   ins_pipe(ialu_mem_reg);
 5298 %}
 5299 
 5300 instruct storeImmI16(memory mem, immI16 src)
 5301 %{
 5302   predicate(UseStoreImmI16);
 5303   match(Set mem (StoreC mem src));
 5304 
 5305   ins_cost(150);
 5306   format %{ "movw    $mem, $src\t# short/char" %}
 5307   ins_encode %{
 5308     __ movw($mem$$Address, $src$$constant);
 5309   %}
 5310   ins_pipe(ialu_mem_imm);
 5311 %}
 5312 
 5313 // Store Byte Immediate
 5314 instruct storeImmB0(memory mem, immI_0 zero)
 5315 %{
 5316   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5317   match(Set mem (StoreB mem zero));
 5318 
 5319   ins_cost(125); // XXX
 5320   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5321   ins_encode %{
 5322     __ movb($mem$$Address, r12);
 5323   %}
 5324   ins_pipe(ialu_mem_reg);
 5325 %}
 5326 
 5327 instruct storeImmB(memory mem, immI8 src)
 5328 %{
 5329   match(Set mem (StoreB mem src));
 5330 
 5331   ins_cost(150); // XXX
 5332   format %{ "movb    $mem, $src\t# byte" %}
 5333   ins_encode %{
 5334     __ movb($mem$$Address, $src$$constant);
 5335   %}
 5336   ins_pipe(ialu_mem_imm);
 5337 %}
 5338 
 5339 // Store Float
 5340 instruct storeF(memory mem, regF src)
 5341 %{
 5342   match(Set mem (StoreF mem src));
 5343 
 5344   ins_cost(95); // XXX
 5345   format %{ "movss   $mem, $src\t# float" %}
 5346   ins_encode %{
 5347     __ movflt($mem$$Address, $src$$XMMRegister);
 5348   %}
 5349   ins_pipe(pipe_slow); // XXX
 5350 %}
 5351 
 5352 // Store immediate Float value (it is faster than store from XMM register)
 5353 instruct storeF0(memory mem, immF0 zero)
 5354 %{
 5355   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5356   match(Set mem (StoreF mem zero));
 5357 
 5358   ins_cost(25); // XXX
 5359   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5360   ins_encode %{
 5361     __ movl($mem$$Address, r12);
 5362   %}
 5363   ins_pipe(ialu_mem_reg);
 5364 %}
 5365 
 5366 instruct storeF_imm(memory mem, immF src)
 5367 %{
 5368   match(Set mem (StoreF mem src));
 5369 
 5370   ins_cost(50);
 5371   format %{ "movl    $mem, $src\t# float" %}
 5372   ins_encode %{
 5373     __ movl($mem$$Address, jint_cast($src$$constant));
 5374   %}
 5375   ins_pipe(ialu_mem_imm);
 5376 %}
 5377 
 5378 // Store Double
 5379 instruct storeD(memory mem, regD src)
 5380 %{
 5381   match(Set mem (StoreD mem src));
 5382 
 5383   ins_cost(95); // XXX
 5384   format %{ "movsd   $mem, $src\t# double" %}
 5385   ins_encode %{
 5386     __ movdbl($mem$$Address, $src$$XMMRegister);
 5387   %}
 5388   ins_pipe(pipe_slow); // XXX
 5389 %}
 5390 
 5391 // Store immediate double 0.0 (it is faster than store from XMM register)
 5392 instruct storeD0_imm(memory mem, immD0 src)
 5393 %{
 5394   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5395   match(Set mem (StoreD mem src));
 5396 
 5397   ins_cost(50);
 5398   format %{ "movq    $mem, $src\t# double 0." %}
 5399   ins_encode %{
 5400     __ movq($mem$$Address, $src$$constant);
 5401   %}
 5402   ins_pipe(ialu_mem_imm);
 5403 %}
 5404 
 5405 instruct storeD0(memory mem, immD0 zero)
 5406 %{
 5407   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5408   match(Set mem (StoreD mem zero));
 5409 
 5410   ins_cost(25); // XXX
 5411   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5412   ins_encode %{
 5413     __ movq($mem$$Address, r12);
 5414   %}
 5415   ins_pipe(ialu_mem_reg);
 5416 %}
 5417 
 5418 instruct storeSSI(stackSlotI dst, rRegI src)
 5419 %{
 5420   match(Set dst src);
 5421 
 5422   ins_cost(100);
 5423   format %{ "movl    $dst, $src\t# int stk" %}
 5424   ins_encode %{
 5425     __ movl($dst$$Address, $src$$Register);
 5426   %}
 5427   ins_pipe( ialu_mem_reg );
 5428 %}
 5429 
 5430 instruct storeSSL(stackSlotL dst, rRegL src)
 5431 %{
 5432   match(Set dst src);
 5433 
 5434   ins_cost(100);
 5435   format %{ "movq    $dst, $src\t# long stk" %}
 5436   ins_encode %{
 5437     __ movq($dst$$Address, $src$$Register);
 5438   %}
 5439   ins_pipe(ialu_mem_reg);
 5440 %}
 5441 
 5442 instruct storeSSP(stackSlotP dst, rRegP src)
 5443 %{
 5444   match(Set dst src);
 5445 
 5446   ins_cost(100);
 5447   format %{ "movq    $dst, $src\t# ptr stk" %}
 5448   ins_encode %{
 5449     __ movq($dst$$Address, $src$$Register);
 5450   %}
 5451   ins_pipe(ialu_mem_reg);
 5452 %}
 5453 
 5454 instruct storeSSF(stackSlotF dst, regF src)
 5455 %{
 5456   match(Set dst src);
 5457 
 5458   ins_cost(95); // XXX
 5459   format %{ "movss   $dst, $src\t# float stk" %}
 5460   ins_encode %{
 5461     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5462   %}
 5463   ins_pipe(pipe_slow); // XXX
 5464 %}
 5465 
 5466 instruct storeSSD(stackSlotD dst, regD src)
 5467 %{
 5468   match(Set dst src);
 5469 
 5470   ins_cost(95); // XXX
 5471   format %{ "movsd   $dst, $src\t# double stk" %}
 5472   ins_encode %{
 5473     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5474   %}
 5475   ins_pipe(pipe_slow); // XXX
 5476 %}
 5477 
 5478 instruct cacheWB(indirect addr)
 5479 %{
 5480   predicate(VM_Version::supports_data_cache_line_flush());
 5481   match(CacheWB addr);
 5482 
 5483   ins_cost(100);
 5484   format %{"cache wb $addr" %}
 5485   ins_encode %{
 5486     assert($addr->index_position() < 0, "should be");
 5487     assert($addr$$disp == 0, "should be");
 5488     __ cache_wb(Address($addr$$base$$Register, 0));
 5489   %}
 5490   ins_pipe(pipe_slow); // XXX
 5491 %}
 5492 
 5493 instruct cacheWBPreSync()
 5494 %{
 5495   predicate(VM_Version::supports_data_cache_line_flush());
 5496   match(CacheWBPreSync);
 5497 
 5498   ins_cost(100);
 5499   format %{"cache wb presync" %}
 5500   ins_encode %{
 5501     __ cache_wbsync(true);
 5502   %}
 5503   ins_pipe(pipe_slow); // XXX
 5504 %}
 5505 
 5506 instruct cacheWBPostSync()
 5507 %{
 5508   predicate(VM_Version::supports_data_cache_line_flush());
 5509   match(CacheWBPostSync);
 5510 
 5511   ins_cost(100);
 5512   format %{"cache wb postsync" %}
 5513   ins_encode %{
 5514     __ cache_wbsync(false);
 5515   %}
 5516   ins_pipe(pipe_slow); // XXX
 5517 %}
 5518 
 5519 //----------BSWAP Instructions-------------------------------------------------
 5520 instruct bytes_reverse_int(rRegI dst) %{
 5521   match(Set dst (ReverseBytesI dst));
 5522 
 5523   format %{ "bswapl  $dst" %}
 5524   ins_encode %{
 5525     __ bswapl($dst$$Register);
 5526   %}
 5527   ins_pipe( ialu_reg );
 5528 %}
 5529 
 5530 instruct bytes_reverse_long(rRegL dst) %{
 5531   match(Set dst (ReverseBytesL dst));
 5532 
 5533   format %{ "bswapq  $dst" %}
 5534   ins_encode %{
 5535     __ bswapq($dst$$Register);
 5536   %}
 5537   ins_pipe( ialu_reg);
 5538 %}
 5539 
 5540 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5541   match(Set dst (ReverseBytesUS dst));
 5542   effect(KILL cr);
 5543 
 5544   format %{ "bswapl  $dst\n\t"
 5545             "shrl    $dst,16\n\t" %}
 5546   ins_encode %{
 5547     __ bswapl($dst$$Register);
 5548     __ shrl($dst$$Register, 16);
 5549   %}
 5550   ins_pipe( ialu_reg );
 5551 %}
 5552 
 5553 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5554   match(Set dst (ReverseBytesS dst));
 5555   effect(KILL cr);
 5556 
 5557   format %{ "bswapl  $dst\n\t"
 5558             "sar     $dst,16\n\t" %}
 5559   ins_encode %{
 5560     __ bswapl($dst$$Register);
 5561     __ sarl($dst$$Register, 16);
 5562   %}
 5563   ins_pipe( ialu_reg );
 5564 %}
 5565 
 5566 //---------- Zeros Count Instructions ------------------------------------------
 5567 
 5568 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5569   predicate(UseCountLeadingZerosInstruction);
 5570   match(Set dst (CountLeadingZerosI src));
 5571   effect(KILL cr);
 5572 
 5573   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5574   ins_encode %{
 5575     __ lzcntl($dst$$Register, $src$$Register);
 5576   %}
 5577   ins_pipe(ialu_reg);
 5578 %}
 5579 
 5580 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5581   predicate(UseCountLeadingZerosInstruction);
 5582   match(Set dst (CountLeadingZerosI (LoadI src)));
 5583   effect(KILL cr);
 5584   ins_cost(175);
 5585   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5586   ins_encode %{
 5587     __ lzcntl($dst$$Register, $src$$Address);
 5588   %}
 5589   ins_pipe(ialu_reg_mem);
 5590 %}
 5591 
 5592 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5593   predicate(!UseCountLeadingZerosInstruction);
 5594   match(Set dst (CountLeadingZerosI src));
 5595   effect(KILL cr);
 5596 
 5597   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5598             "jnz     skip\n\t"
 5599             "movl    $dst, -1\n"
 5600       "skip:\n\t"
 5601             "negl    $dst\n\t"
 5602             "addl    $dst, 31" %}
 5603   ins_encode %{
 5604     Register Rdst = $dst$$Register;
 5605     Register Rsrc = $src$$Register;
 5606     Label skip;
 5607     __ bsrl(Rdst, Rsrc);
 5608     __ jccb(Assembler::notZero, skip);
 5609     __ movl(Rdst, -1);
 5610     __ bind(skip);
 5611     __ negl(Rdst);
 5612     __ addl(Rdst, BitsPerInt - 1);
 5613   %}
 5614   ins_pipe(ialu_reg);
 5615 %}
 5616 
 5617 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5618   predicate(UseCountLeadingZerosInstruction);
 5619   match(Set dst (CountLeadingZerosL src));
 5620   effect(KILL cr);
 5621 
 5622   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5623   ins_encode %{
 5624     __ lzcntq($dst$$Register, $src$$Register);
 5625   %}
 5626   ins_pipe(ialu_reg);
 5627 %}
 5628 
 5629 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5630   predicate(UseCountLeadingZerosInstruction);
 5631   match(Set dst (CountLeadingZerosL (LoadL src)));
 5632   effect(KILL cr);
 5633   ins_cost(175);
 5634   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5635   ins_encode %{
 5636     __ lzcntq($dst$$Register, $src$$Address);
 5637   %}
 5638   ins_pipe(ialu_reg_mem);
 5639 %}
 5640 
 5641 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5642   predicate(!UseCountLeadingZerosInstruction);
 5643   match(Set dst (CountLeadingZerosL src));
 5644   effect(KILL cr);
 5645 
 5646   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5647             "jnz     skip\n\t"
 5648             "movl    $dst, -1\n"
 5649       "skip:\n\t"
 5650             "negl    $dst\n\t"
 5651             "addl    $dst, 63" %}
 5652   ins_encode %{
 5653     Register Rdst = $dst$$Register;
 5654     Register Rsrc = $src$$Register;
 5655     Label skip;
 5656     __ bsrq(Rdst, Rsrc);
 5657     __ jccb(Assembler::notZero, skip);
 5658     __ movl(Rdst, -1);
 5659     __ bind(skip);
 5660     __ negl(Rdst);
 5661     __ addl(Rdst, BitsPerLong - 1);
 5662   %}
 5663   ins_pipe(ialu_reg);
 5664 %}
 5665 
 5666 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5667   predicate(UseCountTrailingZerosInstruction);
 5668   match(Set dst (CountTrailingZerosI src));
 5669   effect(KILL cr);
 5670 
 5671   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5672   ins_encode %{
 5673     __ tzcntl($dst$$Register, $src$$Register);
 5674   %}
 5675   ins_pipe(ialu_reg);
 5676 %}
 5677 
 5678 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5679   predicate(UseCountTrailingZerosInstruction);
 5680   match(Set dst (CountTrailingZerosI (LoadI src)));
 5681   effect(KILL cr);
 5682   ins_cost(175);
 5683   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5684   ins_encode %{
 5685     __ tzcntl($dst$$Register, $src$$Address);
 5686   %}
 5687   ins_pipe(ialu_reg_mem);
 5688 %}
 5689 
 5690 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5691   predicate(!UseCountTrailingZerosInstruction);
 5692   match(Set dst (CountTrailingZerosI src));
 5693   effect(KILL cr);
 5694 
 5695   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5696             "jnz     done\n\t"
 5697             "movl    $dst, 32\n"
 5698       "done:" %}
 5699   ins_encode %{
 5700     Register Rdst = $dst$$Register;
 5701     Label done;
 5702     __ bsfl(Rdst, $src$$Register);
 5703     __ jccb(Assembler::notZero, done);
 5704     __ movl(Rdst, BitsPerInt);
 5705     __ bind(done);
 5706   %}
 5707   ins_pipe(ialu_reg);
 5708 %}
 5709 
 5710 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5711   predicate(UseCountTrailingZerosInstruction);
 5712   match(Set dst (CountTrailingZerosL src));
 5713   effect(KILL cr);
 5714 
 5715   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5716   ins_encode %{
 5717     __ tzcntq($dst$$Register, $src$$Register);
 5718   %}
 5719   ins_pipe(ialu_reg);
 5720 %}
 5721 
 5722 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5723   predicate(UseCountTrailingZerosInstruction);
 5724   match(Set dst (CountTrailingZerosL (LoadL src)));
 5725   effect(KILL cr);
 5726   ins_cost(175);
 5727   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5728   ins_encode %{
 5729     __ tzcntq($dst$$Register, $src$$Address);
 5730   %}
 5731   ins_pipe(ialu_reg_mem);
 5732 %}
 5733 
 5734 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5735   predicate(!UseCountTrailingZerosInstruction);
 5736   match(Set dst (CountTrailingZerosL src));
 5737   effect(KILL cr);
 5738 
 5739   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5740             "jnz     done\n\t"
 5741             "movl    $dst, 64\n"
 5742       "done:" %}
 5743   ins_encode %{
 5744     Register Rdst = $dst$$Register;
 5745     Label done;
 5746     __ bsfq(Rdst, $src$$Register);
 5747     __ jccb(Assembler::notZero, done);
 5748     __ movl(Rdst, BitsPerLong);
 5749     __ bind(done);
 5750   %}
 5751   ins_pipe(ialu_reg);
 5752 %}
 5753 
 5754 //--------------- Reverse Operation Instructions ----------------
 5755 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5756   predicate(!VM_Version::supports_gfni());
 5757   match(Set dst (ReverseI src));
 5758   effect(TEMP dst, TEMP rtmp, KILL cr);
 5759   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5760   ins_encode %{
 5761     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5762   %}
 5763   ins_pipe( ialu_reg );
 5764 %}
 5765 
 5766 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5767   predicate(VM_Version::supports_gfni());
 5768   match(Set dst (ReverseI src));
 5769   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5770   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5771   ins_encode %{
 5772     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5773   %}
 5774   ins_pipe( ialu_reg );
 5775 %}
 5776 
 5777 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5778   predicate(!VM_Version::supports_gfni());
 5779   match(Set dst (ReverseL src));
 5780   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5781   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5782   ins_encode %{
 5783     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5784   %}
 5785   ins_pipe( ialu_reg );
 5786 %}
 5787 
 5788 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5789   predicate(VM_Version::supports_gfni());
 5790   match(Set dst (ReverseL src));
 5791   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5792   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5793   ins_encode %{
 5794     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5795   %}
 5796   ins_pipe( ialu_reg );
 5797 %}
 5798 
 5799 //---------- Population Count Instructions -------------------------------------
 5800 
 5801 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5802   predicate(UsePopCountInstruction);
 5803   match(Set dst (PopCountI src));
 5804   effect(KILL cr);
 5805 
 5806   format %{ "popcnt  $dst, $src" %}
 5807   ins_encode %{
 5808     __ popcntl($dst$$Register, $src$$Register);
 5809   %}
 5810   ins_pipe(ialu_reg);
 5811 %}
 5812 
 5813 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5814   predicate(UsePopCountInstruction);
 5815   match(Set dst (PopCountI (LoadI mem)));
 5816   effect(KILL cr);
 5817 
 5818   format %{ "popcnt  $dst, $mem" %}
 5819   ins_encode %{
 5820     __ popcntl($dst$$Register, $mem$$Address);
 5821   %}
 5822   ins_pipe(ialu_reg);
 5823 %}
 5824 
 5825 // Note: Long.bitCount(long) returns an int.
 5826 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5827   predicate(UsePopCountInstruction);
 5828   match(Set dst (PopCountL src));
 5829   effect(KILL cr);
 5830 
 5831   format %{ "popcnt  $dst, $src" %}
 5832   ins_encode %{
 5833     __ popcntq($dst$$Register, $src$$Register);
 5834   %}
 5835   ins_pipe(ialu_reg);
 5836 %}
 5837 
 5838 // Note: Long.bitCount(long) returns an int.
 5839 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5840   predicate(UsePopCountInstruction);
 5841   match(Set dst (PopCountL (LoadL mem)));
 5842   effect(KILL cr);
 5843 
 5844   format %{ "popcnt  $dst, $mem" %}
 5845   ins_encode %{
 5846     __ popcntq($dst$$Register, $mem$$Address);
 5847   %}
 5848   ins_pipe(ialu_reg);
 5849 %}
 5850 
 5851 
 5852 //----------MemBar Instructions-----------------------------------------------
 5853 // Memory barrier flavors
 5854 
 5855 instruct membar_acquire()
 5856 %{
 5857   match(MemBarAcquire);
 5858   match(LoadFence);
 5859   ins_cost(0);
 5860 
 5861   size(0);
 5862   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5863   ins_encode();
 5864   ins_pipe(empty);
 5865 %}
 5866 
 5867 instruct membar_acquire_lock()
 5868 %{
 5869   match(MemBarAcquireLock);
 5870   ins_cost(0);
 5871 
 5872   size(0);
 5873   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5874   ins_encode();
 5875   ins_pipe(empty);
 5876 %}
 5877 
 5878 instruct membar_release()
 5879 %{
 5880   match(MemBarRelease);
 5881   match(StoreFence);
 5882   ins_cost(0);
 5883 
 5884   size(0);
 5885   format %{ "MEMBAR-release ! (empty encoding)" %}
 5886   ins_encode();
 5887   ins_pipe(empty);
 5888 %}
 5889 
 5890 instruct membar_release_lock()
 5891 %{
 5892   match(MemBarReleaseLock);
 5893   ins_cost(0);
 5894 
 5895   size(0);
 5896   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5897   ins_encode();
 5898   ins_pipe(empty);
 5899 %}
 5900 
 5901 instruct membar_volatile(rFlagsReg cr) %{
 5902   match(MemBarVolatile);
 5903   effect(KILL cr);
 5904   ins_cost(400);
 5905 
 5906   format %{
 5907     $$template
 5908     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5909   %}
 5910   ins_encode %{
 5911     __ membar(Assembler::StoreLoad);
 5912   %}
 5913   ins_pipe(pipe_slow);
 5914 %}
 5915 
 5916 instruct unnecessary_membar_volatile()
 5917 %{
 5918   match(MemBarVolatile);
 5919   predicate(Matcher::post_store_load_barrier(n));
 5920   ins_cost(0);
 5921 
 5922   size(0);
 5923   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5924   ins_encode();
 5925   ins_pipe(empty);
 5926 %}
 5927 
 5928 instruct membar_storestore() %{
 5929   match(MemBarStoreStore);
 5930   match(StoreStoreFence);
 5931   ins_cost(0);
 5932 
 5933   size(0);
 5934   format %{ "MEMBAR-storestore (empty encoding)" %}
 5935   ins_encode( );
 5936   ins_pipe(empty);
 5937 %}
 5938 
 5939 //----------Move Instructions--------------------------------------------------
 5940 
 5941 instruct castX2P(rRegP dst, rRegL src)
 5942 %{
 5943   match(Set dst (CastX2P src));
 5944 
 5945   format %{ "movq    $dst, $src\t# long->ptr" %}
 5946   ins_encode %{
 5947     if ($dst$$reg != $src$$reg) {
 5948       __ movptr($dst$$Register, $src$$Register);
 5949     }
 5950   %}
 5951   ins_pipe(ialu_reg_reg); // XXX
 5952 %}
 5953 
 5954 instruct castP2X(rRegL dst, rRegP src)
 5955 %{
 5956   match(Set dst (CastP2X src));
 5957 
 5958   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5959   ins_encode %{
 5960     if ($dst$$reg != $src$$reg) {
 5961       __ movptr($dst$$Register, $src$$Register);
 5962     }
 5963   %}
 5964   ins_pipe(ialu_reg_reg); // XXX
 5965 %}
 5966 
 5967 // Convert oop into int for vectors alignment masking
 5968 instruct convP2I(rRegI dst, rRegP src)
 5969 %{
 5970   match(Set dst (ConvL2I (CastP2X src)));
 5971 
 5972   format %{ "movl    $dst, $src\t# ptr -> int" %}
 5973   ins_encode %{
 5974     __ movl($dst$$Register, $src$$Register);
 5975   %}
 5976   ins_pipe(ialu_reg_reg); // XXX
 5977 %}
 5978 
 5979 // Convert compressed oop into int for vectors alignment masking
 5980 // in case of 32bit oops (heap < 4Gb).
 5981 instruct convN2I(rRegI dst, rRegN src)
 5982 %{
 5983   predicate(CompressedOops::shift() == 0);
 5984   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 5985 
 5986   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 5987   ins_encode %{
 5988     __ movl($dst$$Register, $src$$Register);
 5989   %}
 5990   ins_pipe(ialu_reg_reg); // XXX
 5991 %}
 5992 
 5993 // Convert oop pointer into compressed form
 5994 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 5995   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 5996   match(Set dst (EncodeP src));
 5997   effect(KILL cr);
 5998   format %{ "encode_heap_oop $dst,$src" %}
 5999   ins_encode %{
 6000     Register s = $src$$Register;
 6001     Register d = $dst$$Register;
 6002     if (s != d) {
 6003       __ movq(d, s);
 6004     }
 6005     __ encode_heap_oop(d);
 6006   %}
 6007   ins_pipe(ialu_reg_long);
 6008 %}
 6009 
 6010 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6011   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6012   match(Set dst (EncodeP src));
 6013   effect(KILL cr);
 6014   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6015   ins_encode %{
 6016     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6017   %}
 6018   ins_pipe(ialu_reg_long);
 6019 %}
 6020 
 6021 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6022   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6023             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6024   match(Set dst (DecodeN src));
 6025   effect(KILL cr);
 6026   format %{ "decode_heap_oop $dst,$src" %}
 6027   ins_encode %{
 6028     Register s = $src$$Register;
 6029     Register d = $dst$$Register;
 6030     if (s != d) {
 6031       __ movq(d, s);
 6032     }
 6033     __ decode_heap_oop(d);
 6034   %}
 6035   ins_pipe(ialu_reg_long);
 6036 %}
 6037 
 6038 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6039   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6040             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6041   match(Set dst (DecodeN src));
 6042   effect(KILL cr);
 6043   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6044   ins_encode %{
 6045     Register s = $src$$Register;
 6046     Register d = $dst$$Register;
 6047     if (s != d) {
 6048       __ decode_heap_oop_not_null(d, s);
 6049     } else {
 6050       __ decode_heap_oop_not_null(d);
 6051     }
 6052   %}
 6053   ins_pipe(ialu_reg_long);
 6054 %}
 6055 
 6056 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6057   match(Set dst (EncodePKlass src));
 6058   effect(TEMP dst, KILL cr);
 6059   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6060   ins_encode %{
 6061     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6062   %}
 6063   ins_pipe(ialu_reg_long);
 6064 %}
 6065 
 6066 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6067   match(Set dst (DecodeNKlass src));
 6068   effect(TEMP dst, KILL cr);
 6069   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6070   ins_encode %{
 6071     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6072   %}
 6073   ins_pipe(ialu_reg_long);
 6074 %}
 6075 
 6076 //----------Conditional Move---------------------------------------------------
 6077 // Jump
 6078 // dummy instruction for generating temp registers
 6079 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6080   match(Jump (LShiftL switch_val shift));
 6081   ins_cost(350);
 6082   predicate(false);
 6083   effect(TEMP dest);
 6084 
 6085   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6086             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6087   ins_encode %{
 6088     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6089     // to do that and the compiler is using that register as one it can allocate.
 6090     // So we build it all by hand.
 6091     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6092     // ArrayAddress dispatch(table, index);
 6093     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6094     __ lea($dest$$Register, $constantaddress);
 6095     __ jmp(dispatch);
 6096   %}
 6097   ins_pipe(pipe_jmp);
 6098 %}
 6099 
 6100 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6101   match(Jump (AddL (LShiftL switch_val shift) offset));
 6102   ins_cost(350);
 6103   effect(TEMP dest);
 6104 
 6105   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6106             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6107   ins_encode %{
 6108     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6109     // to do that and the compiler is using that register as one it can allocate.
 6110     // So we build it all by hand.
 6111     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6112     // ArrayAddress dispatch(table, index);
 6113     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6114     __ lea($dest$$Register, $constantaddress);
 6115     __ jmp(dispatch);
 6116   %}
 6117   ins_pipe(pipe_jmp);
 6118 %}
 6119 
 6120 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6121   match(Jump switch_val);
 6122   ins_cost(350);
 6123   effect(TEMP dest);
 6124 
 6125   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6126             "jmp     [$dest + $switch_val]\n\t" %}
 6127   ins_encode %{
 6128     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6129     // to do that and the compiler is using that register as one it can allocate.
 6130     // So we build it all by hand.
 6131     // Address index(noreg, switch_reg, Address::times_1);
 6132     // ArrayAddress dispatch(table, index);
 6133     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6134     __ lea($dest$$Register, $constantaddress);
 6135     __ jmp(dispatch);
 6136   %}
 6137   ins_pipe(pipe_jmp);
 6138 %}
 6139 
 6140 // Conditional move
 6141 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6142 %{
 6143   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6144   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6145 
 6146   ins_cost(100); // XXX
 6147   format %{ "setbn$cop $dst\t# signed, int" %}
 6148   ins_encode %{
 6149     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6150     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6151   %}
 6152   ins_pipe(ialu_reg);
 6153 %}
 6154 
 6155 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6156 %{
 6157   predicate(!UseAPX);
 6158   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6159 
 6160   ins_cost(200); // XXX
 6161   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6162   ins_encode %{
 6163     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6164   %}
 6165   ins_pipe(pipe_cmov_reg);
 6166 %}
 6167 
 6168 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6169 %{
 6170   predicate(UseAPX);
 6171   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6172 
 6173   ins_cost(200);
 6174   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6175   ins_encode %{
 6176     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6177   %}
 6178   ins_pipe(pipe_cmov_reg);
 6179 %}
 6180 
 6181 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6182 %{
 6183   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6184   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6185 
 6186   ins_cost(100); // XXX
 6187   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6188   ins_encode %{
 6189     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6190     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6191   %}
 6192   ins_pipe(ialu_reg);
 6193 %}
 6194 
 6195 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6196   predicate(!UseAPX);
 6197   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6198 
 6199   ins_cost(200); // XXX
 6200   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6201   ins_encode %{
 6202     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6203   %}
 6204   ins_pipe(pipe_cmov_reg);
 6205 %}
 6206 
 6207 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6208   predicate(UseAPX);
 6209   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6210 
 6211   ins_cost(200);
 6212   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6213   ins_encode %{
 6214     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6215   %}
 6216   ins_pipe(pipe_cmov_reg);
 6217 %}
 6218 
 6219 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6220 %{
 6221   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6222   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6223 
 6224   ins_cost(100); // XXX
 6225   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6226   ins_encode %{
 6227     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6228     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6229   %}
 6230   ins_pipe(ialu_reg);
 6231 %}
 6232 
 6233 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6234   predicate(!UseAPX);
 6235   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6236   ins_cost(200);
 6237   expand %{
 6238     cmovI_regU(cop, cr, dst, src);
 6239   %}
 6240 %}
 6241 
 6242 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6243   predicate(UseAPX);
 6244   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6245   ins_cost(200);
 6246   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6247   ins_encode %{
 6248     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6249   %}
 6250   ins_pipe(pipe_cmov_reg);
 6251 %}
 6252 
 6253 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6254   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6255   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6256 
 6257   ins_cost(200); // XXX
 6258   format %{ "cmovpl  $dst, $src\n\t"
 6259             "cmovnel $dst, $src" %}
 6260   ins_encode %{
 6261     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6262     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6263   %}
 6264   ins_pipe(pipe_cmov_reg);
 6265 %}
 6266 
 6267 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6268   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6269   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6270   effect(TEMP dst);
 6271 
 6272   ins_cost(200);
 6273   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6274             "cmovnel  $dst, $src2" %}
 6275   ins_encode %{
 6276     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6277     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6278   %}
 6279   ins_pipe(pipe_cmov_reg);
 6280 %}
 6281 
 6282 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6283 // inputs of the CMove
 6284 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6285   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6286   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6287   effect(TEMP dst);
 6288 
 6289   ins_cost(200); // XXX
 6290   format %{ "cmovpl  $dst, $src\n\t"
 6291             "cmovnel $dst, $src" %}
 6292   ins_encode %{
 6293     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6294     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6295   %}
 6296   ins_pipe(pipe_cmov_reg);
 6297 %}
 6298 
 6299 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6300 // and parity flag bit is set if any of the operand is a NaN.
 6301 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6302   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6303   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6304   effect(TEMP dst);
 6305 
 6306   ins_cost(200);
 6307   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6308             "cmovnel  $dst, $src2" %}
 6309   ins_encode %{
 6310     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6311     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6312   %}
 6313   ins_pipe(pipe_cmov_reg);
 6314 %}
 6315 
 6316 // Conditional move
 6317 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6318   predicate(!UseAPX);
 6319   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6320 
 6321   ins_cost(250); // XXX
 6322   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6323   ins_encode %{
 6324     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6325   %}
 6326   ins_pipe(pipe_cmov_mem);
 6327 %}
 6328 
 6329 // Conditional move
 6330 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6331 %{
 6332   predicate(UseAPX);
 6333   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6334 
 6335   ins_cost(250);
 6336   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6337   ins_encode %{
 6338     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6339   %}
 6340   ins_pipe(pipe_cmov_mem);
 6341 %}
 6342 
 6343 // Conditional move
 6344 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6345 %{
 6346   predicate(!UseAPX);
 6347   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6348 
 6349   ins_cost(250); // XXX
 6350   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6351   ins_encode %{
 6352     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6353   %}
 6354   ins_pipe(pipe_cmov_mem);
 6355 %}
 6356 
 6357 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6358   predicate(!UseAPX);
 6359   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6360   ins_cost(250);
 6361   expand %{
 6362     cmovI_memU(cop, cr, dst, src);
 6363   %}
 6364 %}
 6365 
 6366 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6367 %{
 6368   predicate(UseAPX);
 6369   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6370 
 6371   ins_cost(250);
 6372   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6373   ins_encode %{
 6374     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6375   %}
 6376   ins_pipe(pipe_cmov_mem);
 6377 %}
 6378 
 6379 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2) 
 6380 %{
 6381   predicate(UseAPX);
 6382   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6383   ins_cost(250);
 6384   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6385   ins_encode %{
 6386     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6387   %}
 6388   ins_pipe(pipe_cmov_mem);
 6389 %}
 6390 
 6391 // Conditional move
 6392 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6393 %{
 6394   predicate(!UseAPX);
 6395   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6396 
 6397   ins_cost(200); // XXX
 6398   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6399   ins_encode %{
 6400     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6401   %}
 6402   ins_pipe(pipe_cmov_reg);
 6403 %}
 6404 
 6405 // Conditional move ndd
 6406 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6407 %{
 6408   predicate(UseAPX);
 6409   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6410 
 6411   ins_cost(200);
 6412   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6413   ins_encode %{
 6414     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6415   %}
 6416   ins_pipe(pipe_cmov_reg);
 6417 %}
 6418 
 6419 // Conditional move
 6420 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6421 %{
 6422   predicate(!UseAPX);
 6423   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6424 
 6425   ins_cost(200); // XXX
 6426   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6427   ins_encode %{
 6428     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6429   %}
 6430   ins_pipe(pipe_cmov_reg);
 6431 %}
 6432 
 6433 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6434   predicate(!UseAPX);
 6435   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6436   ins_cost(200);
 6437   expand %{
 6438     cmovN_regU(cop, cr, dst, src);
 6439   %}
 6440 %}
 6441 
 6442 // Conditional move ndd
 6443 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6444 %{
 6445   predicate(UseAPX);
 6446   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6447 
 6448   ins_cost(200);
 6449   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6450   ins_encode %{
 6451     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6452   %}
 6453   ins_pipe(pipe_cmov_reg);
 6454 %}
 6455 
 6456 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6457   predicate(UseAPX);
 6458   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6459   ins_cost(200);
 6460   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6461   ins_encode %{
 6462     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6463   %}
 6464   ins_pipe(pipe_cmov_reg);
 6465 %}
 6466 
 6467 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6468   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6469   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6470 
 6471   ins_cost(200); // XXX
 6472   format %{ "cmovpl  $dst, $src\n\t"
 6473             "cmovnel $dst, $src" %}
 6474   ins_encode %{
 6475     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6476     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6477   %}
 6478   ins_pipe(pipe_cmov_reg);
 6479 %}
 6480 
 6481 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6482 // inputs of the CMove
 6483 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6484   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6485   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6486 
 6487   ins_cost(200); // XXX
 6488   format %{ "cmovpl  $dst, $src\n\t"
 6489             "cmovnel $dst, $src" %}
 6490   ins_encode %{
 6491     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6492     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6493   %}
 6494   ins_pipe(pipe_cmov_reg);
 6495 %}
 6496 
 6497 // Conditional move
 6498 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6499 %{
 6500   predicate(!UseAPX);
 6501   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6502 
 6503   ins_cost(200); // XXX
 6504   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6505   ins_encode %{
 6506     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6507   %}
 6508   ins_pipe(pipe_cmov_reg);  // XXX
 6509 %}
 6510 
 6511 // Conditional move ndd
 6512 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6513 %{
 6514   predicate(UseAPX);
 6515   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6516 
 6517   ins_cost(200);
 6518   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6519   ins_encode %{
 6520     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6521   %}
 6522   ins_pipe(pipe_cmov_reg);
 6523 %}
 6524 
 6525 // Conditional move
 6526 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6527 %{
 6528   predicate(!UseAPX);
 6529   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6530 
 6531   ins_cost(200); // XXX
 6532   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6533   ins_encode %{
 6534     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6535   %}
 6536   ins_pipe(pipe_cmov_reg); // XXX
 6537 %}
 6538 
 6539 // Conditional move ndd
 6540 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6541 %{
 6542   predicate(UseAPX);
 6543   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6544 
 6545   ins_cost(200);
 6546   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6547   ins_encode %{
 6548     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6549   %}
 6550   ins_pipe(pipe_cmov_reg);
 6551 %}
 6552 
 6553 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6554   predicate(!UseAPX);
 6555   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6556   ins_cost(200);
 6557   expand %{
 6558     cmovP_regU(cop, cr, dst, src);
 6559   %}
 6560 %}
 6561 
 6562 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6563   predicate(UseAPX);
 6564   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6565   ins_cost(200);
 6566   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6567   ins_encode %{
 6568     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6569   %}
 6570   ins_pipe(pipe_cmov_reg);
 6571 %}
 6572 
 6573 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6574   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6575   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6576 
 6577   ins_cost(200); // XXX
 6578   format %{ "cmovpq  $dst, $src\n\t"
 6579             "cmovneq $dst, $src" %}
 6580   ins_encode %{
 6581     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6582     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6583   %}
 6584   ins_pipe(pipe_cmov_reg);
 6585 %}
 6586 
 6587 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6588   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6589   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6590   effect(TEMP dst);
 6591 
 6592   ins_cost(200);
 6593   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6594             "cmovneq  $dst, $src2" %}
 6595   ins_encode %{
 6596     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6597     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6598   %}
 6599   ins_pipe(pipe_cmov_reg);
 6600 %}
 6601 
 6602 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6603 // inputs of the CMove
 6604 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6605   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6606   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6607 
 6608   ins_cost(200); // XXX
 6609   format %{ "cmovpq  $dst, $src\n\t"
 6610             "cmovneq $dst, $src" %}
 6611   ins_encode %{
 6612     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6613     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6614   %}
 6615   ins_pipe(pipe_cmov_reg);
 6616 %}
 6617 
 6618 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6619   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6620   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6621   effect(TEMP dst);
 6622 
 6623   ins_cost(200);
 6624   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6625             "cmovneq  $dst, $src2" %}
 6626   ins_encode %{
 6627     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6628     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6629   %}
 6630   ins_pipe(pipe_cmov_reg);
 6631 %}
 6632 
 6633 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6634 %{
 6635   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6636   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6637 
 6638   ins_cost(100); // XXX
 6639   format %{ "setbn$cop $dst\t# signed, long" %}
 6640   ins_encode %{
 6641     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6642     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6643   %}
 6644   ins_pipe(ialu_reg);
 6645 %}
 6646 
 6647 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6648 %{
 6649   predicate(!UseAPX);
 6650   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6651 
 6652   ins_cost(200); // XXX
 6653   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6654   ins_encode %{
 6655     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6656   %}
 6657   ins_pipe(pipe_cmov_reg);  // XXX
 6658 %}
 6659 
 6660 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6661 %{
 6662   predicate(UseAPX);
 6663   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6664 
 6665   ins_cost(200);
 6666   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6667   ins_encode %{
 6668     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6669   %}
 6670   ins_pipe(pipe_cmov_reg);
 6671 %}
 6672 
 6673 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6674 %{
 6675   predicate(!UseAPX);
 6676   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6677 
 6678   ins_cost(200); // XXX
 6679   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6680   ins_encode %{
 6681     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6682   %}
 6683   ins_pipe(pipe_cmov_mem);  // XXX
 6684 %}
 6685 
 6686 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6687 %{
 6688   predicate(UseAPX);
 6689   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6690 
 6691   ins_cost(200);
 6692   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6693   ins_encode %{
 6694     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6695   %}
 6696   ins_pipe(pipe_cmov_mem);
 6697 %}
 6698 
 6699 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6700 %{
 6701   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6702   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6703 
 6704   ins_cost(100); // XXX
 6705   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6706   ins_encode %{
 6707     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6708     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6709   %}
 6710   ins_pipe(ialu_reg);
 6711 %}
 6712 
 6713 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6714 %{
 6715   predicate(!UseAPX);
 6716   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6717 
 6718   ins_cost(200); // XXX
 6719   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6720   ins_encode %{
 6721     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6722   %}
 6723   ins_pipe(pipe_cmov_reg); // XXX
 6724 %}
 6725 
 6726 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6727 %{
 6728   predicate(UseAPX);
 6729   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6730 
 6731   ins_cost(200);
 6732   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6733   ins_encode %{
 6734     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6735   %}
 6736   ins_pipe(pipe_cmov_reg);
 6737 %}
 6738 
 6739 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6740 %{
 6741   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6742   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6743 
 6744   ins_cost(100); // XXX
 6745   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6746   ins_encode %{
 6747     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6748     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6749   %}
 6750   ins_pipe(ialu_reg);
 6751 %}
 6752 
 6753 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6754   predicate(!UseAPX);
 6755   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6756   ins_cost(200);
 6757   expand %{
 6758     cmovL_regU(cop, cr, dst, src);
 6759   %}
 6760 %}
 6761 
 6762 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2) 
 6763 %{
 6764   predicate(UseAPX);
 6765   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6766   ins_cost(200);
 6767   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6768   ins_encode %{
 6769     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6770   %}
 6771   ins_pipe(pipe_cmov_reg);
 6772 %}
 6773 
 6774 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6775   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6776   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6777 
 6778   ins_cost(200); // XXX
 6779   format %{ "cmovpq  $dst, $src\n\t"
 6780             "cmovneq $dst, $src" %}
 6781   ins_encode %{
 6782     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6783     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6784   %}
 6785   ins_pipe(pipe_cmov_reg);
 6786 %}
 6787 
 6788 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6789   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6790   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6791   effect(TEMP dst);
 6792 
 6793   ins_cost(200);
 6794   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6795             "cmovneq  $dst, $src2" %}
 6796   ins_encode %{
 6797     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6798     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6799   %}
 6800   ins_pipe(pipe_cmov_reg);
 6801 %}
 6802 
 6803 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6804 // inputs of the CMove
 6805 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6806   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6807   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6808 
 6809   ins_cost(200); // XXX
 6810   format %{ "cmovpq  $dst, $src\n\t"
 6811             "cmovneq $dst, $src" %}
 6812   ins_encode %{
 6813     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6814     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6815   %}
 6816   ins_pipe(pipe_cmov_reg);
 6817 %}
 6818 
 6819 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6820   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6821   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6822   effect(TEMP dst);
 6823 
 6824   ins_cost(200);
 6825   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6826             "cmovneq $dst, $src2" %}
 6827   ins_encode %{
 6828     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6829     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6830   %}
 6831   ins_pipe(pipe_cmov_reg);
 6832 %}
 6833 
 6834 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6835 %{
 6836   predicate(!UseAPX);
 6837   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6838 
 6839   ins_cost(200); // XXX
 6840   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6841   ins_encode %{
 6842     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6843   %}
 6844   ins_pipe(pipe_cmov_mem); // XXX
 6845 %}
 6846 
 6847 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6848   predicate(!UseAPX);
 6849   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6850   ins_cost(200);
 6851   expand %{
 6852     cmovL_memU(cop, cr, dst, src);
 6853   %}
 6854 %}
 6855 
 6856 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6857 %{
 6858   predicate(UseAPX);
 6859   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6860 
 6861   ins_cost(200);
 6862   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6863   ins_encode %{
 6864     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6865   %}
 6866   ins_pipe(pipe_cmov_mem);
 6867 %}
 6868 
 6869 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2) 
 6870 %{
 6871   predicate(UseAPX);
 6872   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6873   ins_cost(200);
 6874   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6875   ins_encode %{
 6876     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6877   %}
 6878   ins_pipe(pipe_cmov_mem);
 6879 %}
 6880 
 6881 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6882 %{
 6883   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6884 
 6885   ins_cost(200); // XXX
 6886   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6887             "movss     $dst, $src\n"
 6888     "skip:" %}
 6889   ins_encode %{
 6890     Label Lskip;
 6891     // Invert sense of branch from sense of CMOV
 6892     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6893     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6894     __ bind(Lskip);
 6895   %}
 6896   ins_pipe(pipe_slow);
 6897 %}
 6898 
 6899 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6900 %{
 6901   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6902 
 6903   ins_cost(200); // XXX
 6904   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6905             "movss     $dst, $src\n"
 6906     "skip:" %}
 6907   ins_encode %{
 6908     Label Lskip;
 6909     // Invert sense of branch from sense of CMOV
 6910     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6911     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6912     __ bind(Lskip);
 6913   %}
 6914   ins_pipe(pipe_slow);
 6915 %}
 6916 
 6917 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6918   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6919   ins_cost(200);
 6920   expand %{
 6921     cmovF_regU(cop, cr, dst, src);
 6922   %}
 6923 %}
 6924 
 6925 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6926 %{
 6927   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6928 
 6929   ins_cost(200); // XXX
 6930   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6931             "movsd     $dst, $src\n"
 6932     "skip:" %}
 6933   ins_encode %{
 6934     Label Lskip;
 6935     // Invert sense of branch from sense of CMOV
 6936     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6937     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6938     __ bind(Lskip);
 6939   %}
 6940   ins_pipe(pipe_slow);
 6941 %}
 6942 
 6943 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6944 %{
 6945   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6946 
 6947   ins_cost(200); // XXX
 6948   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6949             "movsd     $dst, $src\n"
 6950     "skip:" %}
 6951   ins_encode %{
 6952     Label Lskip;
 6953     // Invert sense of branch from sense of CMOV
 6954     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6955     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6956     __ bind(Lskip);
 6957   %}
 6958   ins_pipe(pipe_slow);
 6959 %}
 6960 
 6961 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6962   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6963   ins_cost(200);
 6964   expand %{
 6965     cmovD_regU(cop, cr, dst, src);
 6966   %}
 6967 %}
 6968 
 6969 //----------Arithmetic Instructions--------------------------------------------
 6970 //----------Addition Instructions----------------------------------------------
 6971 
 6972 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6973 %{
 6974   predicate(!UseAPX);
 6975   match(Set dst (AddI dst src));
 6976   effect(KILL cr);
 6977   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6978   format %{ "addl    $dst, $src\t# int" %}
 6979   ins_encode %{
 6980     __ addl($dst$$Register, $src$$Register);
 6981   %}
 6982   ins_pipe(ialu_reg_reg);
 6983 %}
 6984 
 6985 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 6986 %{
 6987   predicate(UseAPX);
 6988   match(Set dst (AddI src1 src2));
 6989   effect(KILL cr);
 6990   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6991 
 6992   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 6993   ins_encode %{
 6994     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 6995   %}
 6996   ins_pipe(ialu_reg_reg);
 6997 %}
 6998 
 6999 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7000 %{
 7001   predicate(!UseAPX);
 7002   match(Set dst (AddI dst src));
 7003   effect(KILL cr);
 7004   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7005 
 7006   format %{ "addl    $dst, $src\t# int" %}
 7007   ins_encode %{
 7008     __ addl($dst$$Register, $src$$constant);
 7009   %}
 7010   ins_pipe( ialu_reg );
 7011 %}
 7012 
 7013 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7014 %{
 7015   predicate(UseAPX);
 7016   match(Set dst (AddI src1 src2));
 7017   effect(KILL cr);
 7018   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7019 
 7020   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7021   ins_encode %{
 7022     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7023   %}
 7024   ins_pipe( ialu_reg );
 7025 %}
 7026 
 7027 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7028 %{
 7029   predicate(UseAPX);
 7030   match(Set dst (AddI (LoadI src1) src2));
 7031   effect(KILL cr);
 7032   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7033 
 7034   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7035   ins_encode %{
 7036     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7037   %}
 7038   ins_pipe( ialu_reg );
 7039 %}
 7040 
 7041 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7042 %{
 7043   predicate(!UseAPX);
 7044   match(Set dst (AddI dst (LoadI src)));
 7045   effect(KILL cr);
 7046   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7047 
 7048   ins_cost(150); // XXX
 7049   format %{ "addl    $dst, $src\t# int" %}
 7050   ins_encode %{
 7051     __ addl($dst$$Register, $src$$Address);
 7052   %}
 7053   ins_pipe(ialu_reg_mem);
 7054 %}
 7055 
 7056 instruct addI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 7057 %{
 7058   predicate(UseAPX);
 7059   match(Set dst (AddI (LoadI src1) src2));
 7060   effect(KILL cr);
 7061   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7062 
 7063   ins_cost(150);
 7064   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7065   ins_encode %{
 7066     __ eaddl($dst$$Register, $src1$$Address, $src2$$Register, false);
 7067   %}
 7068   ins_pipe(ialu_reg_mem);
 7069 %}
 7070 
 7071 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7072 %{
 7073   predicate(UseAPX);
 7074   match(Set dst (AddI src1 (LoadI src2)));
 7075   effect(KILL cr);
 7076   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7077 
 7078   ins_cost(150);
 7079   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7080   ins_encode %{
 7081     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7082   %}
 7083   ins_pipe(ialu_reg_mem);
 7084 %}
 7085 
 7086 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7087 %{
 7088   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7089   effect(KILL cr);
 7090   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7091 
 7092   ins_cost(150); // XXX
 7093   format %{ "addl    $dst, $src\t# int" %}
 7094   ins_encode %{
 7095     __ addl($dst$$Address, $src$$Register);
 7096   %}
 7097   ins_pipe(ialu_mem_reg);
 7098 %}
 7099 
 7100 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7101 %{
 7102   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7103   effect(KILL cr);
 7104   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7105 
 7106 
 7107   ins_cost(125); // XXX
 7108   format %{ "addl    $dst, $src\t# int" %}
 7109   ins_encode %{
 7110     __ addl($dst$$Address, $src$$constant);
 7111   %}
 7112   ins_pipe(ialu_mem_imm);
 7113 %}
 7114 
 7115 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7116 %{
 7117   predicate(!UseAPX && UseIncDec);
 7118   match(Set dst (AddI dst src));
 7119   effect(KILL cr);
 7120 
 7121   format %{ "incl    $dst\t# int" %}
 7122   ins_encode %{
 7123     __ incrementl($dst$$Register);
 7124   %}
 7125   ins_pipe(ialu_reg);
 7126 %}
 7127 
 7128 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7129 %{
 7130   predicate(UseAPX && UseIncDec);
 7131   match(Set dst (AddI src val));
 7132   effect(KILL cr);
 7133 
 7134   format %{ "eincl    $dst, $src\t# int ndd" %}
 7135   ins_encode %{
 7136     __ eincl($dst$$Register, $src$$Register, false);
 7137   %}
 7138   ins_pipe(ialu_reg);
 7139 %}
 7140 
 7141 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7142 %{
 7143   predicate(UseAPX && UseIncDec);
 7144   match(Set dst (AddI (LoadI src) val));
 7145   effect(KILL cr);
 7146 
 7147   format %{ "eincl    $dst, $src\t# int ndd" %}
 7148   ins_encode %{
 7149     __ eincl($dst$$Register, $src$$Address, false);
 7150   %}
 7151   ins_pipe(ialu_reg);
 7152 %}
 7153 
 7154 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7155 %{
 7156   predicate(UseIncDec);
 7157   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7158   effect(KILL cr);
 7159 
 7160   ins_cost(125); // XXX
 7161   format %{ "incl    $dst\t# int" %}
 7162   ins_encode %{
 7163     __ incrementl($dst$$Address);
 7164   %}
 7165   ins_pipe(ialu_mem_imm);
 7166 %}
 7167 
 7168 // XXX why does that use AddI
 7169 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7170 %{
 7171   predicate(!UseAPX && UseIncDec);
 7172   match(Set dst (AddI dst src));
 7173   effect(KILL cr);
 7174 
 7175   format %{ "decl    $dst\t# int" %}
 7176   ins_encode %{
 7177     __ decrementl($dst$$Register);
 7178   %}
 7179   ins_pipe(ialu_reg);
 7180 %}
 7181 
 7182 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7183 %{
 7184   predicate(UseAPX && UseIncDec);
 7185   match(Set dst (AddI src val));
 7186   effect(KILL cr);
 7187 
 7188   format %{ "edecl    $dst, $src\t# int ndd" %}
 7189   ins_encode %{
 7190     __ edecl($dst$$Register, $src$$Register, false);
 7191   %}
 7192   ins_pipe(ialu_reg);
 7193 %}
 7194 
 7195 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7196 %{
 7197   predicate(UseAPX && UseIncDec);
 7198   match(Set dst (AddI (LoadI src) val));
 7199   effect(KILL cr);
 7200 
 7201   format %{ "edecl    $dst, $src\t# int ndd" %}
 7202   ins_encode %{
 7203     __ edecl($dst$$Register, $src$$Address, false);
 7204   %}
 7205   ins_pipe(ialu_reg);
 7206 %}
 7207 
 7208 // XXX why does that use AddI
 7209 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7210 %{
 7211   predicate(UseIncDec);
 7212   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7213   effect(KILL cr);
 7214 
 7215   ins_cost(125); // XXX
 7216   format %{ "decl    $dst\t# int" %}
 7217   ins_encode %{
 7218     __ decrementl($dst$$Address);
 7219   %}
 7220   ins_pipe(ialu_mem_imm);
 7221 %}
 7222 
 7223 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7224 %{
 7225   predicate(VM_Version::supports_fast_2op_lea());
 7226   match(Set dst (AddI (LShiftI index scale) disp));
 7227 
 7228   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7229   ins_encode %{
 7230     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7231     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7232   %}
 7233   ins_pipe(ialu_reg_reg);
 7234 %}
 7235 
 7236 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7237 %{
 7238   predicate(VM_Version::supports_fast_3op_lea());
 7239   match(Set dst (AddI (AddI base index) disp));
 7240 
 7241   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7242   ins_encode %{
 7243     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7244   %}
 7245   ins_pipe(ialu_reg_reg);
 7246 %}
 7247 
 7248 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7249 %{
 7250   predicate(VM_Version::supports_fast_2op_lea());
 7251   match(Set dst (AddI base (LShiftI index scale)));
 7252 
 7253   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7254   ins_encode %{
 7255     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7256     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7257   %}
 7258   ins_pipe(ialu_reg_reg);
 7259 %}
 7260 
 7261 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7262 %{
 7263   predicate(VM_Version::supports_fast_3op_lea());
 7264   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7265 
 7266   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7267   ins_encode %{
 7268     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7269     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7270   %}
 7271   ins_pipe(ialu_reg_reg);
 7272 %}
 7273 
 7274 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7275 %{
 7276   predicate(!UseAPX);
 7277   match(Set dst (AddL dst src));
 7278   effect(KILL cr);
 7279   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7280 
 7281   format %{ "addq    $dst, $src\t# long" %}
 7282   ins_encode %{
 7283     __ addq($dst$$Register, $src$$Register);
 7284   %}
 7285   ins_pipe(ialu_reg_reg);
 7286 %}
 7287 
 7288 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7289 %{
 7290   predicate(UseAPX);
 7291   match(Set dst (AddL src1 src2));
 7292   effect(KILL cr);
 7293   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7294 
 7295   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7296   ins_encode %{
 7297     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7298   %}
 7299   ins_pipe(ialu_reg_reg);
 7300 %}
 7301 
 7302 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7303 %{
 7304   predicate(!UseAPX);
 7305   match(Set dst (AddL dst src));
 7306   effect(KILL cr);
 7307   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7308 
 7309   format %{ "addq    $dst, $src\t# long" %}
 7310   ins_encode %{
 7311     __ addq($dst$$Register, $src$$constant);
 7312   %}
 7313   ins_pipe( ialu_reg );
 7314 %}
 7315 
 7316 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7317 %{
 7318   predicate(UseAPX);
 7319   match(Set dst (AddL src1 src2));
 7320   effect(KILL cr);
 7321   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7322 
 7323   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7324   ins_encode %{
 7325     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7326   %}
 7327   ins_pipe( ialu_reg );
 7328 %}
 7329 
 7330 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7331 %{
 7332   predicate(UseAPX);
 7333   match(Set dst (AddL (LoadL src1) src2));
 7334   effect(KILL cr);
 7335   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7336 
 7337   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7338   ins_encode %{
 7339     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7340   %}
 7341   ins_pipe( ialu_reg );
 7342 %}
 7343 
 7344 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7345 %{
 7346   predicate(!UseAPX);
 7347   match(Set dst (AddL dst (LoadL src)));
 7348   effect(KILL cr);
 7349   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7350 
 7351   ins_cost(150); // XXX
 7352   format %{ "addq    $dst, $src\t# long" %}
 7353   ins_encode %{
 7354     __ addq($dst$$Register, $src$$Address);
 7355   %}
 7356   ins_pipe(ialu_reg_mem);
 7357 %}
 7358 
 7359 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7360 %{
 7361   predicate(UseAPX);
 7362   match(Set dst (AddL src1 (LoadL src2)));
 7363   effect(KILL cr);
 7364   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7365 
 7366   ins_cost(150);
 7367   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7368   ins_encode %{
 7369     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7370   %}
 7371   ins_pipe(ialu_reg_mem);
 7372 %}
 7373 
 7374 instruct addL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 7375 %{
 7376   predicate(UseAPX);
 7377   match(Set dst (AddL (LoadL src1) src2));
 7378   effect(KILL cr);
 7379   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7380 
 7381   ins_cost(150);
 7382   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7383   ins_encode %{
 7384     __ eaddq($dst$$Register, $src1$$Address, $src2$$Register, false);
 7385   %}
 7386   ins_pipe(ialu_reg_mem);
 7387 %}
 7388 
 7389 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7390 %{
 7391   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7392   effect(KILL cr);
 7393   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7394 
 7395   ins_cost(150); // XXX
 7396   format %{ "addq    $dst, $src\t# long" %}
 7397   ins_encode %{
 7398     __ addq($dst$$Address, $src$$Register);
 7399   %}
 7400   ins_pipe(ialu_mem_reg);
 7401 %}
 7402 
 7403 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7404 %{
 7405   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7406   effect(KILL cr);
 7407   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7408 
 7409   ins_cost(125); // XXX
 7410   format %{ "addq    $dst, $src\t# long" %}
 7411   ins_encode %{
 7412     __ addq($dst$$Address, $src$$constant);
 7413   %}
 7414   ins_pipe(ialu_mem_imm);
 7415 %}
 7416 
 7417 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7418 %{
 7419   predicate(!UseAPX && UseIncDec);
 7420   match(Set dst (AddL dst src));
 7421   effect(KILL cr);
 7422 
 7423   format %{ "incq    $dst\t# long" %}
 7424   ins_encode %{
 7425     __ incrementq($dst$$Register);
 7426   %}
 7427   ins_pipe(ialu_reg);
 7428 %}
 7429 
 7430 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7431 %{
 7432   predicate(UseAPX && UseIncDec);
 7433   match(Set dst (AddL src val));
 7434   effect(KILL cr);
 7435 
 7436   format %{ "eincq    $dst, $src\t# long ndd" %}
 7437   ins_encode %{
 7438     __ eincq($dst$$Register, $src$$Register, false);
 7439   %}
 7440   ins_pipe(ialu_reg);
 7441 %}
 7442 
 7443 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7444 %{
 7445   predicate(UseAPX && UseIncDec);
 7446   match(Set dst (AddL (LoadL src) val));
 7447   effect(KILL cr);
 7448 
 7449   format %{ "eincq    $dst, $src\t# long ndd" %}
 7450   ins_encode %{
 7451     __ eincq($dst$$Register, $src$$Address, false);
 7452   %}
 7453   ins_pipe(ialu_reg);
 7454 %}
 7455 
 7456 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7457 %{
 7458   predicate(UseIncDec);
 7459   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7460   effect(KILL cr);
 7461 
 7462   ins_cost(125); // XXX
 7463   format %{ "incq    $dst\t# long" %}
 7464   ins_encode %{
 7465     __ incrementq($dst$$Address);
 7466   %}
 7467   ins_pipe(ialu_mem_imm);
 7468 %}
 7469 
 7470 // XXX why does that use AddL
 7471 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7472 %{
 7473   predicate(!UseAPX && UseIncDec);
 7474   match(Set dst (AddL dst src));
 7475   effect(KILL cr);
 7476 
 7477   format %{ "decq    $dst\t# long" %}
 7478   ins_encode %{
 7479     __ decrementq($dst$$Register);
 7480   %}
 7481   ins_pipe(ialu_reg);
 7482 %}
 7483 
 7484 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7485 %{
 7486   predicate(UseAPX && UseIncDec);
 7487   match(Set dst (AddL src val));
 7488   effect(KILL cr);
 7489 
 7490   format %{ "edecq    $dst, $src\t# long ndd" %}
 7491   ins_encode %{
 7492     __ edecq($dst$$Register, $src$$Register, false);
 7493   %}
 7494   ins_pipe(ialu_reg);
 7495 %}
 7496 
 7497 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7498 %{
 7499   predicate(UseAPX && UseIncDec);
 7500   match(Set dst (AddL (LoadL src) val));
 7501   effect(KILL cr);
 7502 
 7503   format %{ "edecq    $dst, $src\t# long ndd" %}
 7504   ins_encode %{
 7505     __ edecq($dst$$Register, $src$$Address, false);
 7506   %}
 7507   ins_pipe(ialu_reg);
 7508 %}
 7509 
 7510 // XXX why does that use AddL
 7511 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7512 %{
 7513   predicate(UseIncDec);
 7514   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7515   effect(KILL cr);
 7516 
 7517   ins_cost(125); // XXX
 7518   format %{ "decq    $dst\t# long" %}
 7519   ins_encode %{
 7520     __ decrementq($dst$$Address);
 7521   %}
 7522   ins_pipe(ialu_mem_imm);
 7523 %}
 7524 
 7525 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7526 %{
 7527   predicate(VM_Version::supports_fast_2op_lea());
 7528   match(Set dst (AddL (LShiftL index scale) disp));
 7529 
 7530   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7531   ins_encode %{
 7532     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7533     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7534   %}
 7535   ins_pipe(ialu_reg_reg);
 7536 %}
 7537 
 7538 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7539 %{
 7540   predicate(VM_Version::supports_fast_3op_lea());
 7541   match(Set dst (AddL (AddL base index) disp));
 7542 
 7543   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7544   ins_encode %{
 7545     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7546   %}
 7547   ins_pipe(ialu_reg_reg);
 7548 %}
 7549 
 7550 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7551 %{
 7552   predicate(VM_Version::supports_fast_2op_lea());
 7553   match(Set dst (AddL base (LShiftL index scale)));
 7554 
 7555   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7556   ins_encode %{
 7557     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7558     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7559   %}
 7560   ins_pipe(ialu_reg_reg);
 7561 %}
 7562 
 7563 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7564 %{
 7565   predicate(VM_Version::supports_fast_3op_lea());
 7566   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7567 
 7568   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7569   ins_encode %{
 7570     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7571     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7572   %}
 7573   ins_pipe(ialu_reg_reg);
 7574 %}
 7575 
 7576 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7577 %{
 7578   match(Set dst (AddP dst src));
 7579   effect(KILL cr);
 7580   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7581 
 7582   format %{ "addq    $dst, $src\t# ptr" %}
 7583   ins_encode %{
 7584     __ addq($dst$$Register, $src$$Register);
 7585   %}
 7586   ins_pipe(ialu_reg_reg);
 7587 %}
 7588 
 7589 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7590 %{
 7591   match(Set dst (AddP dst src));
 7592   effect(KILL cr);
 7593   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7594 
 7595   format %{ "addq    $dst, $src\t# ptr" %}
 7596   ins_encode %{
 7597     __ addq($dst$$Register, $src$$constant);
 7598   %}
 7599   ins_pipe( ialu_reg );
 7600 %}
 7601 
 7602 // XXX addP mem ops ????
 7603 
 7604 instruct checkCastPP(rRegP dst)
 7605 %{
 7606   match(Set dst (CheckCastPP dst));
 7607 
 7608   size(0);
 7609   format %{ "# checkcastPP of $dst" %}
 7610   ins_encode(/* empty encoding */);
 7611   ins_pipe(empty);
 7612 %}
 7613 
 7614 instruct castPP(rRegP dst)
 7615 %{
 7616   match(Set dst (CastPP dst));
 7617 
 7618   size(0);
 7619   format %{ "# castPP of $dst" %}
 7620   ins_encode(/* empty encoding */);
 7621   ins_pipe(empty);
 7622 %}
 7623 
 7624 instruct castII(rRegI dst)
 7625 %{
 7626   predicate(VerifyConstraintCasts == 0);
 7627   match(Set dst (CastII dst));
 7628 
 7629   size(0);
 7630   format %{ "# castII of $dst" %}
 7631   ins_encode(/* empty encoding */);
 7632   ins_cost(0);
 7633   ins_pipe(empty);
 7634 %}
 7635 
 7636 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7637 %{
 7638   predicate(VerifyConstraintCasts > 0);
 7639   match(Set dst (CastII dst));
 7640 
 7641   effect(KILL cr);
 7642   format %{ "# cast_checked_II $dst" %}
 7643   ins_encode %{
 7644     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7645   %}
 7646   ins_pipe(pipe_slow);
 7647 %}
 7648 
 7649 instruct castLL(rRegL dst)
 7650 %{
 7651   predicate(VerifyConstraintCasts == 0);
 7652   match(Set dst (CastLL dst));
 7653 
 7654   size(0);
 7655   format %{ "# castLL of $dst" %}
 7656   ins_encode(/* empty encoding */);
 7657   ins_cost(0);
 7658   ins_pipe(empty);
 7659 %}
 7660 
 7661 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7662 %{
 7663   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7664   match(Set dst (CastLL dst));
 7665 
 7666   effect(KILL cr);
 7667   format %{ "# cast_checked_LL $dst" %}
 7668   ins_encode %{
 7669     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7670   %}
 7671   ins_pipe(pipe_slow);
 7672 %}
 7673 
 7674 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7675 %{
 7676   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7677   match(Set dst (CastLL dst));
 7678 
 7679   effect(KILL cr, TEMP tmp);
 7680   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7681   ins_encode %{
 7682     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7683   %}
 7684   ins_pipe(pipe_slow);
 7685 %}
 7686 
 7687 instruct castFF(regF dst)
 7688 %{
 7689   match(Set dst (CastFF dst));
 7690 
 7691   size(0);
 7692   format %{ "# castFF of $dst" %}
 7693   ins_encode(/* empty encoding */);
 7694   ins_cost(0);
 7695   ins_pipe(empty);
 7696 %}
 7697 
 7698 instruct castHH(regF dst)
 7699 %{
 7700   match(Set dst (CastHH dst));
 7701 
 7702   size(0);
 7703   format %{ "# castHH of $dst" %}
 7704   ins_encode(/* empty encoding */);
 7705   ins_cost(0);
 7706   ins_pipe(empty);
 7707 %}
 7708 
 7709 instruct castDD(regD dst)
 7710 %{
 7711   match(Set dst (CastDD dst));
 7712 
 7713   size(0);
 7714   format %{ "# castDD of $dst" %}
 7715   ins_encode(/* empty encoding */);
 7716   ins_cost(0);
 7717   ins_pipe(empty);
 7718 %}
 7719 
 7720 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7721 instruct compareAndSwapP(rRegI res,
 7722                          memory mem_ptr,
 7723                          rax_RegP oldval, rRegP newval,
 7724                          rFlagsReg cr)
 7725 %{
 7726   predicate(n->as_LoadStore()->barrier_data() == 0);
 7727   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7728   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7729   effect(KILL cr, KILL oldval);
 7730 
 7731   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7732             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7733             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7734   ins_encode %{
 7735     __ lock();
 7736     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7737     __ setcc(Assembler::equal, $res$$Register);
 7738   %}
 7739   ins_pipe( pipe_cmpxchg );
 7740 %}
 7741 
 7742 instruct compareAndSwapL(rRegI res,
 7743                          memory mem_ptr,
 7744                          rax_RegL oldval, rRegL newval,
 7745                          rFlagsReg cr)
 7746 %{
 7747   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7748   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7749   effect(KILL cr, KILL oldval);
 7750 
 7751   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7752             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7753             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7754   ins_encode %{
 7755     __ lock();
 7756     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7757     __ setcc(Assembler::equal, $res$$Register);
 7758   %}
 7759   ins_pipe( pipe_cmpxchg );
 7760 %}
 7761 
 7762 instruct compareAndSwapI(rRegI res,
 7763                          memory mem_ptr,
 7764                          rax_RegI oldval, rRegI newval,
 7765                          rFlagsReg cr)
 7766 %{
 7767   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7768   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7769   effect(KILL cr, KILL oldval);
 7770 
 7771   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7772             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7773             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7774   ins_encode %{
 7775     __ lock();
 7776     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7777     __ setcc(Assembler::equal, $res$$Register);
 7778   %}
 7779   ins_pipe( pipe_cmpxchg );
 7780 %}
 7781 
 7782 instruct compareAndSwapB(rRegI res,
 7783                          memory mem_ptr,
 7784                          rax_RegI oldval, rRegI newval,
 7785                          rFlagsReg cr)
 7786 %{
 7787   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7788   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7789   effect(KILL cr, KILL oldval);
 7790 
 7791   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7792             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7793             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7794   ins_encode %{
 7795     __ lock();
 7796     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7797     __ setcc(Assembler::equal, $res$$Register);
 7798   %}
 7799   ins_pipe( pipe_cmpxchg );
 7800 %}
 7801 
 7802 instruct compareAndSwapS(rRegI res,
 7803                          memory mem_ptr,
 7804                          rax_RegI oldval, rRegI newval,
 7805                          rFlagsReg cr)
 7806 %{
 7807   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7808   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7809   effect(KILL cr, KILL oldval);
 7810 
 7811   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7812             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7813             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7814   ins_encode %{
 7815     __ lock();
 7816     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7817     __ setcc(Assembler::equal, $res$$Register);
 7818   %}
 7819   ins_pipe( pipe_cmpxchg );
 7820 %}
 7821 
 7822 instruct compareAndSwapN(rRegI res,
 7823                           memory mem_ptr,
 7824                           rax_RegN oldval, rRegN newval,
 7825                           rFlagsReg cr) %{
 7826   predicate(n->as_LoadStore()->barrier_data() == 0);
 7827   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7828   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7829   effect(KILL cr, KILL oldval);
 7830 
 7831   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7832             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7833             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7834   ins_encode %{
 7835     __ lock();
 7836     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7837     __ setcc(Assembler::equal, $res$$Register);
 7838   %}
 7839   ins_pipe( pipe_cmpxchg );
 7840 %}
 7841 
 7842 instruct compareAndExchangeB(
 7843                          memory mem_ptr,
 7844                          rax_RegI oldval, rRegI newval,
 7845                          rFlagsReg cr)
 7846 %{
 7847   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7848   effect(KILL cr);
 7849 
 7850   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7851             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7852   ins_encode %{
 7853     __ lock();
 7854     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7855   %}
 7856   ins_pipe( pipe_cmpxchg );
 7857 %}
 7858 
 7859 instruct compareAndExchangeS(
 7860                          memory mem_ptr,
 7861                          rax_RegI oldval, rRegI newval,
 7862                          rFlagsReg cr)
 7863 %{
 7864   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7865   effect(KILL cr);
 7866 
 7867   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7868             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7869   ins_encode %{
 7870     __ lock();
 7871     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7872   %}
 7873   ins_pipe( pipe_cmpxchg );
 7874 %}
 7875 
 7876 instruct compareAndExchangeI(
 7877                          memory mem_ptr,
 7878                          rax_RegI oldval, rRegI newval,
 7879                          rFlagsReg cr)
 7880 %{
 7881   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7882   effect(KILL cr);
 7883 
 7884   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7885             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7886   ins_encode %{
 7887     __ lock();
 7888     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7889   %}
 7890   ins_pipe( pipe_cmpxchg );
 7891 %}
 7892 
 7893 instruct compareAndExchangeL(
 7894                          memory mem_ptr,
 7895                          rax_RegL oldval, rRegL newval,
 7896                          rFlagsReg cr)
 7897 %{
 7898   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7899   effect(KILL cr);
 7900 
 7901   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7902             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7903   ins_encode %{
 7904     __ lock();
 7905     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7906   %}
 7907   ins_pipe( pipe_cmpxchg );
 7908 %}
 7909 
 7910 instruct compareAndExchangeN(
 7911                           memory mem_ptr,
 7912                           rax_RegN oldval, rRegN newval,
 7913                           rFlagsReg cr) %{
 7914   predicate(n->as_LoadStore()->barrier_data() == 0);
 7915   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7916   effect(KILL cr);
 7917 
 7918   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7919             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7920   ins_encode %{
 7921     __ lock();
 7922     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7923   %}
 7924   ins_pipe( pipe_cmpxchg );
 7925 %}
 7926 
 7927 instruct compareAndExchangeP(
 7928                          memory mem_ptr,
 7929                          rax_RegP oldval, rRegP newval,
 7930                          rFlagsReg cr)
 7931 %{
 7932   predicate(n->as_LoadStore()->barrier_data() == 0);
 7933   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7934   effect(KILL cr);
 7935 
 7936   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7937             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7938   ins_encode %{
 7939     __ lock();
 7940     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7941   %}
 7942   ins_pipe( pipe_cmpxchg );
 7943 %}
 7944 
 7945 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7946   predicate(n->as_LoadStore()->result_not_used());
 7947   match(Set dummy (GetAndAddB mem add));
 7948   effect(KILL cr);
 7949   format %{ "addb_lock   $mem, $add" %}
 7950   ins_encode %{
 7951     __ lock();
 7952     __ addb($mem$$Address, $add$$Register);
 7953   %}
 7954   ins_pipe(pipe_cmpxchg);
 7955 %}
 7956 
 7957 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7958   predicate(n->as_LoadStore()->result_not_used());
 7959   match(Set dummy (GetAndAddB mem add));
 7960   effect(KILL cr);
 7961   format %{ "addb_lock   $mem, $add" %}
 7962   ins_encode %{
 7963     __ lock();
 7964     __ addb($mem$$Address, $add$$constant);
 7965   %}
 7966   ins_pipe(pipe_cmpxchg);
 7967 %}
 7968 
 7969 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7970   predicate(!n->as_LoadStore()->result_not_used());
 7971   match(Set newval (GetAndAddB mem newval));
 7972   effect(KILL cr);
 7973   format %{ "xaddb_lock  $mem, $newval" %}
 7974   ins_encode %{
 7975     __ lock();
 7976     __ xaddb($mem$$Address, $newval$$Register);
 7977   %}
 7978   ins_pipe(pipe_cmpxchg);
 7979 %}
 7980 
 7981 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7982   predicate(n->as_LoadStore()->result_not_used());
 7983   match(Set dummy (GetAndAddS mem add));
 7984   effect(KILL cr);
 7985   format %{ "addw_lock   $mem, $add" %}
 7986   ins_encode %{
 7987     __ lock();
 7988     __ addw($mem$$Address, $add$$Register);
 7989   %}
 7990   ins_pipe(pipe_cmpxchg);
 7991 %}
 7992 
 7993 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7994   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7995   match(Set dummy (GetAndAddS mem add));
 7996   effect(KILL cr);
 7997   format %{ "addw_lock   $mem, $add" %}
 7998   ins_encode %{
 7999     __ lock();
 8000     __ addw($mem$$Address, $add$$constant);
 8001   %}
 8002   ins_pipe(pipe_cmpxchg);
 8003 %}
 8004 
 8005 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8006   predicate(!n->as_LoadStore()->result_not_used());
 8007   match(Set newval (GetAndAddS mem newval));
 8008   effect(KILL cr);
 8009   format %{ "xaddw_lock  $mem, $newval" %}
 8010   ins_encode %{
 8011     __ lock();
 8012     __ xaddw($mem$$Address, $newval$$Register);
 8013   %}
 8014   ins_pipe(pipe_cmpxchg);
 8015 %}
 8016 
 8017 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8018   predicate(n->as_LoadStore()->result_not_used());
 8019   match(Set dummy (GetAndAddI mem add));
 8020   effect(KILL cr);
 8021   format %{ "addl_lock   $mem, $add" %}
 8022   ins_encode %{
 8023     __ lock();
 8024     __ addl($mem$$Address, $add$$Register);
 8025   %}
 8026   ins_pipe(pipe_cmpxchg);
 8027 %}
 8028 
 8029 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8030   predicate(n->as_LoadStore()->result_not_used());
 8031   match(Set dummy (GetAndAddI mem add));
 8032   effect(KILL cr);
 8033   format %{ "addl_lock   $mem, $add" %}
 8034   ins_encode %{
 8035     __ lock();
 8036     __ addl($mem$$Address, $add$$constant);
 8037   %}
 8038   ins_pipe(pipe_cmpxchg);
 8039 %}
 8040 
 8041 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8042   predicate(!n->as_LoadStore()->result_not_used());
 8043   match(Set newval (GetAndAddI mem newval));
 8044   effect(KILL cr);
 8045   format %{ "xaddl_lock  $mem, $newval" %}
 8046   ins_encode %{
 8047     __ lock();
 8048     __ xaddl($mem$$Address, $newval$$Register);
 8049   %}
 8050   ins_pipe(pipe_cmpxchg);
 8051 %}
 8052 
 8053 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8054   predicate(n->as_LoadStore()->result_not_used());
 8055   match(Set dummy (GetAndAddL mem add));
 8056   effect(KILL cr);
 8057   format %{ "addq_lock   $mem, $add" %}
 8058   ins_encode %{
 8059     __ lock();
 8060     __ addq($mem$$Address, $add$$Register);
 8061   %}
 8062   ins_pipe(pipe_cmpxchg);
 8063 %}
 8064 
 8065 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8066   predicate(n->as_LoadStore()->result_not_used());
 8067   match(Set dummy (GetAndAddL mem add));
 8068   effect(KILL cr);
 8069   format %{ "addq_lock   $mem, $add" %}
 8070   ins_encode %{
 8071     __ lock();
 8072     __ addq($mem$$Address, $add$$constant);
 8073   %}
 8074   ins_pipe(pipe_cmpxchg);
 8075 %}
 8076 
 8077 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8078   predicate(!n->as_LoadStore()->result_not_used());
 8079   match(Set newval (GetAndAddL mem newval));
 8080   effect(KILL cr);
 8081   format %{ "xaddq_lock  $mem, $newval" %}
 8082   ins_encode %{
 8083     __ lock();
 8084     __ xaddq($mem$$Address, $newval$$Register);
 8085   %}
 8086   ins_pipe(pipe_cmpxchg);
 8087 %}
 8088 
 8089 instruct xchgB( memory mem, rRegI newval) %{
 8090   match(Set newval (GetAndSetB mem newval));
 8091   format %{ "XCHGB  $newval,[$mem]" %}
 8092   ins_encode %{
 8093     __ xchgb($newval$$Register, $mem$$Address);
 8094   %}
 8095   ins_pipe( pipe_cmpxchg );
 8096 %}
 8097 
 8098 instruct xchgS( memory mem, rRegI newval) %{
 8099   match(Set newval (GetAndSetS mem newval));
 8100   format %{ "XCHGW  $newval,[$mem]" %}
 8101   ins_encode %{
 8102     __ xchgw($newval$$Register, $mem$$Address);
 8103   %}
 8104   ins_pipe( pipe_cmpxchg );
 8105 %}
 8106 
 8107 instruct xchgI( memory mem, rRegI newval) %{
 8108   match(Set newval (GetAndSetI mem newval));
 8109   format %{ "XCHGL  $newval,[$mem]" %}
 8110   ins_encode %{
 8111     __ xchgl($newval$$Register, $mem$$Address);
 8112   %}
 8113   ins_pipe( pipe_cmpxchg );
 8114 %}
 8115 
 8116 instruct xchgL( memory mem, rRegL newval) %{
 8117   match(Set newval (GetAndSetL mem newval));
 8118   format %{ "XCHGL  $newval,[$mem]" %}
 8119   ins_encode %{
 8120     __ xchgq($newval$$Register, $mem$$Address);
 8121   %}
 8122   ins_pipe( pipe_cmpxchg );
 8123 %}
 8124 
 8125 instruct xchgP( memory mem, rRegP newval) %{
 8126   match(Set newval (GetAndSetP mem newval));
 8127   predicate(n->as_LoadStore()->barrier_data() == 0);
 8128   format %{ "XCHGQ  $newval,[$mem]" %}
 8129   ins_encode %{
 8130     __ xchgq($newval$$Register, $mem$$Address);
 8131   %}
 8132   ins_pipe( pipe_cmpxchg );
 8133 %}
 8134 
 8135 instruct xchgN( memory mem, rRegN newval) %{
 8136   predicate(n->as_LoadStore()->barrier_data() == 0);
 8137   match(Set newval (GetAndSetN mem newval));
 8138   format %{ "XCHGL  $newval,$mem]" %}
 8139   ins_encode %{
 8140     __ xchgl($newval$$Register, $mem$$Address);
 8141   %}
 8142   ins_pipe( pipe_cmpxchg );
 8143 %}
 8144 
 8145 //----------Abs Instructions-------------------------------------------
 8146 
 8147 // Integer Absolute Instructions
 8148 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8149 %{
 8150   match(Set dst (AbsI src));
 8151   effect(TEMP dst, KILL cr);
 8152   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8153             "subl    $dst, $src\n\t"
 8154             "cmovll  $dst, $src" %}
 8155   ins_encode %{
 8156     __ xorl($dst$$Register, $dst$$Register);
 8157     __ subl($dst$$Register, $src$$Register);
 8158     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8159   %}
 8160 
 8161   ins_pipe(ialu_reg_reg);
 8162 %}
 8163 
 8164 // Long Absolute Instructions
 8165 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8166 %{
 8167   match(Set dst (AbsL src));
 8168   effect(TEMP dst, KILL cr);
 8169   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8170             "subq    $dst, $src\n\t"
 8171             "cmovlq  $dst, $src" %}
 8172   ins_encode %{
 8173     __ xorl($dst$$Register, $dst$$Register);
 8174     __ subq($dst$$Register, $src$$Register);
 8175     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8176   %}
 8177 
 8178   ins_pipe(ialu_reg_reg);
 8179 %}
 8180 
 8181 //----------Subtraction Instructions-------------------------------------------
 8182 
 8183 // Integer Subtraction Instructions
 8184 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8185 %{
 8186   predicate(!UseAPX);
 8187   match(Set dst (SubI dst src));
 8188   effect(KILL cr);
 8189   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8190 
 8191   format %{ "subl    $dst, $src\t# int" %}
 8192   ins_encode %{
 8193     __ subl($dst$$Register, $src$$Register);
 8194   %}
 8195   ins_pipe(ialu_reg_reg);
 8196 %}
 8197 
 8198 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8199 %{
 8200   predicate(UseAPX);
 8201   match(Set dst (SubI src1 src2));
 8202   effect(KILL cr);
 8203   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8204 
 8205   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8206   ins_encode %{
 8207     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8208   %}
 8209   ins_pipe(ialu_reg_reg);
 8210 %}
 8211 
 8212 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8213 %{
 8214   predicate(UseAPX);
 8215   match(Set dst (SubI src1 src2));
 8216   effect(KILL cr);
 8217   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8218 
 8219   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8220   ins_encode %{
 8221     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8222   %}
 8223   ins_pipe(ialu_reg_reg);
 8224 %}
 8225 
 8226 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8227 %{
 8228   predicate(UseAPX);
 8229   match(Set dst (SubI (LoadI src1) src2));
 8230   effect(KILL cr);
 8231   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8232 
 8233   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8234   ins_encode %{
 8235     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8236   %}
 8237   ins_pipe(ialu_reg_reg);
 8238 %}
 8239 
 8240 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8241 %{
 8242   predicate(!UseAPX);
 8243   match(Set dst (SubI dst (LoadI src)));
 8244   effect(KILL cr);
 8245   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8246 
 8247   ins_cost(150);
 8248   format %{ "subl    $dst, $src\t# int" %}
 8249   ins_encode %{
 8250     __ subl($dst$$Register, $src$$Address);
 8251   %}
 8252   ins_pipe(ialu_reg_mem);
 8253 %}
 8254 
 8255 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8256 %{
 8257   predicate(UseAPX);
 8258   match(Set dst (SubI src1 (LoadI src2)));
 8259   effect(KILL cr);
 8260   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8261 
 8262   ins_cost(150);
 8263   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8264   ins_encode %{
 8265     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8266   %}
 8267   ins_pipe(ialu_reg_mem);
 8268 %}
 8269 
 8270 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8271 %{
 8272   predicate(UseAPX);
 8273   match(Set dst (SubI (LoadI src1) src2));
 8274   effect(KILL cr);
 8275   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8276 
 8277   ins_cost(150);
 8278   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8279   ins_encode %{
 8280     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8281   %}
 8282   ins_pipe(ialu_reg_mem);
 8283 %}
 8284 
 8285 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8286 %{
 8287   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8288   effect(KILL cr);
 8289   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8290 
 8291   ins_cost(150);
 8292   format %{ "subl    $dst, $src\t# int" %}
 8293   ins_encode %{
 8294     __ subl($dst$$Address, $src$$Register);
 8295   %}
 8296   ins_pipe(ialu_mem_reg);
 8297 %}
 8298 
 8299 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8300 %{
 8301   predicate(!UseAPX);
 8302   match(Set dst (SubL dst src));
 8303   effect(KILL cr);
 8304   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8305 
 8306   format %{ "subq    $dst, $src\t# long" %}
 8307   ins_encode %{
 8308     __ subq($dst$$Register, $src$$Register);
 8309   %}
 8310   ins_pipe(ialu_reg_reg);
 8311 %}
 8312 
 8313 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8314 %{
 8315   predicate(UseAPX);
 8316   match(Set dst (SubL src1 src2));
 8317   effect(KILL cr);
 8318   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8319 
 8320   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8321   ins_encode %{
 8322     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8323   %}
 8324   ins_pipe(ialu_reg_reg);
 8325 %}
 8326 
 8327 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8328 %{
 8329   predicate(UseAPX);
 8330   match(Set dst (SubL src1 src2));
 8331   effect(KILL cr);
 8332   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8333 
 8334   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8335   ins_encode %{
 8336     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8337   %}
 8338   ins_pipe(ialu_reg_reg);
 8339 %}
 8340 
 8341 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8342 %{
 8343   predicate(UseAPX);
 8344   match(Set dst (SubL (LoadL src1) src2));
 8345   effect(KILL cr);
 8346   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8347 
 8348   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8349   ins_encode %{
 8350     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8351   %}
 8352   ins_pipe(ialu_reg_reg);
 8353 %}
 8354 
 8355 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8356 %{
 8357   predicate(!UseAPX);
 8358   match(Set dst (SubL dst (LoadL src)));
 8359   effect(KILL cr);
 8360   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8361 
 8362   ins_cost(150);
 8363   format %{ "subq    $dst, $src\t# long" %}
 8364   ins_encode %{
 8365     __ subq($dst$$Register, $src$$Address);
 8366   %}
 8367   ins_pipe(ialu_reg_mem);
 8368 %}
 8369 
 8370 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8371 %{
 8372   predicate(UseAPX);
 8373   match(Set dst (SubL src1 (LoadL src2)));
 8374   effect(KILL cr);
 8375   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8376 
 8377   ins_cost(150);
 8378   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8379   ins_encode %{
 8380     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8381   %}
 8382   ins_pipe(ialu_reg_mem);
 8383 %}
 8384 
 8385 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8386 %{
 8387   predicate(UseAPX);
 8388   match(Set dst (SubL (LoadL src1) src2));
 8389   effect(KILL cr);
 8390   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8391 
 8392   ins_cost(150);
 8393   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8394   ins_encode %{
 8395     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8396   %}
 8397   ins_pipe(ialu_reg_mem);
 8398 %}
 8399 
 8400 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8401 %{
 8402   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8403   effect(KILL cr);
 8404   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8405 
 8406   ins_cost(150);
 8407   format %{ "subq    $dst, $src\t# long" %}
 8408   ins_encode %{
 8409     __ subq($dst$$Address, $src$$Register);
 8410   %}
 8411   ins_pipe(ialu_mem_reg);
 8412 %}
 8413 
 8414 // Subtract from a pointer
 8415 // XXX hmpf???
 8416 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8417 %{
 8418   match(Set dst (AddP dst (SubI zero src)));
 8419   effect(KILL cr);
 8420 
 8421   format %{ "subq    $dst, $src\t# ptr - int" %}
 8422   ins_encode %{
 8423     __ subq($dst$$Register, $src$$Register);
 8424   %}
 8425   ins_pipe(ialu_reg_reg);
 8426 %}
 8427 
 8428 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8429 %{
 8430   predicate(!UseAPX);
 8431   match(Set dst (SubI zero dst));
 8432   effect(KILL cr);
 8433   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8434 
 8435   format %{ "negl    $dst\t# int" %}
 8436   ins_encode %{
 8437     __ negl($dst$$Register);
 8438   %}
 8439   ins_pipe(ialu_reg);
 8440 %}
 8441 
 8442 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8443 %{
 8444   predicate(UseAPX);
 8445   match(Set dst (SubI zero src));
 8446   effect(KILL cr);
 8447   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8448 
 8449   format %{ "enegl    $dst, $src\t# int ndd" %}
 8450   ins_encode %{
 8451     __ enegl($dst$$Register, $src$$Register, false);
 8452   %}
 8453   ins_pipe(ialu_reg);
 8454 %}
 8455 
 8456 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8457 %{
 8458   predicate(!UseAPX);
 8459   match(Set dst (NegI dst));
 8460   effect(KILL cr);
 8461   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8462 
 8463   format %{ "negl    $dst\t# int" %}
 8464   ins_encode %{
 8465     __ negl($dst$$Register);
 8466   %}
 8467   ins_pipe(ialu_reg);
 8468 %}
 8469 
 8470 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8471 %{
 8472   predicate(UseAPX);
 8473   match(Set dst (NegI src));
 8474   effect(KILL cr);
 8475   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8476 
 8477   format %{ "enegl    $dst, $src\t# int ndd" %}
 8478   ins_encode %{
 8479     __ enegl($dst$$Register, $src$$Register, false);
 8480   %}
 8481   ins_pipe(ialu_reg);
 8482 %}
 8483 
 8484 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8485 %{
 8486   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8487   effect(KILL cr);
 8488   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8489 
 8490   format %{ "negl    $dst\t# int" %}
 8491   ins_encode %{
 8492     __ negl($dst$$Address);
 8493   %}
 8494   ins_pipe(ialu_reg);
 8495 %}
 8496 
 8497 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8498 %{
 8499   predicate(!UseAPX);
 8500   match(Set dst (SubL zero dst));
 8501   effect(KILL cr);
 8502   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8503 
 8504   format %{ "negq    $dst\t# long" %}
 8505   ins_encode %{
 8506     __ negq($dst$$Register);
 8507   %}
 8508   ins_pipe(ialu_reg);
 8509 %}
 8510 
 8511 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8512 %{
 8513   predicate(UseAPX);
 8514   match(Set dst (SubL zero src));
 8515   effect(KILL cr);
 8516   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8517 
 8518   format %{ "enegq    $dst, $src\t# long ndd" %}
 8519   ins_encode %{
 8520     __ enegq($dst$$Register, $src$$Register, false);
 8521   %}
 8522   ins_pipe(ialu_reg);
 8523 %}
 8524 
 8525 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8526 %{
 8527   predicate(!UseAPX);
 8528   match(Set dst (NegL dst));
 8529   effect(KILL cr);
 8530   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8531 
 8532   format %{ "negq    $dst\t# int" %}
 8533   ins_encode %{
 8534     __ negq($dst$$Register);
 8535   %}
 8536   ins_pipe(ialu_reg);
 8537 %}
 8538 
 8539 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8540 %{
 8541   predicate(UseAPX);
 8542   match(Set dst (NegL src));
 8543   effect(KILL cr);
 8544   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8545 
 8546   format %{ "enegq    $dst, $src\t# long ndd" %}
 8547   ins_encode %{
 8548     __ enegq($dst$$Register, $src$$Register, false);
 8549   %}
 8550   ins_pipe(ialu_reg);
 8551 %}
 8552 
 8553 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8554 %{
 8555   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8556   effect(KILL cr);
 8557   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8558 
 8559   format %{ "negq    $dst\t# long" %}
 8560   ins_encode %{
 8561     __ negq($dst$$Address);
 8562   %}
 8563   ins_pipe(ialu_reg);
 8564 %}
 8565 
 8566 //----------Multiplication/Division Instructions-------------------------------
 8567 // Integer Multiplication Instructions
 8568 // Multiply Register
 8569 
 8570 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8571 %{
 8572   predicate(!UseAPX);
 8573   match(Set dst (MulI dst src));
 8574   effect(KILL cr);
 8575 
 8576   ins_cost(300);
 8577   format %{ "imull   $dst, $src\t# int" %}
 8578   ins_encode %{
 8579     __ imull($dst$$Register, $src$$Register);
 8580   %}
 8581   ins_pipe(ialu_reg_reg_alu0);
 8582 %}
 8583 
 8584 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8585 %{
 8586   predicate(UseAPX);
 8587   match(Set dst (MulI src1 src2));
 8588   effect(KILL cr);
 8589 
 8590   ins_cost(300);
 8591   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8592   ins_encode %{
 8593     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8594   %}
 8595   ins_pipe(ialu_reg_reg_alu0);
 8596 %}
 8597 
 8598 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8599 %{
 8600   predicate(!UseAPX);
 8601   match(Set dst (MulI src imm));
 8602   effect(KILL cr);
 8603 
 8604   ins_cost(300);
 8605   format %{ "imull   $dst, $src, $imm\t# int" %}
 8606   ins_encode %{
 8607     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8608   %}
 8609   ins_pipe(ialu_reg_reg_alu0);
 8610 %}
 8611 
 8612 instruct mulI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8613 %{
 8614   predicate(UseAPX);
 8615   match(Set dst (MulI src1 src2));
 8616   effect(KILL cr);
 8617 
 8618   ins_cost(300);
 8619   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8620   ins_encode %{
 8621     __ eimull($dst$$Register, $src1$$Register, $src2$$constant, false);
 8622   %}
 8623   ins_pipe(ialu_reg_reg_alu0);
 8624 %}
 8625 
 8626 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8627 %{
 8628   predicate(!UseAPX);
 8629   match(Set dst (MulI dst (LoadI src)));
 8630   effect(KILL cr);
 8631 
 8632   ins_cost(350);
 8633   format %{ "imull   $dst, $src\t# int" %}
 8634   ins_encode %{
 8635     __ imull($dst$$Register, $src$$Address);
 8636   %}
 8637   ins_pipe(ialu_reg_mem_alu0);
 8638 %}
 8639 
 8640 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8641 %{
 8642   predicate(UseAPX);
 8643   match(Set dst (MulI src1 (LoadI src2)));
 8644   effect(KILL cr);
 8645 
 8646   ins_cost(350);
 8647   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8648   ins_encode %{
 8649     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8650   %}
 8651   ins_pipe(ialu_reg_mem_alu0);
 8652 %}
 8653 
 8654 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8655 %{
 8656   predicate(!UseAPX);
 8657   match(Set dst (MulI (LoadI src) imm));
 8658   effect(KILL cr);
 8659 
 8660   ins_cost(300);
 8661   format %{ "imull   $dst, $src, $imm\t# int" %}
 8662   ins_encode %{
 8663     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8664   %}
 8665   ins_pipe(ialu_reg_mem_alu0);
 8666 %}
 8667 
 8668 instruct mulI_rReg_mem_imm(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8669 %{
 8670   predicate(UseAPX);
 8671   match(Set dst (MulI (LoadI src1) src2));
 8672   effect(KILL cr);
 8673 
 8674   ins_cost(300);
 8675   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8676   ins_encode %{
 8677     __ eimull($dst$$Register, $src1$$Address, $src2$$constant, false);
 8678   %}
 8679   ins_pipe(ialu_reg_mem_alu0);
 8680 %}
 8681 
 8682 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8683 %{
 8684   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8685   effect(KILL cr, KILL src2);
 8686 
 8687   expand %{ mulI_rReg(dst, src1, cr);
 8688            mulI_rReg(src2, src3, cr);
 8689            addI_rReg(dst, src2, cr); %}
 8690 %}
 8691 
 8692 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8693 %{
 8694   predicate(!UseAPX);
 8695   match(Set dst (MulL dst src));
 8696   effect(KILL cr);
 8697 
 8698   ins_cost(300);
 8699   format %{ "imulq   $dst, $src\t# long" %}
 8700   ins_encode %{
 8701     __ imulq($dst$$Register, $src$$Register);
 8702   %}
 8703   ins_pipe(ialu_reg_reg_alu0);
 8704 %}
 8705 
 8706 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8707 %{
 8708   predicate(UseAPX);
 8709   match(Set dst (MulL src1 src2));
 8710   effect(KILL cr);
 8711 
 8712   ins_cost(300);
 8713   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8714   ins_encode %{
 8715     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8716   %}
 8717   ins_pipe(ialu_reg_reg_alu0);
 8718 %}
 8719 
 8720 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8721 %{
 8722   predicate(!UseAPX);
 8723   match(Set dst (MulL src imm));
 8724   effect(KILL cr);
 8725 
 8726   ins_cost(300);
 8727   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8728   ins_encode %{
 8729     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8730   %}
 8731   ins_pipe(ialu_reg_reg_alu0);
 8732 %}
 8733 
 8734 instruct mulL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8735 %{
 8736   predicate(UseAPX);
 8737   match(Set dst (MulL src1 src2));
 8738   effect(KILL cr);
 8739 
 8740   ins_cost(300);
 8741   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8742   ins_encode %{
 8743     __ eimulq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8744   %}
 8745   ins_pipe(ialu_reg_reg_alu0);
 8746 %}
 8747 
 8748 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8749 %{
 8750   predicate(!UseAPX);
 8751   match(Set dst (MulL dst (LoadL src)));
 8752   effect(KILL cr);
 8753 
 8754   ins_cost(350);
 8755   format %{ "imulq   $dst, $src\t# long" %}
 8756   ins_encode %{
 8757     __ imulq($dst$$Register, $src$$Address);
 8758   %}
 8759   ins_pipe(ialu_reg_mem_alu0);
 8760 %}
 8761 
 8762 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8763 %{
 8764   predicate(UseAPX);
 8765   match(Set dst (MulL src1 (LoadL src2)));
 8766   effect(KILL cr);
 8767 
 8768   ins_cost(350);
 8769   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8770   ins_encode %{
 8771     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8772   %}
 8773   ins_pipe(ialu_reg_mem_alu0);
 8774 %}
 8775 
 8776 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8777 %{
 8778   predicate(!UseAPX);
 8779   match(Set dst (MulL (LoadL src) imm));
 8780   effect(KILL cr);
 8781 
 8782   ins_cost(300);
 8783   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8784   ins_encode %{
 8785     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8786   %}
 8787   ins_pipe(ialu_reg_mem_alu0);
 8788 %}
 8789 
 8790 instruct mulL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8791 %{
 8792   predicate(UseAPX);
 8793   match(Set dst (MulL (LoadL src1) src2));
 8794   effect(KILL cr);
 8795 
 8796   ins_cost(300);
 8797   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8798   ins_encode %{
 8799     __ eimulq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8800   %}
 8801   ins_pipe(ialu_reg_mem_alu0);
 8802 %}
 8803 
 8804 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8805 %{
 8806   match(Set dst (MulHiL src rax));
 8807   effect(USE_KILL rax, KILL cr);
 8808 
 8809   ins_cost(300);
 8810   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8811   ins_encode %{
 8812     __ imulq($src$$Register);
 8813   %}
 8814   ins_pipe(ialu_reg_reg_alu0);
 8815 %}
 8816 
 8817 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8818 %{
 8819   match(Set dst (UMulHiL src rax));
 8820   effect(USE_KILL rax, KILL cr);
 8821 
 8822   ins_cost(300);
 8823   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8824   ins_encode %{
 8825     __ mulq($src$$Register);
 8826   %}
 8827   ins_pipe(ialu_reg_reg_alu0);
 8828 %}
 8829 
 8830 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8831                    rFlagsReg cr)
 8832 %{
 8833   match(Set rax (DivI rax div));
 8834   effect(KILL rdx, KILL cr);
 8835 
 8836   ins_cost(30*100+10*100); // XXX
 8837   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8838             "jne,s   normal\n\t"
 8839             "xorl    rdx, rdx\n\t"
 8840             "cmpl    $div, -1\n\t"
 8841             "je,s    done\n"
 8842     "normal: cdql\n\t"
 8843             "idivl   $div\n"
 8844     "done:"        %}
 8845   ins_encode(cdql_enc(div));
 8846   ins_pipe(ialu_reg_reg_alu0);
 8847 %}
 8848 
 8849 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8850                    rFlagsReg cr)
 8851 %{
 8852   match(Set rax (DivL rax div));
 8853   effect(KILL rdx, KILL cr);
 8854 
 8855   ins_cost(30*100+10*100); // XXX
 8856   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8857             "cmpq    rax, rdx\n\t"
 8858             "jne,s   normal\n\t"
 8859             "xorl    rdx, rdx\n\t"
 8860             "cmpq    $div, -1\n\t"
 8861             "je,s    done\n"
 8862     "normal: cdqq\n\t"
 8863             "idivq   $div\n"
 8864     "done:"        %}
 8865   ins_encode(cdqq_enc(div));
 8866   ins_pipe(ialu_reg_reg_alu0);
 8867 %}
 8868 
 8869 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8870 %{
 8871   match(Set rax (UDivI rax div));
 8872   effect(KILL rdx, KILL cr);
 8873 
 8874   ins_cost(300);
 8875   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8876   ins_encode %{
 8877     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8878   %}
 8879   ins_pipe(ialu_reg_reg_alu0);
 8880 %}
 8881 
 8882 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8883 %{
 8884   match(Set rax (UDivL rax div));
 8885   effect(KILL rdx, KILL cr);
 8886 
 8887   ins_cost(300);
 8888   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8889   ins_encode %{
 8890      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8891   %}
 8892   ins_pipe(ialu_reg_reg_alu0);
 8893 %}
 8894 
 8895 // Integer DIVMOD with Register, both quotient and mod results
 8896 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8897                              rFlagsReg cr)
 8898 %{
 8899   match(DivModI rax div);
 8900   effect(KILL cr);
 8901 
 8902   ins_cost(30*100+10*100); // XXX
 8903   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8904             "jne,s   normal\n\t"
 8905             "xorl    rdx, rdx\n\t"
 8906             "cmpl    $div, -1\n\t"
 8907             "je,s    done\n"
 8908     "normal: cdql\n\t"
 8909             "idivl   $div\n"
 8910     "done:"        %}
 8911   ins_encode(cdql_enc(div));
 8912   ins_pipe(pipe_slow);
 8913 %}
 8914 
 8915 // Long DIVMOD with Register, both quotient and mod results
 8916 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8917                              rFlagsReg cr)
 8918 %{
 8919   match(DivModL rax div);
 8920   effect(KILL cr);
 8921 
 8922   ins_cost(30*100+10*100); // XXX
 8923   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8924             "cmpq    rax, rdx\n\t"
 8925             "jne,s   normal\n\t"
 8926             "xorl    rdx, rdx\n\t"
 8927             "cmpq    $div, -1\n\t"
 8928             "je,s    done\n"
 8929     "normal: cdqq\n\t"
 8930             "idivq   $div\n"
 8931     "done:"        %}
 8932   ins_encode(cdqq_enc(div));
 8933   ins_pipe(pipe_slow);
 8934 %}
 8935 
 8936 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8937 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8938                               no_rax_rdx_RegI div, rFlagsReg cr)
 8939 %{
 8940   match(UDivModI rax div);
 8941   effect(TEMP tmp, KILL cr);
 8942 
 8943   ins_cost(300);
 8944   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8945             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8946           %}
 8947   ins_encode %{
 8948     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8949   %}
 8950   ins_pipe(pipe_slow);
 8951 %}
 8952 
 8953 // Unsigned long DIVMOD with Register, both quotient and mod results
 8954 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8955                               no_rax_rdx_RegL div, rFlagsReg cr)
 8956 %{
 8957   match(UDivModL rax div);
 8958   effect(TEMP tmp, KILL cr);
 8959 
 8960   ins_cost(300);
 8961   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8962             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8963           %}
 8964   ins_encode %{
 8965     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8966   %}
 8967   ins_pipe(pipe_slow);
 8968 %}
 8969 
 8970 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8971                    rFlagsReg cr)
 8972 %{
 8973   match(Set rdx (ModI rax div));
 8974   effect(KILL rax, KILL cr);
 8975 
 8976   ins_cost(300); // XXX
 8977   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8978             "jne,s   normal\n\t"
 8979             "xorl    rdx, rdx\n\t"
 8980             "cmpl    $div, -1\n\t"
 8981             "je,s    done\n"
 8982     "normal: cdql\n\t"
 8983             "idivl   $div\n"
 8984     "done:"        %}
 8985   ins_encode(cdql_enc(div));
 8986   ins_pipe(ialu_reg_reg_alu0);
 8987 %}
 8988 
 8989 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8990                    rFlagsReg cr)
 8991 %{
 8992   match(Set rdx (ModL rax div));
 8993   effect(KILL rax, KILL cr);
 8994 
 8995   ins_cost(300); // XXX
 8996   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8997             "cmpq    rax, rdx\n\t"
 8998             "jne,s   normal\n\t"
 8999             "xorl    rdx, rdx\n\t"
 9000             "cmpq    $div, -1\n\t"
 9001             "je,s    done\n"
 9002     "normal: cdqq\n\t"
 9003             "idivq   $div\n"
 9004     "done:"        %}
 9005   ins_encode(cdqq_enc(div));
 9006   ins_pipe(ialu_reg_reg_alu0);
 9007 %}
 9008 
 9009 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9010 %{
 9011   match(Set rdx (UModI rax div));
 9012   effect(KILL rax, KILL cr);
 9013 
 9014   ins_cost(300);
 9015   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9016   ins_encode %{
 9017     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9018   %}
 9019   ins_pipe(ialu_reg_reg_alu0);
 9020 %}
 9021 
 9022 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9023 %{
 9024   match(Set rdx (UModL rax div));
 9025   effect(KILL rax, KILL cr);
 9026 
 9027   ins_cost(300);
 9028   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9029   ins_encode %{
 9030     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9031   %}
 9032   ins_pipe(ialu_reg_reg_alu0);
 9033 %}
 9034 
 9035 // Integer Shift Instructions
 9036 // Shift Left by one, two, three
 9037 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9038 %{
 9039   predicate(!UseAPX);
 9040   match(Set dst (LShiftI dst shift));
 9041   effect(KILL cr);
 9042 
 9043   format %{ "sall    $dst, $shift" %}
 9044   ins_encode %{
 9045     __ sall($dst$$Register, $shift$$constant);
 9046   %}
 9047   ins_pipe(ialu_reg);
 9048 %}
 9049 
 9050 // Shift Left by one, two, three
 9051 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9052 %{
 9053   predicate(UseAPX);
 9054   match(Set dst (LShiftI src shift));
 9055   effect(KILL cr);
 9056 
 9057   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9058   ins_encode %{
 9059     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9060   %}
 9061   ins_pipe(ialu_reg);
 9062 %}
 9063 
 9064 // Shift Left by 8-bit immediate
 9065 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9066 %{
 9067   predicate(!UseAPX);
 9068   match(Set dst (LShiftI dst shift));
 9069   effect(KILL cr);
 9070 
 9071   format %{ "sall    $dst, $shift" %}
 9072   ins_encode %{
 9073     __ sall($dst$$Register, $shift$$constant);
 9074   %}
 9075   ins_pipe(ialu_reg);
 9076 %}
 9077 
 9078 // Shift Left by 8-bit immediate
 9079 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9080 %{
 9081   predicate(UseAPX);
 9082   match(Set dst (LShiftI src shift));
 9083   effect(KILL cr);
 9084 
 9085   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9086   ins_encode %{
 9087     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9088   %}
 9089   ins_pipe(ialu_reg);
 9090 %}
 9091 
 9092 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9093 %{
 9094   predicate(UseAPX);
 9095   match(Set dst (LShiftI (LoadI src) shift));
 9096   effect(KILL cr);
 9097 
 9098   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9099   ins_encode %{
 9100     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9101   %}
 9102   ins_pipe(ialu_reg);
 9103 %}
 9104 
 9105 // Shift Left by 8-bit immediate
 9106 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9107 %{
 9108   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9109   effect(KILL cr);
 9110 
 9111   format %{ "sall    $dst, $shift" %}
 9112   ins_encode %{
 9113     __ sall($dst$$Address, $shift$$constant);
 9114   %}
 9115   ins_pipe(ialu_mem_imm);
 9116 %}
 9117 
 9118 // Shift Left by variable
 9119 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9120 %{
 9121   predicate(!VM_Version::supports_bmi2());
 9122   match(Set dst (LShiftI dst shift));
 9123   effect(KILL cr);
 9124 
 9125   format %{ "sall    $dst, $shift" %}
 9126   ins_encode %{
 9127     __ sall($dst$$Register);
 9128   %}
 9129   ins_pipe(ialu_reg_reg);
 9130 %}
 9131 
 9132 // Shift Left by variable
 9133 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9134 %{
 9135   predicate(!VM_Version::supports_bmi2());
 9136   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9137   effect(KILL cr);
 9138 
 9139   format %{ "sall    $dst, $shift" %}
 9140   ins_encode %{
 9141     __ sall($dst$$Address);
 9142   %}
 9143   ins_pipe(ialu_mem_reg);
 9144 %}
 9145 
 9146 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9147 %{
 9148   predicate(VM_Version::supports_bmi2());
 9149   match(Set dst (LShiftI src shift));
 9150 
 9151   format %{ "shlxl   $dst, $src, $shift" %}
 9152   ins_encode %{
 9153     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9154   %}
 9155   ins_pipe(ialu_reg_reg);
 9156 %}
 9157 
 9158 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9159 %{
 9160   predicate(VM_Version::supports_bmi2());
 9161   match(Set dst (LShiftI (LoadI src) shift));
 9162   ins_cost(175);
 9163   format %{ "shlxl   $dst, $src, $shift" %}
 9164   ins_encode %{
 9165     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9166   %}
 9167   ins_pipe(ialu_reg_mem);
 9168 %}
 9169 
 9170 // Arithmetic Shift Right by 8-bit immediate
 9171 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9172 %{
 9173   predicate(!UseAPX);
 9174   match(Set dst (RShiftI dst shift));
 9175   effect(KILL cr);
 9176 
 9177   format %{ "sarl    $dst, $shift" %}
 9178   ins_encode %{
 9179     __ sarl($dst$$Register, $shift$$constant);
 9180   %}
 9181   ins_pipe(ialu_mem_imm);
 9182 %}
 9183 
 9184 // Arithmetic Shift Right by 8-bit immediate
 9185 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9186 %{
 9187   predicate(UseAPX);
 9188   match(Set dst (RShiftI src shift));
 9189   effect(KILL cr);
 9190 
 9191   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9192   ins_encode %{
 9193     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9194   %}
 9195   ins_pipe(ialu_mem_imm);
 9196 %}
 9197 
 9198 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9199 %{
 9200   predicate(UseAPX);
 9201   match(Set dst (RShiftI (LoadI src) shift));
 9202   effect(KILL cr);
 9203 
 9204   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9205   ins_encode %{
 9206     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9207   %}
 9208   ins_pipe(ialu_mem_imm);
 9209 %}
 9210 
 9211 // Arithmetic Shift Right by 8-bit immediate
 9212 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9213 %{
 9214   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9215   effect(KILL cr);
 9216 
 9217   format %{ "sarl    $dst, $shift" %}
 9218   ins_encode %{
 9219     __ sarl($dst$$Address, $shift$$constant);
 9220   %}
 9221   ins_pipe(ialu_mem_imm);
 9222 %}
 9223 
 9224 // Arithmetic Shift Right by variable
 9225 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9226 %{
 9227   predicate(!VM_Version::supports_bmi2());
 9228   match(Set dst (RShiftI dst shift));
 9229   effect(KILL cr);
 9230 
 9231   format %{ "sarl    $dst, $shift" %}
 9232   ins_encode %{
 9233     __ sarl($dst$$Register);
 9234   %}
 9235   ins_pipe(ialu_reg_reg);
 9236 %}
 9237 
 9238 // Arithmetic Shift Right by variable
 9239 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9240 %{
 9241   predicate(!VM_Version::supports_bmi2());
 9242   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9243   effect(KILL cr);
 9244 
 9245   format %{ "sarl    $dst, $shift" %}
 9246   ins_encode %{
 9247     __ sarl($dst$$Address);
 9248   %}
 9249   ins_pipe(ialu_mem_reg);
 9250 %}
 9251 
 9252 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9253 %{
 9254   predicate(VM_Version::supports_bmi2());
 9255   match(Set dst (RShiftI src shift));
 9256 
 9257   format %{ "sarxl   $dst, $src, $shift" %}
 9258   ins_encode %{
 9259     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9260   %}
 9261   ins_pipe(ialu_reg_reg);
 9262 %}
 9263 
 9264 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9265 %{
 9266   predicate(VM_Version::supports_bmi2());
 9267   match(Set dst (RShiftI (LoadI src) shift));
 9268   ins_cost(175);
 9269   format %{ "sarxl   $dst, $src, $shift" %}
 9270   ins_encode %{
 9271     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9272   %}
 9273   ins_pipe(ialu_reg_mem);
 9274 %}
 9275 
 9276 // Logical Shift Right by 8-bit immediate
 9277 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9278 %{
 9279   predicate(!UseAPX);
 9280   match(Set dst (URShiftI dst shift));
 9281   effect(KILL cr);
 9282 
 9283   format %{ "shrl    $dst, $shift" %}
 9284   ins_encode %{
 9285     __ shrl($dst$$Register, $shift$$constant);
 9286   %}
 9287   ins_pipe(ialu_reg);
 9288 %}
 9289 
 9290 // Logical Shift Right by 8-bit immediate
 9291 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9292 %{
 9293   predicate(UseAPX);
 9294   match(Set dst (URShiftI src shift));
 9295   effect(KILL cr);
 9296 
 9297   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9298   ins_encode %{
 9299     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9300   %}
 9301   ins_pipe(ialu_reg);
 9302 %}
 9303 
 9304 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9305 %{
 9306   predicate(UseAPX);
 9307   match(Set dst (URShiftI (LoadI src) shift));
 9308   effect(KILL cr);
 9309 
 9310   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9311   ins_encode %{
 9312     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9313   %}
 9314   ins_pipe(ialu_reg);
 9315 %}
 9316 
 9317 // Logical Shift Right by 8-bit immediate
 9318 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9319 %{
 9320   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9321   effect(KILL cr);
 9322 
 9323   format %{ "shrl    $dst, $shift" %}
 9324   ins_encode %{
 9325     __ shrl($dst$$Address, $shift$$constant);
 9326   %}
 9327   ins_pipe(ialu_mem_imm);
 9328 %}
 9329 
 9330 // Logical Shift Right by variable
 9331 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9332 %{
 9333   predicate(!VM_Version::supports_bmi2());
 9334   match(Set dst (URShiftI dst shift));
 9335   effect(KILL cr);
 9336 
 9337   format %{ "shrl    $dst, $shift" %}
 9338   ins_encode %{
 9339     __ shrl($dst$$Register);
 9340   %}
 9341   ins_pipe(ialu_reg_reg);
 9342 %}
 9343 
 9344 // Logical Shift Right by variable
 9345 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9346 %{
 9347   predicate(!VM_Version::supports_bmi2());
 9348   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9349   effect(KILL cr);
 9350 
 9351   format %{ "shrl    $dst, $shift" %}
 9352   ins_encode %{
 9353     __ shrl($dst$$Address);
 9354   %}
 9355   ins_pipe(ialu_mem_reg);
 9356 %}
 9357 
 9358 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9359 %{
 9360   predicate(VM_Version::supports_bmi2());
 9361   match(Set dst (URShiftI src shift));
 9362 
 9363   format %{ "shrxl   $dst, $src, $shift" %}
 9364   ins_encode %{
 9365     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9366   %}
 9367   ins_pipe(ialu_reg_reg);
 9368 %}
 9369 
 9370 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9371 %{
 9372   predicate(VM_Version::supports_bmi2());
 9373   match(Set dst (URShiftI (LoadI src) shift));
 9374   ins_cost(175);
 9375   format %{ "shrxl   $dst, $src, $shift" %}
 9376   ins_encode %{
 9377     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9378   %}
 9379   ins_pipe(ialu_reg_mem);
 9380 %}
 9381 
 9382 // Long Shift Instructions
 9383 // Shift Left by one, two, three
 9384 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9385 %{
 9386   predicate(!UseAPX);
 9387   match(Set dst (LShiftL dst shift));
 9388   effect(KILL cr);
 9389 
 9390   format %{ "salq    $dst, $shift" %}
 9391   ins_encode %{
 9392     __ salq($dst$$Register, $shift$$constant);
 9393   %}
 9394   ins_pipe(ialu_reg);
 9395 %}
 9396 
 9397 // Shift Left by one, two, three
 9398 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9399 %{
 9400   predicate(UseAPX);
 9401   match(Set dst (LShiftL src shift));
 9402   effect(KILL cr);
 9403 
 9404   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9405   ins_encode %{
 9406     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9407   %}
 9408   ins_pipe(ialu_reg);
 9409 %}
 9410 
 9411 // Shift Left by 8-bit immediate
 9412 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9413 %{
 9414   predicate(!UseAPX);
 9415   match(Set dst (LShiftL dst shift));
 9416   effect(KILL cr);
 9417 
 9418   format %{ "salq    $dst, $shift" %}
 9419   ins_encode %{
 9420     __ salq($dst$$Register, $shift$$constant);
 9421   %}
 9422   ins_pipe(ialu_reg);
 9423 %}
 9424 
 9425 // Shift Left by 8-bit immediate
 9426 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9427 %{
 9428   predicate(UseAPX);
 9429   match(Set dst (LShiftL src shift));
 9430   effect(KILL cr);
 9431 
 9432   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9433   ins_encode %{
 9434     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9435   %}
 9436   ins_pipe(ialu_reg);
 9437 %}
 9438 
 9439 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9440 %{
 9441   predicate(UseAPX);
 9442   match(Set dst (LShiftL (LoadL src) shift));
 9443   effect(KILL cr);
 9444 
 9445   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9446   ins_encode %{
 9447     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9448   %}
 9449   ins_pipe(ialu_reg);
 9450 %}
 9451 
 9452 // Shift Left by 8-bit immediate
 9453 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9454 %{
 9455   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9456   effect(KILL cr);
 9457 
 9458   format %{ "salq    $dst, $shift" %}
 9459   ins_encode %{
 9460     __ salq($dst$$Address, $shift$$constant);
 9461   %}
 9462   ins_pipe(ialu_mem_imm);
 9463 %}
 9464 
 9465 // Shift Left by variable
 9466 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9467 %{
 9468   predicate(!VM_Version::supports_bmi2());
 9469   match(Set dst (LShiftL dst shift));
 9470   effect(KILL cr);
 9471 
 9472   format %{ "salq    $dst, $shift" %}
 9473   ins_encode %{
 9474     __ salq($dst$$Register);
 9475   %}
 9476   ins_pipe(ialu_reg_reg);
 9477 %}
 9478 
 9479 // Shift Left by variable
 9480 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9481 %{
 9482   predicate(!VM_Version::supports_bmi2());
 9483   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9484   effect(KILL cr);
 9485 
 9486   format %{ "salq    $dst, $shift" %}
 9487   ins_encode %{
 9488     __ salq($dst$$Address);
 9489   %}
 9490   ins_pipe(ialu_mem_reg);
 9491 %}
 9492 
 9493 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9494 %{
 9495   predicate(VM_Version::supports_bmi2());
 9496   match(Set dst (LShiftL src shift));
 9497 
 9498   format %{ "shlxq   $dst, $src, $shift" %}
 9499   ins_encode %{
 9500     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9501   %}
 9502   ins_pipe(ialu_reg_reg);
 9503 %}
 9504 
 9505 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9506 %{
 9507   predicate(VM_Version::supports_bmi2());
 9508   match(Set dst (LShiftL (LoadL src) shift));
 9509   ins_cost(175);
 9510   format %{ "shlxq   $dst, $src, $shift" %}
 9511   ins_encode %{
 9512     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9513   %}
 9514   ins_pipe(ialu_reg_mem);
 9515 %}
 9516 
 9517 // Arithmetic Shift Right by 8-bit immediate
 9518 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9519 %{
 9520   predicate(!UseAPX);
 9521   match(Set dst (RShiftL dst shift));
 9522   effect(KILL cr);
 9523 
 9524   format %{ "sarq    $dst, $shift" %}
 9525   ins_encode %{
 9526     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9527   %}
 9528   ins_pipe(ialu_mem_imm);
 9529 %}
 9530 
 9531 // Arithmetic Shift Right by 8-bit immediate
 9532 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9533 %{
 9534   predicate(UseAPX);
 9535   match(Set dst (RShiftL src shift));
 9536   effect(KILL cr);
 9537 
 9538   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9539   ins_encode %{
 9540     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9541   %}
 9542   ins_pipe(ialu_mem_imm);
 9543 %}
 9544 
 9545 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9546 %{
 9547   predicate(UseAPX);
 9548   match(Set dst (RShiftL (LoadL src) shift));
 9549   effect(KILL cr);
 9550 
 9551   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9552   ins_encode %{
 9553     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9554   %}
 9555   ins_pipe(ialu_mem_imm);
 9556 %}
 9557 
 9558 // Arithmetic Shift Right by 8-bit immediate
 9559 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9560 %{
 9561   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9562   effect(KILL cr);
 9563 
 9564   format %{ "sarq    $dst, $shift" %}
 9565   ins_encode %{
 9566     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9567   %}
 9568   ins_pipe(ialu_mem_imm);
 9569 %}
 9570 
 9571 // Arithmetic Shift Right by variable
 9572 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9573 %{
 9574   predicate(!VM_Version::supports_bmi2());
 9575   match(Set dst (RShiftL dst shift));
 9576   effect(KILL cr);
 9577 
 9578   format %{ "sarq    $dst, $shift" %}
 9579   ins_encode %{
 9580     __ sarq($dst$$Register);
 9581   %}
 9582   ins_pipe(ialu_reg_reg);
 9583 %}
 9584 
 9585 // Arithmetic Shift Right by variable
 9586 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9587 %{
 9588   predicate(!VM_Version::supports_bmi2());
 9589   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9590   effect(KILL cr);
 9591 
 9592   format %{ "sarq    $dst, $shift" %}
 9593   ins_encode %{
 9594     __ sarq($dst$$Address);
 9595   %}
 9596   ins_pipe(ialu_mem_reg);
 9597 %}
 9598 
 9599 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9600 %{
 9601   predicate(VM_Version::supports_bmi2());
 9602   match(Set dst (RShiftL src shift));
 9603 
 9604   format %{ "sarxq   $dst, $src, $shift" %}
 9605   ins_encode %{
 9606     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9607   %}
 9608   ins_pipe(ialu_reg_reg);
 9609 %}
 9610 
 9611 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9612 %{
 9613   predicate(VM_Version::supports_bmi2());
 9614   match(Set dst (RShiftL (LoadL src) shift));
 9615   ins_cost(175);
 9616   format %{ "sarxq   $dst, $src, $shift" %}
 9617   ins_encode %{
 9618     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9619   %}
 9620   ins_pipe(ialu_reg_mem);
 9621 %}
 9622 
 9623 // Logical Shift Right by 8-bit immediate
 9624 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9625 %{
 9626   predicate(!UseAPX);
 9627   match(Set dst (URShiftL dst shift));
 9628   effect(KILL cr);
 9629 
 9630   format %{ "shrq    $dst, $shift" %}
 9631   ins_encode %{
 9632     __ shrq($dst$$Register, $shift$$constant);
 9633   %}
 9634   ins_pipe(ialu_reg);
 9635 %}
 9636 
 9637 // Logical Shift Right by 8-bit immediate
 9638 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9639 %{
 9640   predicate(UseAPX);
 9641   match(Set dst (URShiftL src shift));
 9642   effect(KILL cr);
 9643 
 9644   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9645   ins_encode %{
 9646     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9647   %}
 9648   ins_pipe(ialu_reg);
 9649 %}
 9650 
 9651 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9652 %{
 9653   predicate(UseAPX);
 9654   match(Set dst (URShiftL (LoadL src) shift));
 9655   effect(KILL cr);
 9656 
 9657   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9658   ins_encode %{
 9659     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9660   %}
 9661   ins_pipe(ialu_reg);
 9662 %}
 9663 
 9664 // Logical Shift Right by 8-bit immediate
 9665 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9666 %{
 9667   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9668   effect(KILL cr);
 9669 
 9670   format %{ "shrq    $dst, $shift" %}
 9671   ins_encode %{
 9672     __ shrq($dst$$Address, $shift$$constant);
 9673   %}
 9674   ins_pipe(ialu_mem_imm);
 9675 %}
 9676 
 9677 // Logical Shift Right by variable
 9678 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9679 %{
 9680   predicate(!VM_Version::supports_bmi2());
 9681   match(Set dst (URShiftL dst shift));
 9682   effect(KILL cr);
 9683 
 9684   format %{ "shrq    $dst, $shift" %}
 9685   ins_encode %{
 9686     __ shrq($dst$$Register);
 9687   %}
 9688   ins_pipe(ialu_reg_reg);
 9689 %}
 9690 
 9691 // Logical Shift Right by variable
 9692 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9693 %{
 9694   predicate(!VM_Version::supports_bmi2());
 9695   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9696   effect(KILL cr);
 9697 
 9698   format %{ "shrq    $dst, $shift" %}
 9699   ins_encode %{
 9700     __ shrq($dst$$Address);
 9701   %}
 9702   ins_pipe(ialu_mem_reg);
 9703 %}
 9704 
 9705 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9706 %{
 9707   predicate(VM_Version::supports_bmi2());
 9708   match(Set dst (URShiftL src shift));
 9709 
 9710   format %{ "shrxq   $dst, $src, $shift" %}
 9711   ins_encode %{
 9712     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9713   %}
 9714   ins_pipe(ialu_reg_reg);
 9715 %}
 9716 
 9717 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9718 %{
 9719   predicate(VM_Version::supports_bmi2());
 9720   match(Set dst (URShiftL (LoadL src) shift));
 9721   ins_cost(175);
 9722   format %{ "shrxq   $dst, $src, $shift" %}
 9723   ins_encode %{
 9724     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9725   %}
 9726   ins_pipe(ialu_reg_mem);
 9727 %}
 9728 
 9729 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9730 // This idiom is used by the compiler for the i2b bytecode.
 9731 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9732 %{
 9733   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9734 
 9735   format %{ "movsbl  $dst, $src\t# i2b" %}
 9736   ins_encode %{
 9737     __ movsbl($dst$$Register, $src$$Register);
 9738   %}
 9739   ins_pipe(ialu_reg_reg);
 9740 %}
 9741 
 9742 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9743 // This idiom is used by the compiler the i2s bytecode.
 9744 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9745 %{
 9746   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9747 
 9748   format %{ "movswl  $dst, $src\t# i2s" %}
 9749   ins_encode %{
 9750     __ movswl($dst$$Register, $src$$Register);
 9751   %}
 9752   ins_pipe(ialu_reg_reg);
 9753 %}
 9754 
 9755 // ROL/ROR instructions
 9756 
 9757 // Rotate left by constant.
 9758 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9759 %{
 9760   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9761   match(Set dst (RotateLeft dst shift));
 9762   effect(KILL cr);
 9763   format %{ "roll    $dst, $shift" %}
 9764   ins_encode %{
 9765     __ roll($dst$$Register, $shift$$constant);
 9766   %}
 9767   ins_pipe(ialu_reg);
 9768 %}
 9769 
 9770 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9771 %{
 9772   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9773   match(Set dst (RotateLeft src shift));
 9774   format %{ "rolxl   $dst, $src, $shift" %}
 9775   ins_encode %{
 9776     int shift = 32 - ($shift$$constant & 31);
 9777     __ rorxl($dst$$Register, $src$$Register, shift);
 9778   %}
 9779   ins_pipe(ialu_reg_reg);
 9780 %}
 9781 
 9782 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9783 %{
 9784   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9785   match(Set dst (RotateLeft (LoadI src) shift));
 9786   ins_cost(175);
 9787   format %{ "rolxl   $dst, $src, $shift" %}
 9788   ins_encode %{
 9789     int shift = 32 - ($shift$$constant & 31);
 9790     __ rorxl($dst$$Register, $src$$Address, shift);
 9791   %}
 9792   ins_pipe(ialu_reg_mem);
 9793 %}
 9794 
 9795 // Rotate Left by variable
 9796 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9797 %{
 9798   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9799   match(Set dst (RotateLeft dst shift));
 9800   effect(KILL cr);
 9801   format %{ "roll    $dst, $shift" %}
 9802   ins_encode %{
 9803     __ roll($dst$$Register);
 9804   %}
 9805   ins_pipe(ialu_reg_reg);
 9806 %}
 9807 
 9808 // Rotate Left by variable
 9809 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9810 %{
 9811   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9812   match(Set dst (RotateLeft src shift));
 9813   effect(KILL cr);
 9814 
 9815   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9816   ins_encode %{
 9817     __ eroll($dst$$Register, $src$$Register, false);
 9818   %}
 9819   ins_pipe(ialu_reg_reg);
 9820 %}
 9821 
 9822 // Rotate Right by constant.
 9823 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9824 %{
 9825   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9826   match(Set dst (RotateRight dst shift));
 9827   effect(KILL cr);
 9828   format %{ "rorl    $dst, $shift" %}
 9829   ins_encode %{
 9830     __ rorl($dst$$Register, $shift$$constant);
 9831   %}
 9832   ins_pipe(ialu_reg);
 9833 %}
 9834 
 9835 // Rotate Right by constant.
 9836 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9837 %{
 9838   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9839   match(Set dst (RotateRight src shift));
 9840   format %{ "rorxl   $dst, $src, $shift" %}
 9841   ins_encode %{
 9842     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9843   %}
 9844   ins_pipe(ialu_reg_reg);
 9845 %}
 9846 
 9847 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9848 %{
 9849   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9850   match(Set dst (RotateRight (LoadI src) shift));
 9851   ins_cost(175);
 9852   format %{ "rorxl   $dst, $src, $shift" %}
 9853   ins_encode %{
 9854     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9855   %}
 9856   ins_pipe(ialu_reg_mem);
 9857 %}
 9858 
 9859 // Rotate Right by variable
 9860 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9861 %{
 9862   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9863   match(Set dst (RotateRight dst shift));
 9864   effect(KILL cr);
 9865   format %{ "rorl    $dst, $shift" %}
 9866   ins_encode %{
 9867     __ rorl($dst$$Register);
 9868   %}
 9869   ins_pipe(ialu_reg_reg);
 9870 %}
 9871 
 9872 // Rotate Right by variable
 9873 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9874 %{
 9875   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9876   match(Set dst (RotateRight src shift));
 9877   effect(KILL cr);
 9878 
 9879   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9880   ins_encode %{
 9881     __ erorl($dst$$Register, $src$$Register, false);
 9882   %}
 9883   ins_pipe(ialu_reg_reg);
 9884 %}
 9885 
 9886 // Rotate Left by constant.
 9887 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9888 %{
 9889   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9890   match(Set dst (RotateLeft dst shift));
 9891   effect(KILL cr);
 9892   format %{ "rolq    $dst, $shift" %}
 9893   ins_encode %{
 9894     __ rolq($dst$$Register, $shift$$constant);
 9895   %}
 9896   ins_pipe(ialu_reg);
 9897 %}
 9898 
 9899 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9900 %{
 9901   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9902   match(Set dst (RotateLeft src shift));
 9903   format %{ "rolxq   $dst, $src, $shift" %}
 9904   ins_encode %{
 9905     int shift = 64 - ($shift$$constant & 63);
 9906     __ rorxq($dst$$Register, $src$$Register, shift);
 9907   %}
 9908   ins_pipe(ialu_reg_reg);
 9909 %}
 9910 
 9911 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9912 %{
 9913   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9914   match(Set dst (RotateLeft (LoadL src) shift));
 9915   ins_cost(175);
 9916   format %{ "rolxq   $dst, $src, $shift" %}
 9917   ins_encode %{
 9918     int shift = 64 - ($shift$$constant & 63);
 9919     __ rorxq($dst$$Register, $src$$Address, shift);
 9920   %}
 9921   ins_pipe(ialu_reg_mem);
 9922 %}
 9923 
 9924 // Rotate Left by variable
 9925 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9926 %{
 9927   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9928   match(Set dst (RotateLeft dst shift));
 9929   effect(KILL cr);
 9930   format %{ "rolq    $dst, $shift" %}
 9931   ins_encode %{
 9932     __ rolq($dst$$Register);
 9933   %}
 9934   ins_pipe(ialu_reg_reg);
 9935 %}
 9936 
 9937 // Rotate Left by variable
 9938 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9939 %{
 9940   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9941   match(Set dst (RotateLeft src shift));
 9942   effect(KILL cr);
 9943 
 9944   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9945   ins_encode %{
 9946     __ erolq($dst$$Register, $src$$Register, false);
 9947   %}
 9948   ins_pipe(ialu_reg_reg);
 9949 %}
 9950 
 9951 // Rotate Right by constant.
 9952 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9953 %{
 9954   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9955   match(Set dst (RotateRight dst shift));
 9956   effect(KILL cr);
 9957   format %{ "rorq    $dst, $shift" %}
 9958   ins_encode %{
 9959     __ rorq($dst$$Register, $shift$$constant);
 9960   %}
 9961   ins_pipe(ialu_reg);
 9962 %}
 9963 
 9964 // Rotate Right by constant
 9965 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9966 %{
 9967   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9968   match(Set dst (RotateRight src shift));
 9969   format %{ "rorxq   $dst, $src, $shift" %}
 9970   ins_encode %{
 9971     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9972   %}
 9973   ins_pipe(ialu_reg_reg);
 9974 %}
 9975 
 9976 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9977 %{
 9978   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9979   match(Set dst (RotateRight (LoadL src) shift));
 9980   ins_cost(175);
 9981   format %{ "rorxq   $dst, $src, $shift" %}
 9982   ins_encode %{
 9983     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9984   %}
 9985   ins_pipe(ialu_reg_mem);
 9986 %}
 9987 
 9988 // Rotate Right by variable
 9989 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9990 %{
 9991   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9992   match(Set dst (RotateRight dst shift));
 9993   effect(KILL cr);
 9994   format %{ "rorq    $dst, $shift" %}
 9995   ins_encode %{
 9996     __ rorq($dst$$Register);
 9997   %}
 9998   ins_pipe(ialu_reg_reg);
 9999 %}
10000 
10001 // Rotate Right by variable
10002 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
10003 %{
10004   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
10005   match(Set dst (RotateRight src shift));
10006   effect(KILL cr);
10007 
10008   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
10009   ins_encode %{
10010     __ erorq($dst$$Register, $src$$Register, false);
10011   %}
10012   ins_pipe(ialu_reg_reg);
10013 %}
10014 
10015 //----------------------------- CompressBits/ExpandBits ------------------------
10016 
10017 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10018   predicate(n->bottom_type()->isa_long());
10019   match(Set dst (CompressBits src mask));
10020   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10021   ins_encode %{
10022     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
10023   %}
10024   ins_pipe( pipe_slow );
10025 %}
10026 
10027 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10028   predicate(n->bottom_type()->isa_long());
10029   match(Set dst (ExpandBits src mask));
10030   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10031   ins_encode %{
10032     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10033   %}
10034   ins_pipe( pipe_slow );
10035 %}
10036 
10037 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10038   predicate(n->bottom_type()->isa_long());
10039   match(Set dst (CompressBits src (LoadL mask)));
10040   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10041   ins_encode %{
10042     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10043   %}
10044   ins_pipe( pipe_slow );
10045 %}
10046 
10047 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10048   predicate(n->bottom_type()->isa_long());
10049   match(Set dst (ExpandBits src (LoadL mask)));
10050   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10051   ins_encode %{
10052     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10053   %}
10054   ins_pipe( pipe_slow );
10055 %}
10056 
10057 
10058 // Logical Instructions
10059 
10060 // Integer Logical Instructions
10061 
10062 // And Instructions
10063 // And Register with Register
10064 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10065 %{
10066   predicate(!UseAPX);
10067   match(Set dst (AndI dst src));
10068   effect(KILL cr);
10069   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10070 
10071   format %{ "andl    $dst, $src\t# int" %}
10072   ins_encode %{
10073     __ andl($dst$$Register, $src$$Register);
10074   %}
10075   ins_pipe(ialu_reg_reg);
10076 %}
10077 
10078 // And Register with Register using New Data Destination (NDD)
10079 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10080 %{
10081   predicate(UseAPX);
10082   match(Set dst (AndI src1 src2));
10083   effect(KILL cr);
10084   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10085 
10086   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10087   ins_encode %{
10088     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10089 
10090   %}
10091   ins_pipe(ialu_reg_reg);
10092 %}
10093 
10094 // And Register with Immediate 255
10095 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10096 %{
10097   match(Set dst (AndI src mask));
10098 
10099   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10100   ins_encode %{
10101     __ movzbl($dst$$Register, $src$$Register);
10102   %}
10103   ins_pipe(ialu_reg);
10104 %}
10105 
10106 // And Register with Immediate 255 and promote to long
10107 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10108 %{
10109   match(Set dst (ConvI2L (AndI src mask)));
10110 
10111   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10112   ins_encode %{
10113     __ movzbl($dst$$Register, $src$$Register);
10114   %}
10115   ins_pipe(ialu_reg);
10116 %}
10117 
10118 // And Register with Immediate 65535
10119 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10120 %{
10121   match(Set dst (AndI src mask));
10122 
10123   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10124   ins_encode %{
10125     __ movzwl($dst$$Register, $src$$Register);
10126   %}
10127   ins_pipe(ialu_reg);
10128 %}
10129 
10130 // And Register with Immediate 65535 and promote to long
10131 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10132 %{
10133   match(Set dst (ConvI2L (AndI src mask)));
10134 
10135   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10136   ins_encode %{
10137     __ movzwl($dst$$Register, $src$$Register);
10138   %}
10139   ins_pipe(ialu_reg);
10140 %}
10141 
10142 // Can skip int2long conversions after AND with small bitmask
10143 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10144 %{
10145   predicate(VM_Version::supports_bmi2());
10146   ins_cost(125);
10147   effect(TEMP tmp, KILL cr);
10148   match(Set dst (ConvI2L (AndI src mask)));
10149   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10150   ins_encode %{
10151     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10152     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10153   %}
10154   ins_pipe(ialu_reg_reg);
10155 %}
10156 
10157 // And Register with Immediate
10158 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10159 %{
10160   predicate(!UseAPX);
10161   match(Set dst (AndI dst src));
10162   effect(KILL cr);
10163   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10164 
10165   format %{ "andl    $dst, $src\t# int" %}
10166   ins_encode %{
10167     __ andl($dst$$Register, $src$$constant);
10168   %}
10169   ins_pipe(ialu_reg);
10170 %}
10171 
10172 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10173 %{
10174   predicate(UseAPX);
10175   match(Set dst (AndI src1 src2));
10176   effect(KILL cr);
10177   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10178 
10179   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10180   ins_encode %{
10181     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10182   %}
10183   ins_pipe(ialu_reg);
10184 %}
10185 
10186 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10187 %{
10188   predicate(UseAPX);
10189   match(Set dst (AndI (LoadI src1) src2));
10190   effect(KILL cr);
10191   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10192 
10193   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10194   ins_encode %{
10195     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10196   %}
10197   ins_pipe(ialu_reg);
10198 %}
10199 
10200 // And Register with Memory
10201 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10202 %{
10203   predicate(!UseAPX);
10204   match(Set dst (AndI dst (LoadI src)));
10205   effect(KILL cr);
10206   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10207 
10208   ins_cost(150);
10209   format %{ "andl    $dst, $src\t# int" %}
10210   ins_encode %{
10211     __ andl($dst$$Register, $src$$Address);
10212   %}
10213   ins_pipe(ialu_reg_mem);
10214 %}
10215 
10216 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10217 %{
10218   predicate(UseAPX);
10219   match(Set dst (AndI src1 (LoadI src2)));
10220   effect(KILL cr);
10221   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10222 
10223   ins_cost(150);
10224   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10225   ins_encode %{
10226     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10227   %}
10228   ins_pipe(ialu_reg_mem);
10229 %}
10230 
10231 // And Memory with Register
10232 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10233 %{
10234   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10235   effect(KILL cr);
10236   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10237 
10238   ins_cost(150);
10239   format %{ "andb    $dst, $src\t# byte" %}
10240   ins_encode %{
10241     __ andb($dst$$Address, $src$$Register);
10242   %}
10243   ins_pipe(ialu_mem_reg);
10244 %}
10245 
10246 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10247 %{
10248   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10249   effect(KILL cr);
10250   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10251 
10252   ins_cost(150);
10253   format %{ "andl    $dst, $src\t# int" %}
10254   ins_encode %{
10255     __ andl($dst$$Address, $src$$Register);
10256   %}
10257   ins_pipe(ialu_mem_reg);
10258 %}
10259 
10260 // And Memory with Immediate
10261 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10262 %{
10263   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10264   effect(KILL cr);
10265   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10266 
10267   ins_cost(125);
10268   format %{ "andl    $dst, $src\t# int" %}
10269   ins_encode %{
10270     __ andl($dst$$Address, $src$$constant);
10271   %}
10272   ins_pipe(ialu_mem_imm);
10273 %}
10274 
10275 // BMI1 instructions
10276 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10277   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10278   predicate(UseBMI1Instructions);
10279   effect(KILL cr);
10280   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10281 
10282   ins_cost(125);
10283   format %{ "andnl  $dst, $src1, $src2" %}
10284 
10285   ins_encode %{
10286     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10287   %}
10288   ins_pipe(ialu_reg_mem);
10289 %}
10290 
10291 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10292   match(Set dst (AndI (XorI src1 minus_1) src2));
10293   predicate(UseBMI1Instructions);
10294   effect(KILL cr);
10295   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10296 
10297   format %{ "andnl  $dst, $src1, $src2" %}
10298 
10299   ins_encode %{
10300     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10301   %}
10302   ins_pipe(ialu_reg);
10303 %}
10304 
10305 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10306   match(Set dst (AndI (SubI imm_zero src) src));
10307   predicate(UseBMI1Instructions);
10308   effect(KILL cr);
10309   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10310 
10311   format %{ "blsil  $dst, $src" %}
10312 
10313   ins_encode %{
10314     __ blsil($dst$$Register, $src$$Register);
10315   %}
10316   ins_pipe(ialu_reg);
10317 %}
10318 
10319 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10320   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10321   predicate(UseBMI1Instructions);
10322   effect(KILL cr);
10323   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10324 
10325   ins_cost(125);
10326   format %{ "blsil  $dst, $src" %}
10327 
10328   ins_encode %{
10329     __ blsil($dst$$Register, $src$$Address);
10330   %}
10331   ins_pipe(ialu_reg_mem);
10332 %}
10333 
10334 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10335 %{
10336   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10337   predicate(UseBMI1Instructions);
10338   effect(KILL cr);
10339   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10340 
10341   ins_cost(125);
10342   format %{ "blsmskl $dst, $src" %}
10343 
10344   ins_encode %{
10345     __ blsmskl($dst$$Register, $src$$Address);
10346   %}
10347   ins_pipe(ialu_reg_mem);
10348 %}
10349 
10350 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10351 %{
10352   match(Set dst (XorI (AddI src minus_1) src));
10353   predicate(UseBMI1Instructions);
10354   effect(KILL cr);
10355   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10356 
10357   format %{ "blsmskl $dst, $src" %}
10358 
10359   ins_encode %{
10360     __ blsmskl($dst$$Register, $src$$Register);
10361   %}
10362 
10363   ins_pipe(ialu_reg);
10364 %}
10365 
10366 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10367 %{
10368   match(Set dst (AndI (AddI src minus_1) src) );
10369   predicate(UseBMI1Instructions);
10370   effect(KILL cr);
10371   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10372 
10373   format %{ "blsrl  $dst, $src" %}
10374 
10375   ins_encode %{
10376     __ blsrl($dst$$Register, $src$$Register);
10377   %}
10378 
10379   ins_pipe(ialu_reg_mem);
10380 %}
10381 
10382 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10383 %{
10384   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10385   predicate(UseBMI1Instructions);
10386   effect(KILL cr);
10387   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10388 
10389   ins_cost(125);
10390   format %{ "blsrl  $dst, $src" %}
10391 
10392   ins_encode %{
10393     __ blsrl($dst$$Register, $src$$Address);
10394   %}
10395 
10396   ins_pipe(ialu_reg);
10397 %}
10398 
10399 // Or Instructions
10400 // Or Register with Register
10401 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10402 %{
10403   predicate(!UseAPX);
10404   match(Set dst (OrI dst src));
10405   effect(KILL cr);
10406   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10407 
10408   format %{ "orl     $dst, $src\t# int" %}
10409   ins_encode %{
10410     __ orl($dst$$Register, $src$$Register);
10411   %}
10412   ins_pipe(ialu_reg_reg);
10413 %}
10414 
10415 // Or Register with Register using New Data Destination (NDD)
10416 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10417 %{
10418   predicate(UseAPX);
10419   match(Set dst (OrI src1 src2));
10420   effect(KILL cr);
10421   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10422 
10423   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10424   ins_encode %{
10425     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10426   %}
10427   ins_pipe(ialu_reg_reg);
10428 %}
10429 
10430 // Or Register with Immediate
10431 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10432 %{
10433   predicate(!UseAPX);
10434   match(Set dst (OrI dst src));
10435   effect(KILL cr);
10436   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10437 
10438   format %{ "orl     $dst, $src\t# int" %}
10439   ins_encode %{
10440     __ orl($dst$$Register, $src$$constant);
10441   %}
10442   ins_pipe(ialu_reg);
10443 %}
10444 
10445 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10446 %{
10447   predicate(UseAPX);
10448   match(Set dst (OrI src1 src2));
10449   effect(KILL cr);
10450   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10451 
10452   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10453   ins_encode %{
10454     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10455   %}
10456   ins_pipe(ialu_reg);
10457 %}
10458 
10459 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10460 %{
10461   predicate(UseAPX);
10462   match(Set dst (OrI src1 src2));
10463   effect(KILL cr);
10464   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10465 
10466   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10467   ins_encode %{
10468     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10469   %}
10470   ins_pipe(ialu_reg);
10471 %}
10472 
10473 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10474 %{
10475   predicate(UseAPX);
10476   match(Set dst (OrI (LoadI src1) src2));
10477   effect(KILL cr);
10478   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10479 
10480   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10481   ins_encode %{
10482     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10483   %}
10484   ins_pipe(ialu_reg);
10485 %}
10486 
10487 // Or Register with Memory
10488 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10489 %{
10490   predicate(!UseAPX);
10491   match(Set dst (OrI dst (LoadI src)));
10492   effect(KILL cr);
10493   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10494 
10495   ins_cost(150);
10496   format %{ "orl     $dst, $src\t# int" %}
10497   ins_encode %{
10498     __ orl($dst$$Register, $src$$Address);
10499   %}
10500   ins_pipe(ialu_reg_mem);
10501 %}
10502 
10503 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10504 %{
10505   predicate(UseAPX);
10506   match(Set dst (OrI src1 (LoadI src2)));
10507   effect(KILL cr);
10508   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10509 
10510   ins_cost(150);
10511   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10512   ins_encode %{
10513     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10514   %}
10515   ins_pipe(ialu_reg_mem);
10516 %}
10517 
10518 // Or Memory with Register
10519 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10520 %{
10521   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10522   effect(KILL cr);
10523   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10524 
10525   ins_cost(150);
10526   format %{ "orb    $dst, $src\t# byte" %}
10527   ins_encode %{
10528     __ orb($dst$$Address, $src$$Register);
10529   %}
10530   ins_pipe(ialu_mem_reg);
10531 %}
10532 
10533 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10534 %{
10535   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10536   effect(KILL cr);
10537   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10538 
10539   ins_cost(150);
10540   format %{ "orl     $dst, $src\t# int" %}
10541   ins_encode %{
10542     __ orl($dst$$Address, $src$$Register);
10543   %}
10544   ins_pipe(ialu_mem_reg);
10545 %}
10546 
10547 // Or Memory with Immediate
10548 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10549 %{
10550   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10551   effect(KILL cr);
10552   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10553 
10554   ins_cost(125);
10555   format %{ "orl     $dst, $src\t# int" %}
10556   ins_encode %{
10557     __ orl($dst$$Address, $src$$constant);
10558   %}
10559   ins_pipe(ialu_mem_imm);
10560 %}
10561 
10562 // Xor Instructions
10563 // Xor Register with Register
10564 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10565 %{
10566   predicate(!UseAPX);
10567   match(Set dst (XorI dst src));
10568   effect(KILL cr);
10569   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10570 
10571   format %{ "xorl    $dst, $src\t# int" %}
10572   ins_encode %{
10573     __ xorl($dst$$Register, $src$$Register);
10574   %}
10575   ins_pipe(ialu_reg_reg);
10576 %}
10577 
10578 // Xor Register with Register using New Data Destination (NDD)
10579 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10580 %{
10581   predicate(UseAPX);
10582   match(Set dst (XorI src1 src2));
10583   effect(KILL cr);
10584   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10585 
10586   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10587   ins_encode %{
10588     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10589   %}
10590   ins_pipe(ialu_reg_reg);
10591 %}
10592 
10593 // Xor Register with Immediate -1
10594 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10595 %{
10596   predicate(!UseAPX);
10597   match(Set dst (XorI dst imm));
10598 
10599   format %{ "notl    $dst" %}
10600   ins_encode %{
10601      __ notl($dst$$Register);
10602   %}
10603   ins_pipe(ialu_reg);
10604 %}
10605 
10606 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10607 %{
10608   match(Set dst (XorI src imm));
10609   predicate(UseAPX);
10610 
10611   format %{ "enotl    $dst, $src" %}
10612   ins_encode %{
10613      __ enotl($dst$$Register, $src$$Register);
10614   %}
10615   ins_pipe(ialu_reg);
10616 %}
10617 
10618 // Xor Register with Immediate
10619 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10620 %{
10621   predicate(!UseAPX);
10622   match(Set dst (XorI dst src));
10623   effect(KILL cr);
10624   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10625 
10626   format %{ "xorl    $dst, $src\t# int" %}
10627   ins_encode %{
10628     __ xorl($dst$$Register, $src$$constant);
10629   %}
10630   ins_pipe(ialu_reg);
10631 %}
10632 
10633 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10634 %{
10635   predicate(UseAPX);
10636   match(Set dst (XorI src1 src2));
10637   effect(KILL cr);
10638   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10639 
10640   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10641   ins_encode %{
10642     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10643   %}
10644   ins_pipe(ialu_reg);
10645 %}
10646 
10647 // Xor Memory with Immediate
10648 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10649 %{
10650   predicate(UseAPX);
10651   match(Set dst (XorI (LoadI src1) src2));
10652   effect(KILL cr);
10653   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10654 
10655   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10656   ins_encode %{
10657     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10658   %}
10659   ins_pipe(ialu_reg);
10660 %}
10661 
10662 // Xor Register with Memory
10663 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10664 %{
10665   predicate(!UseAPX);
10666   match(Set dst (XorI dst (LoadI src)));
10667   effect(KILL cr);
10668   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10669 
10670   ins_cost(150);
10671   format %{ "xorl    $dst, $src\t# int" %}
10672   ins_encode %{
10673     __ xorl($dst$$Register, $src$$Address);
10674   %}
10675   ins_pipe(ialu_reg_mem);
10676 %}
10677 
10678 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10679 %{
10680   predicate(UseAPX);
10681   match(Set dst (XorI src1 (LoadI src2)));
10682   effect(KILL cr);
10683   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10684 
10685   ins_cost(150);
10686   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10687   ins_encode %{
10688     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10689   %}
10690   ins_pipe(ialu_reg_mem);
10691 %}
10692 
10693 instruct xorI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
10694 %{
10695   predicate(UseAPX);
10696   match(Set dst (XorI (LoadI src1) src2));
10697   effect(KILL cr);
10698   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10699 
10700   ins_cost(150);
10701   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10702   ins_encode %{
10703     __ exorl($dst$$Register, $src1$$Address, $src2$$Register, false);
10704   %}
10705   ins_pipe(ialu_reg_mem);
10706 %}
10707 
10708 // Xor Memory with Register
10709 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10710 %{
10711   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10712   effect(KILL cr);
10713   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10714 
10715   ins_cost(150);
10716   format %{ "xorb    $dst, $src\t# byte" %}
10717   ins_encode %{
10718     __ xorb($dst$$Address, $src$$Register);
10719   %}
10720   ins_pipe(ialu_mem_reg);
10721 %}
10722 
10723 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10724 %{
10725   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10726   effect(KILL cr);
10727   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10728 
10729   ins_cost(150);
10730   format %{ "xorl    $dst, $src\t# int" %}
10731   ins_encode %{
10732     __ xorl($dst$$Address, $src$$Register);
10733   %}
10734   ins_pipe(ialu_mem_reg);
10735 %}
10736 
10737 // Xor Memory with Immediate
10738 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10739 %{
10740   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10741   effect(KILL cr);
10742   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10743 
10744   ins_cost(125);
10745   format %{ "xorl    $dst, $src\t# int" %}
10746   ins_encode %{
10747     __ xorl($dst$$Address, $src$$constant);
10748   %}
10749   ins_pipe(ialu_mem_imm);
10750 %}
10751 
10752 
10753 // Long Logical Instructions
10754 
10755 // And Instructions
10756 // And Register with Register
10757 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10758 %{
10759   predicate(!UseAPX);
10760   match(Set dst (AndL dst src));
10761   effect(KILL cr);
10762   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10763 
10764   format %{ "andq    $dst, $src\t# long" %}
10765   ins_encode %{
10766     __ andq($dst$$Register, $src$$Register);
10767   %}
10768   ins_pipe(ialu_reg_reg);
10769 %}
10770 
10771 // And Register with Register using New Data Destination (NDD)
10772 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10773 %{
10774   predicate(UseAPX);
10775   match(Set dst (AndL src1 src2));
10776   effect(KILL cr);
10777   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10778 
10779   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10780   ins_encode %{
10781     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10782 
10783   %}
10784   ins_pipe(ialu_reg_reg);
10785 %}
10786 
10787 // And Register with Immediate 255
10788 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10789 %{
10790   match(Set dst (AndL src mask));
10791 
10792   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10793   ins_encode %{
10794     // movzbl zeroes out the upper 32-bit and does not need REX.W
10795     __ movzbl($dst$$Register, $src$$Register);
10796   %}
10797   ins_pipe(ialu_reg);
10798 %}
10799 
10800 // And Register with Immediate 65535
10801 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10802 %{
10803   match(Set dst (AndL src mask));
10804 
10805   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10806   ins_encode %{
10807     // movzwl zeroes out the upper 32-bit and does not need REX.W
10808     __ movzwl($dst$$Register, $src$$Register);
10809   %}
10810   ins_pipe(ialu_reg);
10811 %}
10812 
10813 // And Register with Immediate
10814 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10815 %{
10816   predicate(!UseAPX);
10817   match(Set dst (AndL dst src));
10818   effect(KILL cr);
10819   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10820 
10821   format %{ "andq    $dst, $src\t# long" %}
10822   ins_encode %{
10823     __ andq($dst$$Register, $src$$constant);
10824   %}
10825   ins_pipe(ialu_reg);
10826 %}
10827 
10828 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10829 %{
10830   predicate(UseAPX);
10831   match(Set dst (AndL src1 src2));
10832   effect(KILL cr);
10833   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10834 
10835   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10836   ins_encode %{
10837     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10838   %}
10839   ins_pipe(ialu_reg);
10840 %}
10841 
10842 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10843 %{
10844   predicate(UseAPX);
10845   match(Set dst (AndL (LoadL src1) src2));
10846   effect(KILL cr);
10847   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10848 
10849   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10850   ins_encode %{
10851     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10852   %}
10853   ins_pipe(ialu_reg);
10854 %}
10855 
10856 // And Register with Memory
10857 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10858 %{
10859   predicate(!UseAPX);
10860   match(Set dst (AndL dst (LoadL src)));
10861   effect(KILL cr);
10862   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10863 
10864   ins_cost(150);
10865   format %{ "andq    $dst, $src\t# long" %}
10866   ins_encode %{
10867     __ andq($dst$$Register, $src$$Address);
10868   %}
10869   ins_pipe(ialu_reg_mem);
10870 %}
10871 
10872 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10873 %{
10874   predicate(UseAPX);
10875   match(Set dst (AndL src1 (LoadL src2)));
10876   effect(KILL cr);
10877   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10878 
10879   ins_cost(150);
10880   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10881   ins_encode %{
10882     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10883   %}
10884   ins_pipe(ialu_reg_mem);
10885 %}
10886 
10887 instruct andL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
10888 %{
10889   predicate(UseAPX);
10890   match(Set dst (AndL (LoadL src1) src2));
10891   effect(KILL cr);
10892   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10893 
10894   ins_cost(150);
10895   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10896   ins_encode %{
10897     __ eandq($dst$$Register, $src1$$Address, $src2$$Register, false);
10898   %}
10899   ins_pipe(ialu_reg_mem);
10900 %}
10901 
10902 // And Memory with Register
10903 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10904 %{
10905   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10906   effect(KILL cr);
10907   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10908 
10909   ins_cost(150);
10910   format %{ "andq    $dst, $src\t# long" %}
10911   ins_encode %{
10912     __ andq($dst$$Address, $src$$Register);
10913   %}
10914   ins_pipe(ialu_mem_reg);
10915 %}
10916 
10917 // And Memory with Immediate
10918 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10919 %{
10920   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10921   effect(KILL cr);
10922   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10923 
10924   ins_cost(125);
10925   format %{ "andq    $dst, $src\t# long" %}
10926   ins_encode %{
10927     __ andq($dst$$Address, $src$$constant);
10928   %}
10929   ins_pipe(ialu_mem_imm);
10930 %}
10931 
10932 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10933 %{
10934   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10935   // because AND/OR works well enough for 8/32-bit values.
10936   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10937 
10938   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10939   effect(KILL cr);
10940 
10941   ins_cost(125);
10942   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10943   ins_encode %{
10944     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10945   %}
10946   ins_pipe(ialu_mem_imm);
10947 %}
10948 
10949 // BMI1 instructions
10950 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10951   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10952   predicate(UseBMI1Instructions);
10953   effect(KILL cr);
10954   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10955 
10956   ins_cost(125);
10957   format %{ "andnq  $dst, $src1, $src2" %}
10958 
10959   ins_encode %{
10960     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10961   %}
10962   ins_pipe(ialu_reg_mem);
10963 %}
10964 
10965 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10966   match(Set dst (AndL (XorL src1 minus_1) src2));
10967   predicate(UseBMI1Instructions);
10968   effect(KILL cr);
10969   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10970 
10971   format %{ "andnq  $dst, $src1, $src2" %}
10972 
10973   ins_encode %{
10974   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10975   %}
10976   ins_pipe(ialu_reg_mem);
10977 %}
10978 
10979 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10980   match(Set dst (AndL (SubL imm_zero src) src));
10981   predicate(UseBMI1Instructions);
10982   effect(KILL cr);
10983   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10984 
10985   format %{ "blsiq  $dst, $src" %}
10986 
10987   ins_encode %{
10988     __ blsiq($dst$$Register, $src$$Register);
10989   %}
10990   ins_pipe(ialu_reg);
10991 %}
10992 
10993 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10994   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10995   predicate(UseBMI1Instructions);
10996   effect(KILL cr);
10997   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10998 
10999   ins_cost(125);
11000   format %{ "blsiq  $dst, $src" %}
11001 
11002   ins_encode %{
11003     __ blsiq($dst$$Register, $src$$Address);
11004   %}
11005   ins_pipe(ialu_reg_mem);
11006 %}
11007 
11008 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11009 %{
11010   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
11011   predicate(UseBMI1Instructions);
11012   effect(KILL cr);
11013   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11014 
11015   ins_cost(125);
11016   format %{ "blsmskq $dst, $src" %}
11017 
11018   ins_encode %{
11019     __ blsmskq($dst$$Register, $src$$Address);
11020   %}
11021   ins_pipe(ialu_reg_mem);
11022 %}
11023 
11024 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11025 %{
11026   match(Set dst (XorL (AddL src minus_1) src));
11027   predicate(UseBMI1Instructions);
11028   effect(KILL cr);
11029   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11030 
11031   format %{ "blsmskq $dst, $src" %}
11032 
11033   ins_encode %{
11034     __ blsmskq($dst$$Register, $src$$Register);
11035   %}
11036 
11037   ins_pipe(ialu_reg);
11038 %}
11039 
11040 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11041 %{
11042   match(Set dst (AndL (AddL src minus_1) src) );
11043   predicate(UseBMI1Instructions);
11044   effect(KILL cr);
11045   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11046 
11047   format %{ "blsrq  $dst, $src" %}
11048 
11049   ins_encode %{
11050     __ blsrq($dst$$Register, $src$$Register);
11051   %}
11052 
11053   ins_pipe(ialu_reg);
11054 %}
11055 
11056 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11057 %{
11058   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
11059   predicate(UseBMI1Instructions);
11060   effect(KILL cr);
11061   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11062 
11063   ins_cost(125);
11064   format %{ "blsrq  $dst, $src" %}
11065 
11066   ins_encode %{
11067     __ blsrq($dst$$Register, $src$$Address);
11068   %}
11069 
11070   ins_pipe(ialu_reg);
11071 %}
11072 
11073 // Or Instructions
11074 // Or Register with Register
11075 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11076 %{
11077   predicate(!UseAPX);
11078   match(Set dst (OrL dst src));
11079   effect(KILL cr);
11080   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11081 
11082   format %{ "orq     $dst, $src\t# long" %}
11083   ins_encode %{
11084     __ orq($dst$$Register, $src$$Register);
11085   %}
11086   ins_pipe(ialu_reg_reg);
11087 %}
11088 
11089 // Or Register with Register using New Data Destination (NDD)
11090 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11091 %{
11092   predicate(UseAPX);
11093   match(Set dst (OrL src1 src2));
11094   effect(KILL cr);
11095   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11096 
11097   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11098   ins_encode %{
11099     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11100 
11101   %}
11102   ins_pipe(ialu_reg_reg);
11103 %}
11104 
11105 // Use any_RegP to match R15 (TLS register) without spilling.
11106 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11107   match(Set dst (OrL dst (CastP2X src)));
11108   effect(KILL cr);
11109   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11110 
11111   format %{ "orq     $dst, $src\t# long" %}
11112   ins_encode %{
11113     __ orq($dst$$Register, $src$$Register);
11114   %}
11115   ins_pipe(ialu_reg_reg);
11116 %}
11117 
11118 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11119   match(Set dst (OrL src1 (CastP2X src2)));
11120   effect(KILL cr);
11121   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11122 
11123   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11124   ins_encode %{
11125     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11126   %}
11127   ins_pipe(ialu_reg_reg);
11128 %}
11129 
11130 // Or Register with Immediate
11131 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11132 %{
11133   predicate(!UseAPX);
11134   match(Set dst (OrL dst src));
11135   effect(KILL cr);
11136   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11137 
11138   format %{ "orq     $dst, $src\t# long" %}
11139   ins_encode %{
11140     __ orq($dst$$Register, $src$$constant);
11141   %}
11142   ins_pipe(ialu_reg);
11143 %}
11144 
11145 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11146 %{
11147   predicate(UseAPX);
11148   match(Set dst (OrL src1 src2));
11149   effect(KILL cr);
11150   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11151 
11152   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11153   ins_encode %{
11154     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11155   %}
11156   ins_pipe(ialu_reg);
11157 %}
11158 
11159 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11160 %{
11161   predicate(UseAPX);
11162   match(Set dst (OrL src1 src2));
11163   effect(KILL cr);
11164   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11165 
11166   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11167   ins_encode %{
11168     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11169   %}
11170   ins_pipe(ialu_reg);
11171 %}
11172 
11173 // Or Memory with Immediate
11174 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11175 %{
11176   predicate(UseAPX);
11177   match(Set dst (OrL (LoadL src1) src2));
11178   effect(KILL cr);
11179   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11180 
11181   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11182   ins_encode %{
11183     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11184   %}
11185   ins_pipe(ialu_reg);
11186 %}
11187 
11188 // Or Register with Memory
11189 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11190 %{
11191   predicate(!UseAPX);
11192   match(Set dst (OrL dst (LoadL src)));
11193   effect(KILL cr);
11194   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11195 
11196   ins_cost(150);
11197   format %{ "orq     $dst, $src\t# long" %}
11198   ins_encode %{
11199     __ orq($dst$$Register, $src$$Address);
11200   %}
11201   ins_pipe(ialu_reg_mem);
11202 %}
11203 
11204 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11205 %{
11206   predicate(UseAPX);
11207   match(Set dst (OrL src1 (LoadL src2)));
11208   effect(KILL cr);
11209   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11210 
11211   ins_cost(150);
11212   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11213   ins_encode %{
11214     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11215   %}
11216   ins_pipe(ialu_reg_mem);
11217 %}
11218 
11219 // Or Memory with Register
11220 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11221 %{
11222   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11223   effect(KILL cr);
11224   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11225 
11226   ins_cost(150);
11227   format %{ "orq     $dst, $src\t# long" %}
11228   ins_encode %{
11229     __ orq($dst$$Address, $src$$Register);
11230   %}
11231   ins_pipe(ialu_mem_reg);
11232 %}
11233 
11234 // Or Memory with Immediate
11235 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11236 %{
11237   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11238   effect(KILL cr);
11239   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11240 
11241   ins_cost(125);
11242   format %{ "orq     $dst, $src\t# long" %}
11243   ins_encode %{
11244     __ orq($dst$$Address, $src$$constant);
11245   %}
11246   ins_pipe(ialu_mem_imm);
11247 %}
11248 
11249 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11250 %{
11251   // con should be a pure 64-bit power of 2 immediate
11252   // because AND/OR works well enough for 8/32-bit values.
11253   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11254 
11255   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11256   effect(KILL cr);
11257 
11258   ins_cost(125);
11259   format %{ "btsq    $dst, log2($con)\t# long" %}
11260   ins_encode %{
11261     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11262   %}
11263   ins_pipe(ialu_mem_imm);
11264 %}
11265 
11266 // Xor Instructions
11267 // Xor Register with Register
11268 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11269 %{
11270   predicate(!UseAPX);
11271   match(Set dst (XorL dst src));
11272   effect(KILL cr);
11273   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11274 
11275   format %{ "xorq    $dst, $src\t# long" %}
11276   ins_encode %{
11277     __ xorq($dst$$Register, $src$$Register);
11278   %}
11279   ins_pipe(ialu_reg_reg);
11280 %}
11281 
11282 // Xor Register with Register using New Data Destination (NDD)
11283 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11284 %{
11285   predicate(UseAPX);
11286   match(Set dst (XorL src1 src2));
11287   effect(KILL cr);
11288   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11289 
11290   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11291   ins_encode %{
11292     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11293   %}
11294   ins_pipe(ialu_reg_reg);
11295 %}
11296 
11297 // Xor Register with Immediate -1
11298 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11299 %{
11300   predicate(!UseAPX);
11301   match(Set dst (XorL dst imm));
11302 
11303   format %{ "notq   $dst" %}
11304   ins_encode %{
11305      __ notq($dst$$Register);
11306   %}
11307   ins_pipe(ialu_reg);
11308 %}
11309 
11310 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11311 %{
11312   predicate(UseAPX);
11313   match(Set dst (XorL src imm));
11314 
11315   format %{ "enotq   $dst, $src" %}
11316   ins_encode %{
11317     __ enotq($dst$$Register, $src$$Register);
11318   %}
11319   ins_pipe(ialu_reg);
11320 %}
11321 
11322 // Xor Register with Immediate
11323 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11324 %{
11325   predicate(!UseAPX);
11326   match(Set dst (XorL dst src));
11327   effect(KILL cr);
11328   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11329 
11330   format %{ "xorq    $dst, $src\t# long" %}
11331   ins_encode %{
11332     __ xorq($dst$$Register, $src$$constant);
11333   %}
11334   ins_pipe(ialu_reg);
11335 %}
11336 
11337 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11338 %{
11339   predicate(UseAPX);
11340   match(Set dst (XorL src1 src2));
11341   effect(KILL cr);
11342   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11343 
11344   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11345   ins_encode %{
11346     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11347   %}
11348   ins_pipe(ialu_reg);
11349 %}
11350 
11351 // Xor Memory with Immediate
11352 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11353 %{
11354   predicate(UseAPX);
11355   match(Set dst (XorL (LoadL src1) src2));
11356   effect(KILL cr);
11357   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11358 
11359   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11360   ins_encode %{
11361     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11362   %}
11363   ins_pipe(ialu_reg);
11364 %}
11365 
11366 // Xor Register with Memory
11367 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11368 %{
11369   predicate(!UseAPX);
11370   match(Set dst (XorL dst (LoadL src)));
11371   effect(KILL cr);
11372   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11373 
11374   ins_cost(150);
11375   format %{ "xorq    $dst, $src\t# long" %}
11376   ins_encode %{
11377     __ xorq($dst$$Register, $src$$Address);
11378   %}
11379   ins_pipe(ialu_reg_mem);
11380 %}
11381 
11382 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11383 %{
11384   predicate(UseAPX);
11385   match(Set dst (XorL src1 (LoadL src2)));
11386   effect(KILL cr);
11387   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11388 
11389   ins_cost(150);
11390   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11391   ins_encode %{
11392     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11393   %}
11394   ins_pipe(ialu_reg_mem);
11395 %}
11396 
11397 instruct xorL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
11398 %{
11399   predicate(UseAPX);
11400   match(Set dst (XorL (LoadL src1) src2));
11401   effect(KILL cr);
11402   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11403 
11404   ins_cost(150);
11405   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11406   ins_encode %{
11407     __ exorq($dst$$Register, $src1$$Address, $src2$$Register, false);
11408   %}
11409   ins_pipe(ialu_reg_mem);
11410 %}
11411 
11412 // Xor Memory with Register
11413 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11414 %{
11415   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11416   effect(KILL cr);
11417   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11418 
11419   ins_cost(150);
11420   format %{ "xorq    $dst, $src\t# long" %}
11421   ins_encode %{
11422     __ xorq($dst$$Address, $src$$Register);
11423   %}
11424   ins_pipe(ialu_mem_reg);
11425 %}
11426 
11427 // Xor Memory with Immediate
11428 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11429 %{
11430   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11431   effect(KILL cr);
11432   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11433 
11434   ins_cost(125);
11435   format %{ "xorq    $dst, $src\t# long" %}
11436   ins_encode %{
11437     __ xorq($dst$$Address, $src$$constant);
11438   %}
11439   ins_pipe(ialu_mem_imm);
11440 %}
11441 
11442 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11443 %{
11444   match(Set dst (CmpLTMask p q));
11445   effect(KILL cr);
11446 
11447   ins_cost(400);
11448   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11449             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11450             "negl    $dst" %}
11451   ins_encode %{
11452     __ cmpl($p$$Register, $q$$Register);
11453     __ setcc(Assembler::less, $dst$$Register);
11454     __ negl($dst$$Register);
11455   %}
11456   ins_pipe(pipe_slow);
11457 %}
11458 
11459 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11460 %{
11461   match(Set dst (CmpLTMask dst zero));
11462   effect(KILL cr);
11463 
11464   ins_cost(100);
11465   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11466   ins_encode %{
11467     __ sarl($dst$$Register, 31);
11468   %}
11469   ins_pipe(ialu_reg);
11470 %}
11471 
11472 /* Better to save a register than avoid a branch */
11473 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11474 %{
11475   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11476   effect(KILL cr);
11477   ins_cost(300);
11478   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11479             "jge     done\n\t"
11480             "addl    $p,$y\n"
11481             "done:   " %}
11482   ins_encode %{
11483     Register Rp = $p$$Register;
11484     Register Rq = $q$$Register;
11485     Register Ry = $y$$Register;
11486     Label done;
11487     __ subl(Rp, Rq);
11488     __ jccb(Assembler::greaterEqual, done);
11489     __ addl(Rp, Ry);
11490     __ bind(done);
11491   %}
11492   ins_pipe(pipe_cmplt);
11493 %}
11494 
11495 /* Better to save a register than avoid a branch */
11496 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11497 %{
11498   match(Set y (AndI (CmpLTMask p q) y));
11499   effect(KILL cr);
11500 
11501   ins_cost(300);
11502 
11503   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11504             "jlt     done\n\t"
11505             "xorl    $y, $y\n"
11506             "done:   " %}
11507   ins_encode %{
11508     Register Rp = $p$$Register;
11509     Register Rq = $q$$Register;
11510     Register Ry = $y$$Register;
11511     Label done;
11512     __ cmpl(Rp, Rq);
11513     __ jccb(Assembler::less, done);
11514     __ xorl(Ry, Ry);
11515     __ bind(done);
11516   %}
11517   ins_pipe(pipe_cmplt);
11518 %}
11519 
11520 
11521 //---------- FP Instructions------------------------------------------------
11522 
11523 // Really expensive, avoid
11524 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11525 %{
11526   match(Set cr (CmpF src1 src2));
11527 
11528   ins_cost(500);
11529   format %{ "ucomiss $src1, $src2\n\t"
11530             "jnp,s   exit\n\t"
11531             "pushfq\t# saw NaN, set CF\n\t"
11532             "andq    [rsp], #0xffffff2b\n\t"
11533             "popfq\n"
11534     "exit:" %}
11535   ins_encode %{
11536     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11537     emit_cmpfp_fixup(masm);
11538   %}
11539   ins_pipe(pipe_slow);
11540 %}
11541 
11542 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11543   match(Set cr (CmpF src1 src2));
11544 
11545   ins_cost(100);
11546   format %{ "ucomiss $src1, $src2" %}
11547   ins_encode %{
11548     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11549   %}
11550   ins_pipe(pipe_slow);
11551 %}
11552 
11553 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11554   match(Set cr (CmpF src1 (LoadF src2)));
11555 
11556   ins_cost(100);
11557   format %{ "ucomiss $src1, $src2" %}
11558   ins_encode %{
11559     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11560   %}
11561   ins_pipe(pipe_slow);
11562 %}
11563 
11564 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11565   match(Set cr (CmpF src con));
11566   ins_cost(100);
11567   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11568   ins_encode %{
11569     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11570   %}
11571   ins_pipe(pipe_slow);
11572 %}
11573 
11574 // Really expensive, avoid
11575 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11576 %{
11577   match(Set cr (CmpD src1 src2));
11578 
11579   ins_cost(500);
11580   format %{ "ucomisd $src1, $src2\n\t"
11581             "jnp,s   exit\n\t"
11582             "pushfq\t# saw NaN, set CF\n\t"
11583             "andq    [rsp], #0xffffff2b\n\t"
11584             "popfq\n"
11585     "exit:" %}
11586   ins_encode %{
11587     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11588     emit_cmpfp_fixup(masm);
11589   %}
11590   ins_pipe(pipe_slow);
11591 %}
11592 
11593 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11594   match(Set cr (CmpD src1 src2));
11595 
11596   ins_cost(100);
11597   format %{ "ucomisd $src1, $src2 test" %}
11598   ins_encode %{
11599     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11600   %}
11601   ins_pipe(pipe_slow);
11602 %}
11603 
11604 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11605   match(Set cr (CmpD src1 (LoadD src2)));
11606 
11607   ins_cost(100);
11608   format %{ "ucomisd $src1, $src2" %}
11609   ins_encode %{
11610     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11611   %}
11612   ins_pipe(pipe_slow);
11613 %}
11614 
11615 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11616   match(Set cr (CmpD src con));
11617   ins_cost(100);
11618   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11619   ins_encode %{
11620     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11621   %}
11622   ins_pipe(pipe_slow);
11623 %}
11624 
11625 // Compare into -1,0,1
11626 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11627 %{
11628   match(Set dst (CmpF3 src1 src2));
11629   effect(KILL cr);
11630 
11631   ins_cost(275);
11632   format %{ "ucomiss $src1, $src2\n\t"
11633             "movl    $dst, #-1\n\t"
11634             "jp,s    done\n\t"
11635             "jb,s    done\n\t"
11636             "setne   $dst\n\t"
11637             "movzbl  $dst, $dst\n"
11638     "done:" %}
11639   ins_encode %{
11640     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11641     emit_cmpfp3(masm, $dst$$Register);
11642   %}
11643   ins_pipe(pipe_slow);
11644 %}
11645 
11646 // Compare into -1,0,1
11647 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11648 %{
11649   match(Set dst (CmpF3 src1 (LoadF src2)));
11650   effect(KILL cr);
11651 
11652   ins_cost(275);
11653   format %{ "ucomiss $src1, $src2\n\t"
11654             "movl    $dst, #-1\n\t"
11655             "jp,s    done\n\t"
11656             "jb,s    done\n\t"
11657             "setne   $dst\n\t"
11658             "movzbl  $dst, $dst\n"
11659     "done:" %}
11660   ins_encode %{
11661     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11662     emit_cmpfp3(masm, $dst$$Register);
11663   %}
11664   ins_pipe(pipe_slow);
11665 %}
11666 
11667 // Compare into -1,0,1
11668 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11669   match(Set dst (CmpF3 src con));
11670   effect(KILL cr);
11671 
11672   ins_cost(275);
11673   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11674             "movl    $dst, #-1\n\t"
11675             "jp,s    done\n\t"
11676             "jb,s    done\n\t"
11677             "setne   $dst\n\t"
11678             "movzbl  $dst, $dst\n"
11679     "done:" %}
11680   ins_encode %{
11681     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11682     emit_cmpfp3(masm, $dst$$Register);
11683   %}
11684   ins_pipe(pipe_slow);
11685 %}
11686 
11687 // Compare into -1,0,1
11688 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11689 %{
11690   match(Set dst (CmpD3 src1 src2));
11691   effect(KILL cr);
11692 
11693   ins_cost(275);
11694   format %{ "ucomisd $src1, $src2\n\t"
11695             "movl    $dst, #-1\n\t"
11696             "jp,s    done\n\t"
11697             "jb,s    done\n\t"
11698             "setne   $dst\n\t"
11699             "movzbl  $dst, $dst\n"
11700     "done:" %}
11701   ins_encode %{
11702     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11703     emit_cmpfp3(masm, $dst$$Register);
11704   %}
11705   ins_pipe(pipe_slow);
11706 %}
11707 
11708 // Compare into -1,0,1
11709 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11710 %{
11711   match(Set dst (CmpD3 src1 (LoadD src2)));
11712   effect(KILL cr);
11713 
11714   ins_cost(275);
11715   format %{ "ucomisd $src1, $src2\n\t"
11716             "movl    $dst, #-1\n\t"
11717             "jp,s    done\n\t"
11718             "jb,s    done\n\t"
11719             "setne   $dst\n\t"
11720             "movzbl  $dst, $dst\n"
11721     "done:" %}
11722   ins_encode %{
11723     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11724     emit_cmpfp3(masm, $dst$$Register);
11725   %}
11726   ins_pipe(pipe_slow);
11727 %}
11728 
11729 // Compare into -1,0,1
11730 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11731   match(Set dst (CmpD3 src con));
11732   effect(KILL cr);
11733 
11734   ins_cost(275);
11735   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11736             "movl    $dst, #-1\n\t"
11737             "jp,s    done\n\t"
11738             "jb,s    done\n\t"
11739             "setne   $dst\n\t"
11740             "movzbl  $dst, $dst\n"
11741     "done:" %}
11742   ins_encode %{
11743     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11744     emit_cmpfp3(masm, $dst$$Register);
11745   %}
11746   ins_pipe(pipe_slow);
11747 %}
11748 
11749 //----------Arithmetic Conversion Instructions---------------------------------
11750 
11751 instruct convF2D_reg_reg(regD dst, regF src)
11752 %{
11753   match(Set dst (ConvF2D src));
11754 
11755   format %{ "cvtss2sd $dst, $src" %}
11756   ins_encode %{
11757     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11758   %}
11759   ins_pipe(pipe_slow); // XXX
11760 %}
11761 
11762 instruct convF2D_reg_mem(regD dst, memory src)
11763 %{
11764   predicate(UseAVX == 0);
11765   match(Set dst (ConvF2D (LoadF src)));
11766 
11767   format %{ "cvtss2sd $dst, $src" %}
11768   ins_encode %{
11769     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11770   %}
11771   ins_pipe(pipe_slow); // XXX
11772 %}
11773 
11774 instruct convD2F_reg_reg(regF dst, regD src)
11775 %{
11776   match(Set dst (ConvD2F src));
11777 
11778   format %{ "cvtsd2ss $dst, $src" %}
11779   ins_encode %{
11780     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11781   %}
11782   ins_pipe(pipe_slow); // XXX
11783 %}
11784 
11785 instruct convD2F_reg_mem(regF dst, memory src)
11786 %{
11787   predicate(UseAVX == 0);
11788   match(Set dst (ConvD2F (LoadD src)));
11789 
11790   format %{ "cvtsd2ss $dst, $src" %}
11791   ins_encode %{
11792     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11793   %}
11794   ins_pipe(pipe_slow); // XXX
11795 %}
11796 
11797 // XXX do mem variants
11798 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11799 %{
11800   match(Set dst (ConvF2I src));
11801   effect(KILL cr);
11802   format %{ "convert_f2i $dst, $src" %}
11803   ins_encode %{
11804     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11805   %}
11806   ins_pipe(pipe_slow);
11807 %}
11808 
11809 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11810 %{
11811   match(Set dst (ConvF2L src));
11812   effect(KILL cr);
11813   format %{ "convert_f2l $dst, $src"%}
11814   ins_encode %{
11815     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11816   %}
11817   ins_pipe(pipe_slow);
11818 %}
11819 
11820 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11821 %{
11822   match(Set dst (ConvD2I src));
11823   effect(KILL cr);
11824   format %{ "convert_d2i $dst, $src"%}
11825   ins_encode %{
11826     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11827   %}
11828   ins_pipe(pipe_slow);
11829 %}
11830 
11831 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11832 %{
11833   match(Set dst (ConvD2L src));
11834   effect(KILL cr);
11835   format %{ "convert_d2l $dst, $src"%}
11836   ins_encode %{
11837     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11838   %}
11839   ins_pipe(pipe_slow);
11840 %}
11841 
11842 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11843 %{
11844   match(Set dst (RoundD src));
11845   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11846   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11847   ins_encode %{
11848     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11849   %}
11850   ins_pipe(pipe_slow);
11851 %}
11852 
11853 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11854 %{
11855   match(Set dst (RoundF src));
11856   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11857   format %{ "round_float $dst,$src" %}
11858   ins_encode %{
11859     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11860   %}
11861   ins_pipe(pipe_slow);
11862 %}
11863 
11864 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11865 %{
11866   predicate(!UseXmmI2F);
11867   match(Set dst (ConvI2F src));
11868 
11869   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11870   ins_encode %{
11871     if (UseAVX > 0) {
11872       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11873     }
11874     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11875   %}
11876   ins_pipe(pipe_slow); // XXX
11877 %}
11878 
11879 instruct convI2F_reg_mem(regF dst, memory src)
11880 %{
11881   predicate(UseAVX == 0);
11882   match(Set dst (ConvI2F (LoadI src)));
11883 
11884   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11885   ins_encode %{
11886     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11887   %}
11888   ins_pipe(pipe_slow); // XXX
11889 %}
11890 
11891 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11892 %{
11893   predicate(!UseXmmI2D);
11894   match(Set dst (ConvI2D src));
11895 
11896   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11897   ins_encode %{
11898     if (UseAVX > 0) {
11899       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11900     }
11901     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11902   %}
11903   ins_pipe(pipe_slow); // XXX
11904 %}
11905 
11906 instruct convI2D_reg_mem(regD dst, memory src)
11907 %{
11908   predicate(UseAVX == 0);
11909   match(Set dst (ConvI2D (LoadI src)));
11910 
11911   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11912   ins_encode %{
11913     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11914   %}
11915   ins_pipe(pipe_slow); // XXX
11916 %}
11917 
11918 instruct convXI2F_reg(regF dst, rRegI src)
11919 %{
11920   predicate(UseXmmI2F);
11921   match(Set dst (ConvI2F src));
11922 
11923   format %{ "movdl $dst, $src\n\t"
11924             "cvtdq2psl $dst, $dst\t# i2f" %}
11925   ins_encode %{
11926     __ movdl($dst$$XMMRegister, $src$$Register);
11927     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11928   %}
11929   ins_pipe(pipe_slow); // XXX
11930 %}
11931 
11932 instruct convXI2D_reg(regD dst, rRegI src)
11933 %{
11934   predicate(UseXmmI2D);
11935   match(Set dst (ConvI2D src));
11936 
11937   format %{ "movdl $dst, $src\n\t"
11938             "cvtdq2pdl $dst, $dst\t# i2d" %}
11939   ins_encode %{
11940     __ movdl($dst$$XMMRegister, $src$$Register);
11941     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11942   %}
11943   ins_pipe(pipe_slow); // XXX
11944 %}
11945 
11946 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11947 %{
11948   match(Set dst (ConvL2F src));
11949 
11950   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11951   ins_encode %{
11952     if (UseAVX > 0) {
11953       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11954     }
11955     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11956   %}
11957   ins_pipe(pipe_slow); // XXX
11958 %}
11959 
11960 instruct convL2F_reg_mem(regF dst, memory src)
11961 %{
11962   predicate(UseAVX == 0);
11963   match(Set dst (ConvL2F (LoadL src)));
11964 
11965   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11966   ins_encode %{
11967     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11968   %}
11969   ins_pipe(pipe_slow); // XXX
11970 %}
11971 
11972 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11973 %{
11974   match(Set dst (ConvL2D src));
11975 
11976   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11977   ins_encode %{
11978     if (UseAVX > 0) {
11979       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11980     }
11981     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11982   %}
11983   ins_pipe(pipe_slow); // XXX
11984 %}
11985 
11986 instruct convL2D_reg_mem(regD dst, memory src)
11987 %{
11988   predicate(UseAVX == 0);
11989   match(Set dst (ConvL2D (LoadL src)));
11990 
11991   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11992   ins_encode %{
11993     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11994   %}
11995   ins_pipe(pipe_slow); // XXX
11996 %}
11997 
11998 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11999 %{
12000   match(Set dst (ConvI2L src));
12001 
12002   ins_cost(125);
12003   format %{ "movslq  $dst, $src\t# i2l" %}
12004   ins_encode %{
12005     __ movslq($dst$$Register, $src$$Register);
12006   %}
12007   ins_pipe(ialu_reg_reg);
12008 %}
12009 
12010 // Zero-extend convert int to long
12011 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
12012 %{
12013   match(Set dst (AndL (ConvI2L src) mask));
12014 
12015   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12016   ins_encode %{
12017     if ($dst$$reg != $src$$reg) {
12018       __ movl($dst$$Register, $src$$Register);
12019     }
12020   %}
12021   ins_pipe(ialu_reg_reg);
12022 %}
12023 
12024 // Zero-extend convert int to long
12025 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
12026 %{
12027   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
12028 
12029   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12030   ins_encode %{
12031     __ movl($dst$$Register, $src$$Address);
12032   %}
12033   ins_pipe(ialu_reg_mem);
12034 %}
12035 
12036 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
12037 %{
12038   match(Set dst (AndL src mask));
12039 
12040   format %{ "movl    $dst, $src\t# zero-extend long" %}
12041   ins_encode %{
12042     __ movl($dst$$Register, $src$$Register);
12043   %}
12044   ins_pipe(ialu_reg_reg);
12045 %}
12046 
12047 instruct convL2I_reg_reg(rRegI dst, rRegL src)
12048 %{
12049   match(Set dst (ConvL2I src));
12050 
12051   format %{ "movl    $dst, $src\t# l2i" %}
12052   ins_encode %{
12053     __ movl($dst$$Register, $src$$Register);
12054   %}
12055   ins_pipe(ialu_reg_reg);
12056 %}
12057 
12058 
12059 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
12060   match(Set dst (MoveF2I src));
12061   effect(DEF dst, USE src);
12062 
12063   ins_cost(125);
12064   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
12065   ins_encode %{
12066     __ movl($dst$$Register, Address(rsp, $src$$disp));
12067   %}
12068   ins_pipe(ialu_reg_mem);
12069 %}
12070 
12071 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12072   match(Set dst (MoveI2F src));
12073   effect(DEF dst, USE src);
12074 
12075   ins_cost(125);
12076   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
12077   ins_encode %{
12078     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
12079   %}
12080   ins_pipe(pipe_slow);
12081 %}
12082 
12083 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12084   match(Set dst (MoveD2L src));
12085   effect(DEF dst, USE src);
12086 
12087   ins_cost(125);
12088   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12089   ins_encode %{
12090     __ movq($dst$$Register, Address(rsp, $src$$disp));
12091   %}
12092   ins_pipe(ialu_reg_mem);
12093 %}
12094 
12095 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12096   predicate(!UseXmmLoadAndClearUpper);
12097   match(Set dst (MoveL2D src));
12098   effect(DEF dst, USE src);
12099 
12100   ins_cost(125);
12101   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12102   ins_encode %{
12103     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12104   %}
12105   ins_pipe(pipe_slow);
12106 %}
12107 
12108 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12109   predicate(UseXmmLoadAndClearUpper);
12110   match(Set dst (MoveL2D src));
12111   effect(DEF dst, USE src);
12112 
12113   ins_cost(125);
12114   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12115   ins_encode %{
12116     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12117   %}
12118   ins_pipe(pipe_slow);
12119 %}
12120 
12121 
12122 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12123   match(Set dst (MoveF2I src));
12124   effect(DEF dst, USE src);
12125 
12126   ins_cost(95); // XXX
12127   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12128   ins_encode %{
12129     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12130   %}
12131   ins_pipe(pipe_slow);
12132 %}
12133 
12134 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12135   match(Set dst (MoveI2F src));
12136   effect(DEF dst, USE src);
12137 
12138   ins_cost(100);
12139   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12140   ins_encode %{
12141     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12142   %}
12143   ins_pipe( ialu_mem_reg );
12144 %}
12145 
12146 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12147   match(Set dst (MoveD2L src));
12148   effect(DEF dst, USE src);
12149 
12150   ins_cost(95); // XXX
12151   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12152   ins_encode %{
12153     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12154   %}
12155   ins_pipe(pipe_slow);
12156 %}
12157 
12158 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12159   match(Set dst (MoveL2D src));
12160   effect(DEF dst, USE src);
12161 
12162   ins_cost(100);
12163   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12164   ins_encode %{
12165     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12166   %}
12167   ins_pipe(ialu_mem_reg);
12168 %}
12169 
12170 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12171   match(Set dst (MoveF2I src));
12172   effect(DEF dst, USE src);
12173   ins_cost(85);
12174   format %{ "movd    $dst,$src\t# MoveF2I" %}
12175   ins_encode %{
12176     __ movdl($dst$$Register, $src$$XMMRegister);
12177   %}
12178   ins_pipe( pipe_slow );
12179 %}
12180 
12181 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12182   match(Set dst (MoveD2L src));
12183   effect(DEF dst, USE src);
12184   ins_cost(85);
12185   format %{ "movd    $dst,$src\t# MoveD2L" %}
12186   ins_encode %{
12187     __ movdq($dst$$Register, $src$$XMMRegister);
12188   %}
12189   ins_pipe( pipe_slow );
12190 %}
12191 
12192 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12193   match(Set dst (MoveI2F src));
12194   effect(DEF dst, USE src);
12195   ins_cost(100);
12196   format %{ "movd    $dst,$src\t# MoveI2F" %}
12197   ins_encode %{
12198     __ movdl($dst$$XMMRegister, $src$$Register);
12199   %}
12200   ins_pipe( pipe_slow );
12201 %}
12202 
12203 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12204   match(Set dst (MoveL2D src));
12205   effect(DEF dst, USE src);
12206   ins_cost(100);
12207   format %{ "movd    $dst,$src\t# MoveL2D" %}
12208   ins_encode %{
12209      __ movdq($dst$$XMMRegister, $src$$Register);
12210   %}
12211   ins_pipe( pipe_slow );
12212 %}
12213 
12214 // Fast clearing of an array
12215 // Small non-constant lenght ClearArray for non-AVX512 targets.
12216 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12217                   Universe dummy, rFlagsReg cr)
12218 %{
12219   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12220   match(Set dummy (ClearArray cnt base));
12221   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12222 
12223   format %{ $$template
12224     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12225     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12226     $$emit$$"jg      LARGE\n\t"
12227     $$emit$$"dec     rcx\n\t"
12228     $$emit$$"js      DONE\t# Zero length\n\t"
12229     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12230     $$emit$$"dec     rcx\n\t"
12231     $$emit$$"jge     LOOP\n\t"
12232     $$emit$$"jmp     DONE\n\t"
12233     $$emit$$"# LARGE:\n\t"
12234     if (UseFastStosb) {
12235        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12236        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12237     } else if (UseXMMForObjInit) {
12238        $$emit$$"mov     rdi,rax\n\t"
12239        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12240        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12241        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12242        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12243        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12244        $$emit$$"add     0x40,rax\n\t"
12245        $$emit$$"# L_zero_64_bytes:\n\t"
12246        $$emit$$"sub     0x8,rcx\n\t"
12247        $$emit$$"jge     L_loop\n\t"
12248        $$emit$$"add     0x4,rcx\n\t"
12249        $$emit$$"jl      L_tail\n\t"
12250        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12251        $$emit$$"add     0x20,rax\n\t"
12252        $$emit$$"sub     0x4,rcx\n\t"
12253        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12254        $$emit$$"add     0x4,rcx\n\t"
12255        $$emit$$"jle     L_end\n\t"
12256        $$emit$$"dec     rcx\n\t"
12257        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12258        $$emit$$"vmovq   xmm0,(rax)\n\t"
12259        $$emit$$"add     0x8,rax\n\t"
12260        $$emit$$"dec     rcx\n\t"
12261        $$emit$$"jge     L_sloop\n\t"
12262        $$emit$$"# L_end:\n\t"
12263     } else {
12264        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12265     }
12266     $$emit$$"# DONE"
12267   %}
12268   ins_encode %{
12269     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12270                  $tmp$$XMMRegister, false, knoreg);
12271   %}
12272   ins_pipe(pipe_slow);
12273 %}
12274 
12275 // Small non-constant length ClearArray for AVX512 targets.
12276 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12277                        Universe dummy, rFlagsReg cr)
12278 %{
12279   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12280   match(Set dummy (ClearArray cnt base));
12281   ins_cost(125);
12282   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12283 
12284   format %{ $$template
12285     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12286     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12287     $$emit$$"jg      LARGE\n\t"
12288     $$emit$$"dec     rcx\n\t"
12289     $$emit$$"js      DONE\t# Zero length\n\t"
12290     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12291     $$emit$$"dec     rcx\n\t"
12292     $$emit$$"jge     LOOP\n\t"
12293     $$emit$$"jmp     DONE\n\t"
12294     $$emit$$"# LARGE:\n\t"
12295     if (UseFastStosb) {
12296        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12297        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12298     } else if (UseXMMForObjInit) {
12299        $$emit$$"mov     rdi,rax\n\t"
12300        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12301        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12302        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12303        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12304        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12305        $$emit$$"add     0x40,rax\n\t"
12306        $$emit$$"# L_zero_64_bytes:\n\t"
12307        $$emit$$"sub     0x8,rcx\n\t"
12308        $$emit$$"jge     L_loop\n\t"
12309        $$emit$$"add     0x4,rcx\n\t"
12310        $$emit$$"jl      L_tail\n\t"
12311        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12312        $$emit$$"add     0x20,rax\n\t"
12313        $$emit$$"sub     0x4,rcx\n\t"
12314        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12315        $$emit$$"add     0x4,rcx\n\t"
12316        $$emit$$"jle     L_end\n\t"
12317        $$emit$$"dec     rcx\n\t"
12318        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12319        $$emit$$"vmovq   xmm0,(rax)\n\t"
12320        $$emit$$"add     0x8,rax\n\t"
12321        $$emit$$"dec     rcx\n\t"
12322        $$emit$$"jge     L_sloop\n\t"
12323        $$emit$$"# L_end:\n\t"
12324     } else {
12325        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12326     }
12327     $$emit$$"# DONE"
12328   %}
12329   ins_encode %{
12330     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12331                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12332   %}
12333   ins_pipe(pipe_slow);
12334 %}
12335 
12336 // Large non-constant length ClearArray for non-AVX512 targets.
12337 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12338                         Universe dummy, rFlagsReg cr)
12339 %{
12340   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12341   match(Set dummy (ClearArray cnt base));
12342   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12343 
12344   format %{ $$template
12345     if (UseFastStosb) {
12346        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12347        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12348        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12349     } else if (UseXMMForObjInit) {
12350        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12351        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12352        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12353        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12354        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12355        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12356        $$emit$$"add     0x40,rax\n\t"
12357        $$emit$$"# L_zero_64_bytes:\n\t"
12358        $$emit$$"sub     0x8,rcx\n\t"
12359        $$emit$$"jge     L_loop\n\t"
12360        $$emit$$"add     0x4,rcx\n\t"
12361        $$emit$$"jl      L_tail\n\t"
12362        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12363        $$emit$$"add     0x20,rax\n\t"
12364        $$emit$$"sub     0x4,rcx\n\t"
12365        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12366        $$emit$$"add     0x4,rcx\n\t"
12367        $$emit$$"jle     L_end\n\t"
12368        $$emit$$"dec     rcx\n\t"
12369        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12370        $$emit$$"vmovq   xmm0,(rax)\n\t"
12371        $$emit$$"add     0x8,rax\n\t"
12372        $$emit$$"dec     rcx\n\t"
12373        $$emit$$"jge     L_sloop\n\t"
12374        $$emit$$"# L_end:\n\t"
12375     } else {
12376        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12377        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12378     }
12379   %}
12380   ins_encode %{
12381     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12382                  $tmp$$XMMRegister, true, knoreg);
12383   %}
12384   ins_pipe(pipe_slow);
12385 %}
12386 
12387 // Large non-constant length ClearArray for AVX512 targets.
12388 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12389                              Universe dummy, rFlagsReg cr)
12390 %{
12391   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12392   match(Set dummy (ClearArray cnt base));
12393   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12394 
12395   format %{ $$template
12396     if (UseFastStosb) {
12397        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12398        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12399        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12400     } else if (UseXMMForObjInit) {
12401        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12402        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12403        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12404        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12405        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12406        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12407        $$emit$$"add     0x40,rax\n\t"
12408        $$emit$$"# L_zero_64_bytes:\n\t"
12409        $$emit$$"sub     0x8,rcx\n\t"
12410        $$emit$$"jge     L_loop\n\t"
12411        $$emit$$"add     0x4,rcx\n\t"
12412        $$emit$$"jl      L_tail\n\t"
12413        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12414        $$emit$$"add     0x20,rax\n\t"
12415        $$emit$$"sub     0x4,rcx\n\t"
12416        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12417        $$emit$$"add     0x4,rcx\n\t"
12418        $$emit$$"jle     L_end\n\t"
12419        $$emit$$"dec     rcx\n\t"
12420        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12421        $$emit$$"vmovq   xmm0,(rax)\n\t"
12422        $$emit$$"add     0x8,rax\n\t"
12423        $$emit$$"dec     rcx\n\t"
12424        $$emit$$"jge     L_sloop\n\t"
12425        $$emit$$"# L_end:\n\t"
12426     } else {
12427        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12428        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12429     }
12430   %}
12431   ins_encode %{
12432     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12433                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12434   %}
12435   ins_pipe(pipe_slow);
12436 %}
12437 
12438 // Small constant length ClearArray for AVX512 targets.
12439 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12440 %{
12441   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12442   match(Set dummy (ClearArray cnt base));
12443   ins_cost(100);
12444   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12445   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12446   ins_encode %{
12447    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12448   %}
12449   ins_pipe(pipe_slow);
12450 %}
12451 
12452 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12453                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12454 %{
12455   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12456   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12457   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12458 
12459   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12460   ins_encode %{
12461     __ string_compare($str1$$Register, $str2$$Register,
12462                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12463                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12464   %}
12465   ins_pipe( pipe_slow );
12466 %}
12467 
12468 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12469                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12470 %{
12471   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12472   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12473   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12474 
12475   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12476   ins_encode %{
12477     __ string_compare($str1$$Register, $str2$$Register,
12478                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12479                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12480   %}
12481   ins_pipe( pipe_slow );
12482 %}
12483 
12484 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12485                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12486 %{
12487   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12488   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12489   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12490 
12491   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12492   ins_encode %{
12493     __ string_compare($str1$$Register, $str2$$Register,
12494                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12495                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12496   %}
12497   ins_pipe( pipe_slow );
12498 %}
12499 
12500 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12501                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12502 %{
12503   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12504   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12505   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12506 
12507   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12508   ins_encode %{
12509     __ string_compare($str1$$Register, $str2$$Register,
12510                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12511                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12512   %}
12513   ins_pipe( pipe_slow );
12514 %}
12515 
12516 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12517                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12518 %{
12519   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12520   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12521   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12522 
12523   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12524   ins_encode %{
12525     __ string_compare($str1$$Register, $str2$$Register,
12526                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12527                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12528   %}
12529   ins_pipe( pipe_slow );
12530 %}
12531 
12532 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12533                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12534 %{
12535   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12536   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12537   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12538 
12539   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12540   ins_encode %{
12541     __ string_compare($str1$$Register, $str2$$Register,
12542                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12543                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12544   %}
12545   ins_pipe( pipe_slow );
12546 %}
12547 
12548 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12549                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12550 %{
12551   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12552   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12553   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12554 
12555   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12556   ins_encode %{
12557     __ string_compare($str2$$Register, $str1$$Register,
12558                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12559                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12560   %}
12561   ins_pipe( pipe_slow );
12562 %}
12563 
12564 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12565                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12566 %{
12567   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12568   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12569   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12570 
12571   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12572   ins_encode %{
12573     __ string_compare($str2$$Register, $str1$$Register,
12574                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12575                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12576   %}
12577   ins_pipe( pipe_slow );
12578 %}
12579 
12580 // fast search of substring with known size.
12581 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12582                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12583 %{
12584   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12585   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12586   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12587 
12588   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12589   ins_encode %{
12590     int icnt2 = (int)$int_cnt2$$constant;
12591     if (icnt2 >= 16) {
12592       // IndexOf for constant substrings with size >= 16 elements
12593       // which don't need to be loaded through stack.
12594       __ string_indexofC8($str1$$Register, $str2$$Register,
12595                           $cnt1$$Register, $cnt2$$Register,
12596                           icnt2, $result$$Register,
12597                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12598     } else {
12599       // Small strings are loaded through stack if they cross page boundary.
12600       __ string_indexof($str1$$Register, $str2$$Register,
12601                         $cnt1$$Register, $cnt2$$Register,
12602                         icnt2, $result$$Register,
12603                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12604     }
12605   %}
12606   ins_pipe( pipe_slow );
12607 %}
12608 
12609 // fast search of substring with known size.
12610 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12611                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12612 %{
12613   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12614   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12615   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12616 
12617   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12618   ins_encode %{
12619     int icnt2 = (int)$int_cnt2$$constant;
12620     if (icnt2 >= 8) {
12621       // IndexOf for constant substrings with size >= 8 elements
12622       // which don't need to be loaded through stack.
12623       __ string_indexofC8($str1$$Register, $str2$$Register,
12624                           $cnt1$$Register, $cnt2$$Register,
12625                           icnt2, $result$$Register,
12626                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12627     } else {
12628       // Small strings are loaded through stack if they cross page boundary.
12629       __ string_indexof($str1$$Register, $str2$$Register,
12630                         $cnt1$$Register, $cnt2$$Register,
12631                         icnt2, $result$$Register,
12632                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12633     }
12634   %}
12635   ins_pipe( pipe_slow );
12636 %}
12637 
12638 // fast search of substring with known size.
12639 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12640                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12641 %{
12642   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12643   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12644   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12645 
12646   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12647   ins_encode %{
12648     int icnt2 = (int)$int_cnt2$$constant;
12649     if (icnt2 >= 8) {
12650       // IndexOf for constant substrings with size >= 8 elements
12651       // which don't need to be loaded through stack.
12652       __ string_indexofC8($str1$$Register, $str2$$Register,
12653                           $cnt1$$Register, $cnt2$$Register,
12654                           icnt2, $result$$Register,
12655                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12656     } else {
12657       // Small strings are loaded through stack if they cross page boundary.
12658       __ string_indexof($str1$$Register, $str2$$Register,
12659                         $cnt1$$Register, $cnt2$$Register,
12660                         icnt2, $result$$Register,
12661                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12662     }
12663   %}
12664   ins_pipe( pipe_slow );
12665 %}
12666 
12667 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12668                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12669 %{
12670   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12671   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12672   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12673 
12674   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12675   ins_encode %{
12676     __ string_indexof($str1$$Register, $str2$$Register,
12677                       $cnt1$$Register, $cnt2$$Register,
12678                       (-1), $result$$Register,
12679                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12680   %}
12681   ins_pipe( pipe_slow );
12682 %}
12683 
12684 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12685                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12686 %{
12687   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12688   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12689   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12690 
12691   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12692   ins_encode %{
12693     __ string_indexof($str1$$Register, $str2$$Register,
12694                       $cnt1$$Register, $cnt2$$Register,
12695                       (-1), $result$$Register,
12696                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12697   %}
12698   ins_pipe( pipe_slow );
12699 %}
12700 
12701 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12702                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12703 %{
12704   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12705   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12706   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12707 
12708   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12709   ins_encode %{
12710     __ string_indexof($str1$$Register, $str2$$Register,
12711                       $cnt1$$Register, $cnt2$$Register,
12712                       (-1), $result$$Register,
12713                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12714   %}
12715   ins_pipe( pipe_slow );
12716 %}
12717 
12718 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12719                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12720 %{
12721   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12722   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12723   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12724   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12725   ins_encode %{
12726     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12727                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12728   %}
12729   ins_pipe( pipe_slow );
12730 %}
12731 
12732 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12733                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12734 %{
12735   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12736   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12737   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12738   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12739   ins_encode %{
12740     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12741                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12742   %}
12743   ins_pipe( pipe_slow );
12744 %}
12745 
12746 // fast string equals
12747 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12748                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12749 %{
12750   predicate(!VM_Version::supports_avx512vlbw());
12751   match(Set result (StrEquals (Binary str1 str2) cnt));
12752   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12753 
12754   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12755   ins_encode %{
12756     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12757                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12758                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12759   %}
12760   ins_pipe( pipe_slow );
12761 %}
12762 
12763 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12764                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12765 %{
12766   predicate(VM_Version::supports_avx512vlbw());
12767   match(Set result (StrEquals (Binary str1 str2) cnt));
12768   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12769 
12770   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12771   ins_encode %{
12772     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12773                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12774                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12775   %}
12776   ins_pipe( pipe_slow );
12777 %}
12778 
12779 // fast array equals
12780 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12781                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12782 %{
12783   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12784   match(Set result (AryEq ary1 ary2));
12785   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12786 
12787   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12788   ins_encode %{
12789     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12790                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12791                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12792   %}
12793   ins_pipe( pipe_slow );
12794 %}
12795 
12796 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12797                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12798 %{
12799   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12800   match(Set result (AryEq ary1 ary2));
12801   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12802 
12803   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12804   ins_encode %{
12805     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12806                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12807                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12808   %}
12809   ins_pipe( pipe_slow );
12810 %}
12811 
12812 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12813                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12814 %{
12815   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12816   match(Set result (AryEq ary1 ary2));
12817   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12818 
12819   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12820   ins_encode %{
12821     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12822                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12823                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12824   %}
12825   ins_pipe( pipe_slow );
12826 %}
12827 
12828 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12829                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12830 %{
12831   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12832   match(Set result (AryEq ary1 ary2));
12833   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12834 
12835   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12836   ins_encode %{
12837     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12838                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12839                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12840   %}
12841   ins_pipe( pipe_slow );
12842 %}
12843 
12844 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12845                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12846                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12847                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12848                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12849 %{
12850   predicate(UseAVX >= 2);
12851   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12852   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12853          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12854          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12855          USE basic_type, KILL cr);
12856 
12857   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12858   ins_encode %{
12859     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12860                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12861                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12862                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12863                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12864                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12865                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12866   %}
12867   ins_pipe( pipe_slow );
12868 %}
12869 
12870 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12871                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12872 %{
12873   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12874   match(Set result (CountPositives ary1 len));
12875   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12876 
12877   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12878   ins_encode %{
12879     __ count_positives($ary1$$Register, $len$$Register,
12880                        $result$$Register, $tmp3$$Register,
12881                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12882   %}
12883   ins_pipe( pipe_slow );
12884 %}
12885 
12886 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12887                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12888 %{
12889   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12890   match(Set result (CountPositives ary1 len));
12891   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12892 
12893   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12894   ins_encode %{
12895     __ count_positives($ary1$$Register, $len$$Register,
12896                        $result$$Register, $tmp3$$Register,
12897                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12898   %}
12899   ins_pipe( pipe_slow );
12900 %}
12901 
12902 // fast char[] to byte[] compression
12903 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12904                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12905   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12906   match(Set result (StrCompressedCopy src (Binary dst len)));
12907   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12908          USE_KILL len, KILL tmp5, KILL cr);
12909 
12910   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12911   ins_encode %{
12912     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12913                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12914                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12915                            knoreg, knoreg);
12916   %}
12917   ins_pipe( pipe_slow );
12918 %}
12919 
12920 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12921                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12922   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12923   match(Set result (StrCompressedCopy src (Binary dst len)));
12924   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12925          USE_KILL len, KILL tmp5, KILL cr);
12926 
12927   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12928   ins_encode %{
12929     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12930                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12931                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12932                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12933   %}
12934   ins_pipe( pipe_slow );
12935 %}
12936 // fast byte[] to char[] inflation
12937 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12938                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12939   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12940   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12941   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12942 
12943   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12944   ins_encode %{
12945     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12946                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12947   %}
12948   ins_pipe( pipe_slow );
12949 %}
12950 
12951 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12952                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12953   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12954   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12955   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12956 
12957   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12958   ins_encode %{
12959     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12960                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12961   %}
12962   ins_pipe( pipe_slow );
12963 %}
12964 
12965 // encode char[] to byte[] in ISO_8859_1
12966 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12967                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12968                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12969   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12970   match(Set result (EncodeISOArray src (Binary dst len)));
12971   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12972 
12973   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12974   ins_encode %{
12975     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12976                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12977                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12978   %}
12979   ins_pipe( pipe_slow );
12980 %}
12981 
12982 // encode char[] to byte[] in ASCII
12983 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12984                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12985                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12986   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12987   match(Set result (EncodeISOArray src (Binary dst len)));
12988   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12989 
12990   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12991   ins_encode %{
12992     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12993                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12994                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12995   %}
12996   ins_pipe( pipe_slow );
12997 %}
12998 
12999 //----------Overflow Math Instructions-----------------------------------------
13000 
13001 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13002 %{
13003   match(Set cr (OverflowAddI op1 op2));
13004   effect(DEF cr, USE_KILL op1, USE op2);
13005 
13006   format %{ "addl    $op1, $op2\t# overflow check int" %}
13007 
13008   ins_encode %{
13009     __ addl($op1$$Register, $op2$$Register);
13010   %}
13011   ins_pipe(ialu_reg_reg);
13012 %}
13013 
13014 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
13015 %{
13016   match(Set cr (OverflowAddI op1 op2));
13017   effect(DEF cr, USE_KILL op1, USE op2);
13018 
13019   format %{ "addl    $op1, $op2\t# overflow check int" %}
13020 
13021   ins_encode %{
13022     __ addl($op1$$Register, $op2$$constant);
13023   %}
13024   ins_pipe(ialu_reg_reg);
13025 %}
13026 
13027 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13028 %{
13029   match(Set cr (OverflowAddL op1 op2));
13030   effect(DEF cr, USE_KILL op1, USE op2);
13031 
13032   format %{ "addq    $op1, $op2\t# overflow check long" %}
13033   ins_encode %{
13034     __ addq($op1$$Register, $op2$$Register);
13035   %}
13036   ins_pipe(ialu_reg_reg);
13037 %}
13038 
13039 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
13040 %{
13041   match(Set cr (OverflowAddL op1 op2));
13042   effect(DEF cr, USE_KILL op1, USE op2);
13043 
13044   format %{ "addq    $op1, $op2\t# overflow check long" %}
13045   ins_encode %{
13046     __ addq($op1$$Register, $op2$$constant);
13047   %}
13048   ins_pipe(ialu_reg_reg);
13049 %}
13050 
13051 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13052 %{
13053   match(Set cr (OverflowSubI op1 op2));
13054 
13055   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13056   ins_encode %{
13057     __ cmpl($op1$$Register, $op2$$Register);
13058   %}
13059   ins_pipe(ialu_reg_reg);
13060 %}
13061 
13062 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13063 %{
13064   match(Set cr (OverflowSubI op1 op2));
13065 
13066   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13067   ins_encode %{
13068     __ cmpl($op1$$Register, $op2$$constant);
13069   %}
13070   ins_pipe(ialu_reg_reg);
13071 %}
13072 
13073 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13074 %{
13075   match(Set cr (OverflowSubL op1 op2));
13076 
13077   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13078   ins_encode %{
13079     __ cmpq($op1$$Register, $op2$$Register);
13080   %}
13081   ins_pipe(ialu_reg_reg);
13082 %}
13083 
13084 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13085 %{
13086   match(Set cr (OverflowSubL op1 op2));
13087 
13088   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13089   ins_encode %{
13090     __ cmpq($op1$$Register, $op2$$constant);
13091   %}
13092   ins_pipe(ialu_reg_reg);
13093 %}
13094 
13095 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13096 %{
13097   match(Set cr (OverflowSubI zero op2));
13098   effect(DEF cr, USE_KILL op2);
13099 
13100   format %{ "negl    $op2\t# overflow check int" %}
13101   ins_encode %{
13102     __ negl($op2$$Register);
13103   %}
13104   ins_pipe(ialu_reg_reg);
13105 %}
13106 
13107 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13108 %{
13109   match(Set cr (OverflowSubL zero op2));
13110   effect(DEF cr, USE_KILL op2);
13111 
13112   format %{ "negq    $op2\t# overflow check long" %}
13113   ins_encode %{
13114     __ negq($op2$$Register);
13115   %}
13116   ins_pipe(ialu_reg_reg);
13117 %}
13118 
13119 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13120 %{
13121   match(Set cr (OverflowMulI op1 op2));
13122   effect(DEF cr, USE_KILL op1, USE op2);
13123 
13124   format %{ "imull    $op1, $op2\t# overflow check int" %}
13125   ins_encode %{
13126     __ imull($op1$$Register, $op2$$Register);
13127   %}
13128   ins_pipe(ialu_reg_reg_alu0);
13129 %}
13130 
13131 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13132 %{
13133   match(Set cr (OverflowMulI op1 op2));
13134   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13135 
13136   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13137   ins_encode %{
13138     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13139   %}
13140   ins_pipe(ialu_reg_reg_alu0);
13141 %}
13142 
13143 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13144 %{
13145   match(Set cr (OverflowMulL op1 op2));
13146   effect(DEF cr, USE_KILL op1, USE op2);
13147 
13148   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13149   ins_encode %{
13150     __ imulq($op1$$Register, $op2$$Register);
13151   %}
13152   ins_pipe(ialu_reg_reg_alu0);
13153 %}
13154 
13155 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13156 %{
13157   match(Set cr (OverflowMulL op1 op2));
13158   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13159 
13160   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13161   ins_encode %{
13162     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13163   %}
13164   ins_pipe(ialu_reg_reg_alu0);
13165 %}
13166 
13167 
13168 //----------Control Flow Instructions------------------------------------------
13169 // Signed compare Instructions
13170 
13171 // XXX more variants!!
13172 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13173 %{
13174   match(Set cr (CmpI op1 op2));
13175   effect(DEF cr, USE op1, USE op2);
13176 
13177   format %{ "cmpl    $op1, $op2" %}
13178   ins_encode %{
13179     __ cmpl($op1$$Register, $op2$$Register);
13180   %}
13181   ins_pipe(ialu_cr_reg_reg);
13182 %}
13183 
13184 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13185 %{
13186   match(Set cr (CmpI op1 op2));
13187 
13188   format %{ "cmpl    $op1, $op2" %}
13189   ins_encode %{
13190     __ cmpl($op1$$Register, $op2$$constant);
13191   %}
13192   ins_pipe(ialu_cr_reg_imm);
13193 %}
13194 
13195 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13196 %{
13197   match(Set cr (CmpI op1 (LoadI op2)));
13198 
13199   ins_cost(500); // XXX
13200   format %{ "cmpl    $op1, $op2" %}
13201   ins_encode %{
13202     __ cmpl($op1$$Register, $op2$$Address);
13203   %}
13204   ins_pipe(ialu_cr_reg_mem);
13205 %}
13206 
13207 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13208 %{
13209   match(Set cr (CmpI src zero));
13210 
13211   format %{ "testl   $src, $src" %}
13212   ins_encode %{
13213     __ testl($src$$Register, $src$$Register);
13214   %}
13215   ins_pipe(ialu_cr_reg_imm);
13216 %}
13217 
13218 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13219 %{
13220   match(Set cr (CmpI (AndI src con) zero));
13221 
13222   format %{ "testl   $src, $con" %}
13223   ins_encode %{
13224     __ testl($src$$Register, $con$$constant);
13225   %}
13226   ins_pipe(ialu_cr_reg_imm);
13227 %}
13228 
13229 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13230 %{
13231   match(Set cr (CmpI (AndI src1 src2) zero));
13232 
13233   format %{ "testl   $src1, $src2" %}
13234   ins_encode %{
13235     __ testl($src1$$Register, $src2$$Register);
13236   %}
13237   ins_pipe(ialu_cr_reg_imm);
13238 %}
13239 
13240 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13241 %{
13242   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13243 
13244   format %{ "testl   $src, $mem" %}
13245   ins_encode %{
13246     __ testl($src$$Register, $mem$$Address);
13247   %}
13248   ins_pipe(ialu_cr_reg_mem);
13249 %}
13250 
13251 // Unsigned compare Instructions; really, same as signed except they
13252 // produce an rFlagsRegU instead of rFlagsReg.
13253 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13254 %{
13255   match(Set cr (CmpU op1 op2));
13256 
13257   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13258   ins_encode %{
13259     __ cmpl($op1$$Register, $op2$$Register);
13260   %}
13261   ins_pipe(ialu_cr_reg_reg);
13262 %}
13263 
13264 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13265 %{
13266   match(Set cr (CmpU op1 op2));
13267 
13268   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13269   ins_encode %{
13270     __ cmpl($op1$$Register, $op2$$constant);
13271   %}
13272   ins_pipe(ialu_cr_reg_imm);
13273 %}
13274 
13275 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13276 %{
13277   match(Set cr (CmpU op1 (LoadI op2)));
13278 
13279   ins_cost(500); // XXX
13280   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13281   ins_encode %{
13282     __ cmpl($op1$$Register, $op2$$Address);
13283   %}
13284   ins_pipe(ialu_cr_reg_mem);
13285 %}
13286 
13287 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13288 %{
13289   match(Set cr (CmpU src zero));
13290 
13291   format %{ "testl   $src, $src\t# unsigned" %}
13292   ins_encode %{
13293     __ testl($src$$Register, $src$$Register);
13294   %}
13295   ins_pipe(ialu_cr_reg_imm);
13296 %}
13297 
13298 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13299 %{
13300   match(Set cr (CmpP op1 op2));
13301 
13302   format %{ "cmpq    $op1, $op2\t# ptr" %}
13303   ins_encode %{
13304     __ cmpq($op1$$Register, $op2$$Register);
13305   %}
13306   ins_pipe(ialu_cr_reg_reg);
13307 %}
13308 
13309 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13310 %{
13311   match(Set cr (CmpP op1 (LoadP op2)));
13312   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13313 
13314   ins_cost(500); // XXX
13315   format %{ "cmpq    $op1, $op2\t# ptr" %}
13316   ins_encode %{
13317     __ cmpq($op1$$Register, $op2$$Address);
13318   %}
13319   ins_pipe(ialu_cr_reg_mem);
13320 %}
13321 
13322 // XXX this is generalized by compP_rReg_mem???
13323 // Compare raw pointer (used in out-of-heap check).
13324 // Only works because non-oop pointers must be raw pointers
13325 // and raw pointers have no anti-dependencies.
13326 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13327 %{
13328   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13329             n->in(2)->as_Load()->barrier_data() == 0);
13330   match(Set cr (CmpP op1 (LoadP op2)));
13331 
13332   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13333   ins_encode %{
13334     __ cmpq($op1$$Register, $op2$$Address);
13335   %}
13336   ins_pipe(ialu_cr_reg_mem);
13337 %}
13338 
13339 // This will generate a signed flags result. This should be OK since
13340 // any compare to a zero should be eq/neq.
13341 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13342 %{
13343   match(Set cr (CmpP src zero));
13344 
13345   format %{ "testq   $src, $src\t# ptr" %}
13346   ins_encode %{
13347     __ testq($src$$Register, $src$$Register);
13348   %}
13349   ins_pipe(ialu_cr_reg_imm);
13350 %}
13351 
13352 // This will generate a signed flags result. This should be OK since
13353 // any compare to a zero should be eq/neq.
13354 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13355 %{
13356   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13357             n->in(1)->as_Load()->barrier_data() == 0);
13358   match(Set cr (CmpP (LoadP op) zero));
13359 
13360   ins_cost(500); // XXX
13361   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13362   ins_encode %{
13363     __ testq($op$$Address, 0xFFFFFFFF);
13364   %}
13365   ins_pipe(ialu_cr_reg_imm);
13366 %}
13367 
13368 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13369 %{
13370   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13371             n->in(1)->as_Load()->barrier_data() == 0);
13372   match(Set cr (CmpP (LoadP mem) zero));
13373 
13374   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13375   ins_encode %{
13376     __ cmpq(r12, $mem$$Address);
13377   %}
13378   ins_pipe(ialu_cr_reg_mem);
13379 %}
13380 
13381 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13382 %{
13383   match(Set cr (CmpN op1 op2));
13384 
13385   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13386   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13387   ins_pipe(ialu_cr_reg_reg);
13388 %}
13389 
13390 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13391 %{
13392   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13393   match(Set cr (CmpN src (LoadN mem)));
13394 
13395   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13396   ins_encode %{
13397     __ cmpl($src$$Register, $mem$$Address);
13398   %}
13399   ins_pipe(ialu_cr_reg_mem);
13400 %}
13401 
13402 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13403   match(Set cr (CmpN op1 op2));
13404 
13405   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13406   ins_encode %{
13407     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13408   %}
13409   ins_pipe(ialu_cr_reg_imm);
13410 %}
13411 
13412 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13413 %{
13414   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13415   match(Set cr (CmpN src (LoadN mem)));
13416 
13417   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13418   ins_encode %{
13419     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13420   %}
13421   ins_pipe(ialu_cr_reg_mem);
13422 %}
13423 
13424 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13425   match(Set cr (CmpN op1 op2));
13426 
13427   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13428   ins_encode %{
13429     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13430   %}
13431   ins_pipe(ialu_cr_reg_imm);
13432 %}
13433 
13434 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13435 %{
13436   predicate(!UseCompactObjectHeaders);
13437   match(Set cr (CmpN src (LoadNKlass mem)));
13438 
13439   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13440   ins_encode %{
13441     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13442   %}
13443   ins_pipe(ialu_cr_reg_mem);
13444 %}
13445 
13446 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13447   match(Set cr (CmpN src zero));
13448 
13449   format %{ "testl   $src, $src\t# compressed ptr" %}
13450   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13451   ins_pipe(ialu_cr_reg_imm);
13452 %}
13453 
13454 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13455 %{
13456   predicate(CompressedOops::base() != nullptr &&
13457             n->in(1)->as_Load()->barrier_data() == 0);
13458   match(Set cr (CmpN (LoadN mem) zero));
13459 
13460   ins_cost(500); // XXX
13461   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13462   ins_encode %{
13463     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13464   %}
13465   ins_pipe(ialu_cr_reg_mem);
13466 %}
13467 
13468 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13469 %{
13470   predicate(CompressedOops::base() == nullptr &&
13471             n->in(1)->as_Load()->barrier_data() == 0);
13472   match(Set cr (CmpN (LoadN mem) zero));
13473 
13474   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13475   ins_encode %{
13476     __ cmpl(r12, $mem$$Address);
13477   %}
13478   ins_pipe(ialu_cr_reg_mem);
13479 %}
13480 
13481 // Yanked all unsigned pointer compare operations.
13482 // Pointer compares are done with CmpP which is already unsigned.
13483 
13484 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13485 %{
13486   match(Set cr (CmpL op1 op2));
13487 
13488   format %{ "cmpq    $op1, $op2" %}
13489   ins_encode %{
13490     __ cmpq($op1$$Register, $op2$$Register);
13491   %}
13492   ins_pipe(ialu_cr_reg_reg);
13493 %}
13494 
13495 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13496 %{
13497   match(Set cr (CmpL op1 op2));
13498 
13499   format %{ "cmpq    $op1, $op2" %}
13500   ins_encode %{
13501     __ cmpq($op1$$Register, $op2$$constant);
13502   %}
13503   ins_pipe(ialu_cr_reg_imm);
13504 %}
13505 
13506 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13507 %{
13508   match(Set cr (CmpL op1 (LoadL op2)));
13509 
13510   format %{ "cmpq    $op1, $op2" %}
13511   ins_encode %{
13512     __ cmpq($op1$$Register, $op2$$Address);
13513   %}
13514   ins_pipe(ialu_cr_reg_mem);
13515 %}
13516 
13517 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13518 %{
13519   match(Set cr (CmpL src zero));
13520 
13521   format %{ "testq   $src, $src" %}
13522   ins_encode %{
13523     __ testq($src$$Register, $src$$Register);
13524   %}
13525   ins_pipe(ialu_cr_reg_imm);
13526 %}
13527 
13528 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13529 %{
13530   match(Set cr (CmpL (AndL src con) zero));
13531 
13532   format %{ "testq   $src, $con\t# long" %}
13533   ins_encode %{
13534     __ testq($src$$Register, $con$$constant);
13535   %}
13536   ins_pipe(ialu_cr_reg_imm);
13537 %}
13538 
13539 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13540 %{
13541   match(Set cr (CmpL (AndL src1 src2) zero));
13542 
13543   format %{ "testq   $src1, $src2\t# long" %}
13544   ins_encode %{
13545     __ testq($src1$$Register, $src2$$Register);
13546   %}
13547   ins_pipe(ialu_cr_reg_imm);
13548 %}
13549 
13550 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13551 %{
13552   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13553 
13554   format %{ "testq   $src, $mem" %}
13555   ins_encode %{
13556     __ testq($src$$Register, $mem$$Address);
13557   %}
13558   ins_pipe(ialu_cr_reg_mem);
13559 %}
13560 
13561 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13562 %{
13563   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13564 
13565   format %{ "testq   $src, $mem" %}
13566   ins_encode %{
13567     __ testq($src$$Register, $mem$$Address);
13568   %}
13569   ins_pipe(ialu_cr_reg_mem);
13570 %}
13571 
13572 // Manifest a CmpU result in an integer register.  Very painful.
13573 // This is the test to avoid.
13574 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13575 %{
13576   match(Set dst (CmpU3 src1 src2));
13577   effect(KILL flags);
13578 
13579   ins_cost(275); // XXX
13580   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13581             "movl    $dst, -1\n\t"
13582             "jb,u    done\n\t"
13583             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13584     "done:" %}
13585   ins_encode %{
13586     Label done;
13587     __ cmpl($src1$$Register, $src2$$Register);
13588     __ movl($dst$$Register, -1);
13589     __ jccb(Assembler::below, done);
13590     __ setcc(Assembler::notZero, $dst$$Register);
13591     __ bind(done);
13592   %}
13593   ins_pipe(pipe_slow);
13594 %}
13595 
13596 // Manifest a CmpL result in an integer register.  Very painful.
13597 // This is the test to avoid.
13598 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13599 %{
13600   match(Set dst (CmpL3 src1 src2));
13601   effect(KILL flags);
13602 
13603   ins_cost(275); // XXX
13604   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13605             "movl    $dst, -1\n\t"
13606             "jl,s    done\n\t"
13607             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13608     "done:" %}
13609   ins_encode %{
13610     Label done;
13611     __ cmpq($src1$$Register, $src2$$Register);
13612     __ movl($dst$$Register, -1);
13613     __ jccb(Assembler::less, done);
13614     __ setcc(Assembler::notZero, $dst$$Register);
13615     __ bind(done);
13616   %}
13617   ins_pipe(pipe_slow);
13618 %}
13619 
13620 // Manifest a CmpUL result in an integer register.  Very painful.
13621 // This is the test to avoid.
13622 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13623 %{
13624   match(Set dst (CmpUL3 src1 src2));
13625   effect(KILL flags);
13626 
13627   ins_cost(275); // XXX
13628   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13629             "movl    $dst, -1\n\t"
13630             "jb,u    done\n\t"
13631             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13632     "done:" %}
13633   ins_encode %{
13634     Label done;
13635     __ cmpq($src1$$Register, $src2$$Register);
13636     __ movl($dst$$Register, -1);
13637     __ jccb(Assembler::below, done);
13638     __ setcc(Assembler::notZero, $dst$$Register);
13639     __ bind(done);
13640   %}
13641   ins_pipe(pipe_slow);
13642 %}
13643 
13644 // Unsigned long compare Instructions; really, same as signed long except they
13645 // produce an rFlagsRegU instead of rFlagsReg.
13646 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13647 %{
13648   match(Set cr (CmpUL op1 op2));
13649 
13650   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13651   ins_encode %{
13652     __ cmpq($op1$$Register, $op2$$Register);
13653   %}
13654   ins_pipe(ialu_cr_reg_reg);
13655 %}
13656 
13657 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13658 %{
13659   match(Set cr (CmpUL op1 op2));
13660 
13661   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13662   ins_encode %{
13663     __ cmpq($op1$$Register, $op2$$constant);
13664   %}
13665   ins_pipe(ialu_cr_reg_imm);
13666 %}
13667 
13668 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13669 %{
13670   match(Set cr (CmpUL op1 (LoadL op2)));
13671 
13672   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13673   ins_encode %{
13674     __ cmpq($op1$$Register, $op2$$Address);
13675   %}
13676   ins_pipe(ialu_cr_reg_mem);
13677 %}
13678 
13679 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13680 %{
13681   match(Set cr (CmpUL src zero));
13682 
13683   format %{ "testq   $src, $src\t# unsigned" %}
13684   ins_encode %{
13685     __ testq($src$$Register, $src$$Register);
13686   %}
13687   ins_pipe(ialu_cr_reg_imm);
13688 %}
13689 
13690 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13691 %{
13692   match(Set cr (CmpI (LoadB mem) imm));
13693 
13694   ins_cost(125);
13695   format %{ "cmpb    $mem, $imm" %}
13696   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13697   ins_pipe(ialu_cr_reg_mem);
13698 %}
13699 
13700 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13701 %{
13702   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13703 
13704   ins_cost(125);
13705   format %{ "testb   $mem, $imm\t# ubyte" %}
13706   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13707   ins_pipe(ialu_cr_reg_mem);
13708 %}
13709 
13710 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13711 %{
13712   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13713 
13714   ins_cost(125);
13715   format %{ "testb   $mem, $imm\t# byte" %}
13716   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13717   ins_pipe(ialu_cr_reg_mem);
13718 %}
13719 
13720 //----------Max and Min--------------------------------------------------------
13721 // Min Instructions
13722 
13723 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13724 %{
13725   predicate(!UseAPX);
13726   effect(USE_DEF dst, USE src, USE cr);
13727 
13728   format %{ "cmovlgt $dst, $src\t# min" %}
13729   ins_encode %{
13730     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13731   %}
13732   ins_pipe(pipe_cmov_reg);
13733 %}
13734 
13735 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13736 %{
13737   predicate(UseAPX);
13738   effect(DEF dst, USE src1, USE src2, USE cr);
13739 
13740   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13741   ins_encode %{
13742     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13743   %}
13744   ins_pipe(pipe_cmov_reg);
13745 %}
13746 
13747 instruct minI_rReg(rRegI dst, rRegI src)
13748 %{
13749   predicate(!UseAPX);
13750   match(Set dst (MinI dst src));
13751 
13752   ins_cost(200);
13753   expand %{
13754     rFlagsReg cr;
13755     compI_rReg(cr, dst, src);
13756     cmovI_reg_g(dst, src, cr);
13757   %}
13758 %}
13759 
13760 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13761 %{
13762   predicate(UseAPX);
13763   match(Set dst (MinI src1 src2));
13764   effect(DEF dst, USE src1, USE src2);
13765 
13766   ins_cost(200);
13767   expand %{
13768     rFlagsReg cr;
13769     compI_rReg(cr, src1, src2);
13770     cmovI_reg_g_ndd(dst, src1, src2, cr);
13771   %}
13772 %}
13773 
13774 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13775 %{
13776   predicate(!UseAPX);
13777   effect(USE_DEF dst, USE src, USE cr);
13778 
13779   format %{ "cmovllt $dst, $src\t# max" %}
13780   ins_encode %{
13781     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13782   %}
13783   ins_pipe(pipe_cmov_reg);
13784 %}
13785 
13786 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13787 %{
13788   predicate(UseAPX);
13789   effect(DEF dst, USE src1, USE src2, USE cr);
13790 
13791   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13792   ins_encode %{
13793     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13794   %}
13795   ins_pipe(pipe_cmov_reg);
13796 %}
13797 
13798 instruct maxI_rReg(rRegI dst, rRegI src)
13799 %{
13800   predicate(!UseAPX);
13801   match(Set dst (MaxI dst src));
13802 
13803   ins_cost(200);
13804   expand %{
13805     rFlagsReg cr;
13806     compI_rReg(cr, dst, src);
13807     cmovI_reg_l(dst, src, cr);
13808   %}
13809 %}
13810 
13811 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13812 %{
13813   predicate(UseAPX);
13814   match(Set dst (MaxI src1 src2));
13815   effect(DEF dst, USE src1, USE src2);
13816 
13817   ins_cost(200);
13818   expand %{
13819     rFlagsReg cr;
13820     compI_rReg(cr, src1, src2);
13821     cmovI_reg_l_ndd(dst, src1, src2, cr);
13822   %}
13823 %}
13824 
13825 // ============================================================================
13826 // Branch Instructions
13827 
13828 // Jump Direct - Label defines a relative address from JMP+1
13829 instruct jmpDir(label labl)
13830 %{
13831   match(Goto);
13832   effect(USE labl);
13833 
13834   ins_cost(300);
13835   format %{ "jmp     $labl" %}
13836   size(5);
13837   ins_encode %{
13838     Label* L = $labl$$label;
13839     __ jmp(*L, false); // Always long jump
13840   %}
13841   ins_pipe(pipe_jmp);
13842 %}
13843 
13844 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13845 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13846 %{
13847   match(If cop cr);
13848   effect(USE labl);
13849 
13850   ins_cost(300);
13851   format %{ "j$cop     $labl" %}
13852   size(6);
13853   ins_encode %{
13854     Label* L = $labl$$label;
13855     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13856   %}
13857   ins_pipe(pipe_jcc);
13858 %}
13859 
13860 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13861 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13862 %{
13863   match(CountedLoopEnd cop cr);
13864   effect(USE labl);
13865 
13866   ins_cost(300);
13867   format %{ "j$cop     $labl\t# loop end" %}
13868   size(6);
13869   ins_encode %{
13870     Label* L = $labl$$label;
13871     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13872   %}
13873   ins_pipe(pipe_jcc);
13874 %}
13875 
13876 // Jump Direct Conditional - using unsigned comparison
13877 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13878   match(If cop cmp);
13879   effect(USE labl);
13880 
13881   ins_cost(300);
13882   format %{ "j$cop,u   $labl" %}
13883   size(6);
13884   ins_encode %{
13885     Label* L = $labl$$label;
13886     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13887   %}
13888   ins_pipe(pipe_jcc);
13889 %}
13890 
13891 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13892   match(If cop cmp);
13893   effect(USE labl);
13894 
13895   ins_cost(200);
13896   format %{ "j$cop,u   $labl" %}
13897   size(6);
13898   ins_encode %{
13899     Label* L = $labl$$label;
13900     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13901   %}
13902   ins_pipe(pipe_jcc);
13903 %}
13904 
13905 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13906   match(If cop cmp);
13907   effect(USE labl);
13908 
13909   ins_cost(200);
13910   format %{ $$template
13911     if ($cop$$cmpcode == Assembler::notEqual) {
13912       $$emit$$"jp,u    $labl\n\t"
13913       $$emit$$"j$cop,u   $labl"
13914     } else {
13915       $$emit$$"jp,u    done\n\t"
13916       $$emit$$"j$cop,u   $labl\n\t"
13917       $$emit$$"done:"
13918     }
13919   %}
13920   ins_encode %{
13921     Label* l = $labl$$label;
13922     if ($cop$$cmpcode == Assembler::notEqual) {
13923       __ jcc(Assembler::parity, *l, false);
13924       __ jcc(Assembler::notEqual, *l, false);
13925     } else if ($cop$$cmpcode == Assembler::equal) {
13926       Label done;
13927       __ jccb(Assembler::parity, done);
13928       __ jcc(Assembler::equal, *l, false);
13929       __ bind(done);
13930     } else {
13931        ShouldNotReachHere();
13932     }
13933   %}
13934   ins_pipe(pipe_jcc);
13935 %}
13936 
13937 // ============================================================================
13938 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13939 // superklass array for an instance of the superklass.  Set a hidden
13940 // internal cache on a hit (cache is checked with exposed code in
13941 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13942 // encoding ALSO sets flags.
13943 
13944 instruct partialSubtypeCheck(rdi_RegP result,
13945                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13946                              rFlagsReg cr)
13947 %{
13948   match(Set result (PartialSubtypeCheck sub super));
13949   predicate(!UseSecondarySupersTable);
13950   effect(KILL rcx, KILL cr);
13951 
13952   ins_cost(1100);  // slightly larger than the next version
13953   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13954             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13955             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13956             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13957             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13958             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13959             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13960     "miss:\t" %}
13961 
13962   ins_encode %{
13963     Label miss;
13964     // NB: Callers may assume that, when $result is a valid register,
13965     // check_klass_subtype_slow_path_linear sets it to a nonzero
13966     // value.
13967     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13968                                             $rcx$$Register, $result$$Register,
13969                                             nullptr, &miss,
13970                                             /*set_cond_codes:*/ true);
13971     __ xorptr($result$$Register, $result$$Register);
13972     __ bind(miss);
13973   %}
13974 
13975   ins_pipe(pipe_slow);
13976 %}
13977 
13978 // ============================================================================
13979 // Two versions of hashtable-based partialSubtypeCheck, both used when
13980 // we need to search for a super class in the secondary supers array.
13981 // The first is used when we don't know _a priori_ the class being
13982 // searched for. The second, far more common, is used when we do know:
13983 // this is used for instanceof, checkcast, and any case where C2 can
13984 // determine it by constant propagation.
13985 
13986 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
13987                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13988                                        rFlagsReg cr)
13989 %{
13990   match(Set result (PartialSubtypeCheck sub super));
13991   predicate(UseSecondarySupersTable);
13992   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13993 
13994   ins_cost(1000);
13995   format %{ "partialSubtypeCheck $result, $sub, $super" %}
13996 
13997   ins_encode %{
13998     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
13999 					 $temp3$$Register, $temp4$$Register, $result$$Register);
14000   %}
14001 
14002   ins_pipe(pipe_slow);
14003 %}
14004 
14005 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
14006                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14007                                        rFlagsReg cr)
14008 %{
14009   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
14010   predicate(UseSecondarySupersTable);
14011   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14012 
14013   ins_cost(700);  // smaller than the next version
14014   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
14015 
14016   ins_encode %{
14017     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
14018     if (InlineSecondarySupersTest) {
14019       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
14020                                        $temp3$$Register, $temp4$$Register, $result$$Register,
14021                                        super_klass_slot);
14022     } else {
14023       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
14024     }
14025   %}
14026 
14027   ins_pipe(pipe_slow);
14028 %}
14029 
14030 // ============================================================================
14031 // Branch Instructions -- short offset versions
14032 //
14033 // These instructions are used to replace jumps of a long offset (the default
14034 // match) with jumps of a shorter offset.  These instructions are all tagged
14035 // with the ins_short_branch attribute, which causes the ADLC to suppress the
14036 // match rules in general matching.  Instead, the ADLC generates a conversion
14037 // method in the MachNode which can be used to do in-place replacement of the
14038 // long variant with the shorter variant.  The compiler will determine if a
14039 // branch can be taken by the is_short_branch_offset() predicate in the machine
14040 // specific code section of the file.
14041 
14042 // Jump Direct - Label defines a relative address from JMP+1
14043 instruct jmpDir_short(label labl) %{
14044   match(Goto);
14045   effect(USE labl);
14046 
14047   ins_cost(300);
14048   format %{ "jmp,s   $labl" %}
14049   size(2);
14050   ins_encode %{
14051     Label* L = $labl$$label;
14052     __ jmpb(*L);
14053   %}
14054   ins_pipe(pipe_jmp);
14055   ins_short_branch(1);
14056 %}
14057 
14058 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14059 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
14060   match(If cop cr);
14061   effect(USE labl);
14062 
14063   ins_cost(300);
14064   format %{ "j$cop,s   $labl" %}
14065   size(2);
14066   ins_encode %{
14067     Label* L = $labl$$label;
14068     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14069   %}
14070   ins_pipe(pipe_jcc);
14071   ins_short_branch(1);
14072 %}
14073 
14074 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14075 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
14076   match(CountedLoopEnd cop cr);
14077   effect(USE labl);
14078 
14079   ins_cost(300);
14080   format %{ "j$cop,s   $labl\t# loop end" %}
14081   size(2);
14082   ins_encode %{
14083     Label* L = $labl$$label;
14084     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14085   %}
14086   ins_pipe(pipe_jcc);
14087   ins_short_branch(1);
14088 %}
14089 
14090 // Jump Direct Conditional - using unsigned comparison
14091 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14092   match(If cop cmp);
14093   effect(USE labl);
14094 
14095   ins_cost(300);
14096   format %{ "j$cop,us  $labl" %}
14097   size(2);
14098   ins_encode %{
14099     Label* L = $labl$$label;
14100     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14101   %}
14102   ins_pipe(pipe_jcc);
14103   ins_short_branch(1);
14104 %}
14105 
14106 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14107   match(If cop cmp);
14108   effect(USE labl);
14109 
14110   ins_cost(300);
14111   format %{ "j$cop,us  $labl" %}
14112   size(2);
14113   ins_encode %{
14114     Label* L = $labl$$label;
14115     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14116   %}
14117   ins_pipe(pipe_jcc);
14118   ins_short_branch(1);
14119 %}
14120 
14121 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14122   match(If cop cmp);
14123   effect(USE labl);
14124 
14125   ins_cost(300);
14126   format %{ $$template
14127     if ($cop$$cmpcode == Assembler::notEqual) {
14128       $$emit$$"jp,u,s  $labl\n\t"
14129       $$emit$$"j$cop,u,s  $labl"
14130     } else {
14131       $$emit$$"jp,u,s  done\n\t"
14132       $$emit$$"j$cop,u,s  $labl\n\t"
14133       $$emit$$"done:"
14134     }
14135   %}
14136   size(4);
14137   ins_encode %{
14138     Label* l = $labl$$label;
14139     if ($cop$$cmpcode == Assembler::notEqual) {
14140       __ jccb(Assembler::parity, *l);
14141       __ jccb(Assembler::notEqual, *l);
14142     } else if ($cop$$cmpcode == Assembler::equal) {
14143       Label done;
14144       __ jccb(Assembler::parity, done);
14145       __ jccb(Assembler::equal, *l);
14146       __ bind(done);
14147     } else {
14148        ShouldNotReachHere();
14149     }
14150   %}
14151   ins_pipe(pipe_jcc);
14152   ins_short_branch(1);
14153 %}
14154 
14155 // ============================================================================
14156 // inlined locking and unlocking
14157 
14158 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
14159   predicate(LockingMode != LM_LIGHTWEIGHT);
14160   match(Set cr (FastLock object box));
14161   effect(TEMP tmp, TEMP scr, USE_KILL box);
14162   ins_cost(300);
14163   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
14164   ins_encode %{
14165     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
14166                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
14167   %}
14168   ins_pipe(pipe_slow);
14169 %}
14170 
14171 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
14172   predicate(LockingMode != LM_LIGHTWEIGHT);
14173   match(Set cr (FastUnlock object box));
14174   effect(TEMP tmp, USE_KILL box);
14175   ins_cost(300);
14176   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
14177   ins_encode %{
14178     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
14179   %}
14180   ins_pipe(pipe_slow);
14181 %}
14182 
14183 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14184   predicate(LockingMode == LM_LIGHTWEIGHT);
14185   match(Set cr (FastLock object box));
14186   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14187   ins_cost(300);
14188   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14189   ins_encode %{
14190     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14191   %}
14192   ins_pipe(pipe_slow);
14193 %}
14194 
14195 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14196   predicate(LockingMode == LM_LIGHTWEIGHT);
14197   match(Set cr (FastUnlock object rax_reg));
14198   effect(TEMP tmp, USE_KILL rax_reg);
14199   ins_cost(300);
14200   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14201   ins_encode %{
14202     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14203   %}
14204   ins_pipe(pipe_slow);
14205 %}
14206 
14207 
14208 // ============================================================================
14209 // Safepoint Instructions
14210 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14211 %{
14212   match(SafePoint poll);
14213   effect(KILL cr, USE poll);
14214 
14215   format %{ "testl   rax, [$poll]\t"
14216             "# Safepoint: poll for GC" %}
14217   ins_cost(125);
14218   ins_encode %{
14219     __ relocate(relocInfo::poll_type);
14220     address pre_pc = __ pc();
14221     __ testl(rax, Address($poll$$Register, 0));
14222     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14223   %}
14224   ins_pipe(ialu_reg_mem);
14225 %}
14226 
14227 instruct mask_all_evexL(kReg dst, rRegL src) %{
14228   match(Set dst (MaskAll src));
14229   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14230   ins_encode %{
14231     int mask_len = Matcher::vector_length(this);
14232     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14233   %}
14234   ins_pipe( pipe_slow );
14235 %}
14236 
14237 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14238   predicate(Matcher::vector_length(n) > 32);
14239   match(Set dst (MaskAll src));
14240   effect(TEMP tmp);
14241   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14242   ins_encode %{
14243     int mask_len = Matcher::vector_length(this);
14244     __ movslq($tmp$$Register, $src$$Register);
14245     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14246   %}
14247   ins_pipe( pipe_slow );
14248 %}
14249 
14250 // ============================================================================
14251 // Procedure Call/Return Instructions
14252 // Call Java Static Instruction
14253 // Note: If this code changes, the corresponding ret_addr_offset() and
14254 //       compute_padding() functions will have to be adjusted.
14255 instruct CallStaticJavaDirect(method meth) %{
14256   match(CallStaticJava);
14257   effect(USE meth);
14258 
14259   ins_cost(300);
14260   format %{ "call,static " %}
14261   opcode(0xE8); /* E8 cd */
14262   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14263   ins_pipe(pipe_slow);
14264   ins_alignment(4);
14265 %}
14266 
14267 // Call Java Dynamic Instruction
14268 // Note: If this code changes, the corresponding ret_addr_offset() and
14269 //       compute_padding() functions will have to be adjusted.
14270 instruct CallDynamicJavaDirect(method meth)
14271 %{
14272   match(CallDynamicJava);
14273   effect(USE meth);
14274 
14275   ins_cost(300);
14276   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14277             "call,dynamic " %}
14278   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14279   ins_pipe(pipe_slow);
14280   ins_alignment(4);
14281 %}
14282 
14283 // Call Runtime Instruction
14284 instruct CallRuntimeDirect(method meth)
14285 %{
14286   match(CallRuntime);
14287   effect(USE meth);
14288 
14289   ins_cost(300);
14290   format %{ "call,runtime " %}
14291   ins_encode(clear_avx, Java_To_Runtime(meth));
14292   ins_pipe(pipe_slow);
14293 %}
14294 
14295 // Call runtime without safepoint
14296 instruct CallLeafDirect(method meth)
14297 %{
14298   match(CallLeaf);
14299   effect(USE meth);
14300 
14301   ins_cost(300);
14302   format %{ "call_leaf,runtime " %}
14303   ins_encode(clear_avx, Java_To_Runtime(meth));
14304   ins_pipe(pipe_slow);
14305 %}
14306 
14307 // Call runtime without safepoint and with vector arguments
14308 instruct CallLeafDirectVector(method meth)
14309 %{
14310   match(CallLeafVector);
14311   effect(USE meth);
14312 
14313   ins_cost(300);
14314   format %{ "call_leaf,vector " %}
14315   ins_encode(Java_To_Runtime(meth));
14316   ins_pipe(pipe_slow);
14317 %}
14318 
14319 // Call runtime without safepoint
14320 instruct CallLeafNoFPDirect(method meth)
14321 %{
14322   match(CallLeafNoFP);
14323   effect(USE meth);
14324 
14325   ins_cost(300);
14326   format %{ "call_leaf_nofp,runtime " %}
14327   ins_encode(clear_avx, Java_To_Runtime(meth));
14328   ins_pipe(pipe_slow);
14329 %}
14330 
14331 // Return Instruction
14332 // Remove the return address & jump to it.
14333 // Notice: We always emit a nop after a ret to make sure there is room
14334 // for safepoint patching
14335 instruct Ret()
14336 %{
14337   match(Return);
14338 
14339   format %{ "ret" %}
14340   ins_encode %{
14341     __ ret(0);
14342   %}
14343   ins_pipe(pipe_jmp);
14344 %}
14345 
14346 // Tail Call; Jump from runtime stub to Java code.
14347 // Also known as an 'interprocedural jump'.
14348 // Target of jump will eventually return to caller.
14349 // TailJump below removes the return address.
14350 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14351 // emitted just above the TailCall which has reset rbp to the caller state.
14352 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14353 %{
14354   match(TailCall jump_target method_ptr);
14355 
14356   ins_cost(300);
14357   format %{ "jmp     $jump_target\t# rbx holds method" %}
14358   ins_encode %{
14359     __ jmp($jump_target$$Register);
14360   %}
14361   ins_pipe(pipe_jmp);
14362 %}
14363 
14364 // Tail Jump; remove the return address; jump to target.
14365 // TailCall above leaves the return address around.
14366 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14367 %{
14368   match(TailJump jump_target ex_oop);
14369 
14370   ins_cost(300);
14371   format %{ "popq    rdx\t# pop return address\n\t"
14372             "jmp     $jump_target" %}
14373   ins_encode %{
14374     __ popq(as_Register(RDX_enc));
14375     __ jmp($jump_target$$Register);
14376   %}
14377   ins_pipe(pipe_jmp);
14378 %}
14379 
14380 // Forward exception.
14381 instruct ForwardExceptionjmp()
14382 %{
14383   match(ForwardException);
14384 
14385   format %{ "jmp     forward_exception_stub" %}
14386   ins_encode %{
14387     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14388   %}
14389   ins_pipe(pipe_jmp);
14390 %}
14391 
14392 // Create exception oop: created by stack-crawling runtime code.
14393 // Created exception is now available to this handler, and is setup
14394 // just prior to jumping to this handler.  No code emitted.
14395 instruct CreateException(rax_RegP ex_oop)
14396 %{
14397   match(Set ex_oop (CreateEx));
14398 
14399   size(0);
14400   // use the following format syntax
14401   format %{ "# exception oop is in rax; no code emitted" %}
14402   ins_encode();
14403   ins_pipe(empty);
14404 %}
14405 
14406 // Rethrow exception:
14407 // The exception oop will come in the first argument position.
14408 // Then JUMP (not call) to the rethrow stub code.
14409 instruct RethrowException()
14410 %{
14411   match(Rethrow);
14412 
14413   // use the following format syntax
14414   format %{ "jmp     rethrow_stub" %}
14415   ins_encode %{
14416     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14417   %}
14418   ins_pipe(pipe_jmp);
14419 %}
14420 
14421 // ============================================================================
14422 // This name is KNOWN by the ADLC and cannot be changed.
14423 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14424 // for this guy.
14425 instruct tlsLoadP(r15_RegP dst) %{
14426   match(Set dst (ThreadLocal));
14427   effect(DEF dst);
14428 
14429   size(0);
14430   format %{ "# TLS is in R15" %}
14431   ins_encode( /*empty encoding*/ );
14432   ins_pipe(ialu_reg_reg);
14433 %}
14434 
14435 
14436 //----------PEEPHOLE RULES-----------------------------------------------------
14437 // These must follow all instruction definitions as they use the names
14438 // defined in the instructions definitions.
14439 //
14440 // peeppredicate ( rule_predicate );
14441 // // the predicate unless which the peephole rule will be ignored
14442 //
14443 // peepmatch ( root_instr_name [preceding_instruction]* );
14444 //
14445 // peepprocedure ( procedure_name );
14446 // // provide a procedure name to perform the optimization, the procedure should
14447 // // reside in the architecture dependent peephole file, the method has the
14448 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14449 // // with the arguments being the basic block, the current node index inside the
14450 // // block, the register allocator, the functions upon invoked return a new node
14451 // // defined in peepreplace, and the rules of the nodes appearing in the
14452 // // corresponding peepmatch, the function return true if successful, else
14453 // // return false
14454 //
14455 // peepconstraint %{
14456 // (instruction_number.operand_name relational_op instruction_number.operand_name
14457 //  [, ...] );
14458 // // instruction numbers are zero-based using left to right order in peepmatch
14459 //
14460 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14461 // // provide an instruction_number.operand_name for each operand that appears
14462 // // in the replacement instruction's match rule
14463 //
14464 // ---------VM FLAGS---------------------------------------------------------
14465 //
14466 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14467 //
14468 // Each peephole rule is given an identifying number starting with zero and
14469 // increasing by one in the order seen by the parser.  An individual peephole
14470 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14471 // on the command-line.
14472 //
14473 // ---------CURRENT LIMITATIONS----------------------------------------------
14474 //
14475 // Only transformations inside a basic block (do we need more for peephole)
14476 //
14477 // ---------EXAMPLE----------------------------------------------------------
14478 //
14479 // // pertinent parts of existing instructions in architecture description
14480 // instruct movI(rRegI dst, rRegI src)
14481 // %{
14482 //   match(Set dst (CopyI src));
14483 // %}
14484 //
14485 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14486 // %{
14487 //   match(Set dst (AddI dst src));
14488 //   effect(KILL cr);
14489 // %}
14490 //
14491 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14492 // %{
14493 //   match(Set dst (AddI dst src));
14494 // %}
14495 //
14496 // 1. Simple replacement
14497 // - Only match adjacent instructions in same basic block
14498 // - Only equality constraints
14499 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14500 // - Only one replacement instruction
14501 //
14502 // // Change (inc mov) to lea
14503 // peephole %{
14504 //   // lea should only be emitted when beneficial
14505 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14506 //   // increment preceded by register-register move
14507 //   peepmatch ( incI_rReg movI );
14508 //   // require that the destination register of the increment
14509 //   // match the destination register of the move
14510 //   peepconstraint ( 0.dst == 1.dst );
14511 //   // construct a replacement instruction that sets
14512 //   // the destination to ( move's source register + one )
14513 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14514 // %}
14515 //
14516 // 2. Procedural replacement
14517 // - More flexible finding relevent nodes
14518 // - More flexible constraints
14519 // - More flexible transformations
14520 // - May utilise architecture-dependent API more effectively
14521 // - Currently only one replacement instruction due to adlc parsing capabilities
14522 //
14523 // // Change (inc mov) to lea
14524 // peephole %{
14525 //   // lea should only be emitted when beneficial
14526 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14527 //   // the rule numbers of these nodes inside are passed into the function below
14528 //   peepmatch ( incI_rReg movI );
14529 //   // the method that takes the responsibility of transformation
14530 //   peepprocedure ( inc_mov_to_lea );
14531 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14532 //   // node is passed into the function above
14533 //   peepreplace ( leaI_rReg_immI() );
14534 // %}
14535 
14536 // These instructions is not matched by the matcher but used by the peephole
14537 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14538 %{
14539   predicate(false);
14540   match(Set dst (AddI src1 src2));
14541   format %{ "leal    $dst, [$src1 + $src2]" %}
14542   ins_encode %{
14543     Register dst = $dst$$Register;
14544     Register src1 = $src1$$Register;
14545     Register src2 = $src2$$Register;
14546     if (src1 != rbp && src1 != r13) {
14547       __ leal(dst, Address(src1, src2, Address::times_1));
14548     } else {
14549       assert(src2 != rbp && src2 != r13, "");
14550       __ leal(dst, Address(src2, src1, Address::times_1));
14551     }
14552   %}
14553   ins_pipe(ialu_reg_reg);
14554 %}
14555 
14556 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14557 %{
14558   predicate(false);
14559   match(Set dst (AddI src1 src2));
14560   format %{ "leal    $dst, [$src1 + $src2]" %}
14561   ins_encode %{
14562     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14563   %}
14564   ins_pipe(ialu_reg_reg);
14565 %}
14566 
14567 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14568 %{
14569   predicate(false);
14570   match(Set dst (LShiftI src shift));
14571   format %{ "leal    $dst, [$src << $shift]" %}
14572   ins_encode %{
14573     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14574     Register src = $src$$Register;
14575     if (scale == Address::times_2 && src != rbp && src != r13) {
14576       __ leal($dst$$Register, Address(src, src, Address::times_1));
14577     } else {
14578       __ leal($dst$$Register, Address(noreg, src, scale));
14579     }
14580   %}
14581   ins_pipe(ialu_reg_reg);
14582 %}
14583 
14584 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14585 %{
14586   predicate(false);
14587   match(Set dst (AddL src1 src2));
14588   format %{ "leaq    $dst, [$src1 + $src2]" %}
14589   ins_encode %{
14590     Register dst = $dst$$Register;
14591     Register src1 = $src1$$Register;
14592     Register src2 = $src2$$Register;
14593     if (src1 != rbp && src1 != r13) {
14594       __ leaq(dst, Address(src1, src2, Address::times_1));
14595     } else {
14596       assert(src2 != rbp && src2 != r13, "");
14597       __ leaq(dst, Address(src2, src1, Address::times_1));
14598     }
14599   %}
14600   ins_pipe(ialu_reg_reg);
14601 %}
14602 
14603 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14604 %{
14605   predicate(false);
14606   match(Set dst (AddL src1 src2));
14607   format %{ "leaq    $dst, [$src1 + $src2]" %}
14608   ins_encode %{
14609     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14610   %}
14611   ins_pipe(ialu_reg_reg);
14612 %}
14613 
14614 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14615 %{
14616   predicate(false);
14617   match(Set dst (LShiftL src shift));
14618   format %{ "leaq    $dst, [$src << $shift]" %}
14619   ins_encode %{
14620     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14621     Register src = $src$$Register;
14622     if (scale == Address::times_2 && src != rbp && src != r13) {
14623       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14624     } else {
14625       __ leaq($dst$$Register, Address(noreg, src, scale));
14626     }
14627   %}
14628   ins_pipe(ialu_reg_reg);
14629 %}
14630 
14631 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14632 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14633 // processors with at least partial ALU support for lea
14634 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14635 // beneficial for processors with full ALU support
14636 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14637 
14638 peephole
14639 %{
14640   peeppredicate(VM_Version::supports_fast_2op_lea());
14641   peepmatch (addI_rReg);
14642   peepprocedure (lea_coalesce_reg);
14643   peepreplace (leaI_rReg_rReg_peep());
14644 %}
14645 
14646 peephole
14647 %{
14648   peeppredicate(VM_Version::supports_fast_2op_lea());
14649   peepmatch (addI_rReg_imm);
14650   peepprocedure (lea_coalesce_imm);
14651   peepreplace (leaI_rReg_immI_peep());
14652 %}
14653 
14654 peephole
14655 %{
14656   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14657                 VM_Version::is_intel_cascade_lake());
14658   peepmatch (incI_rReg);
14659   peepprocedure (lea_coalesce_imm);
14660   peepreplace (leaI_rReg_immI_peep());
14661 %}
14662 
14663 peephole
14664 %{
14665   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14666                 VM_Version::is_intel_cascade_lake());
14667   peepmatch (decI_rReg);
14668   peepprocedure (lea_coalesce_imm);
14669   peepreplace (leaI_rReg_immI_peep());
14670 %}
14671 
14672 peephole
14673 %{
14674   peeppredicate(VM_Version::supports_fast_2op_lea());
14675   peepmatch (salI_rReg_immI2);
14676   peepprocedure (lea_coalesce_imm);
14677   peepreplace (leaI_rReg_immI2_peep());
14678 %}
14679 
14680 peephole
14681 %{
14682   peeppredicate(VM_Version::supports_fast_2op_lea());
14683   peepmatch (addL_rReg);
14684   peepprocedure (lea_coalesce_reg);
14685   peepreplace (leaL_rReg_rReg_peep());
14686 %}
14687 
14688 peephole
14689 %{
14690   peeppredicate(VM_Version::supports_fast_2op_lea());
14691   peepmatch (addL_rReg_imm);
14692   peepprocedure (lea_coalesce_imm);
14693   peepreplace (leaL_rReg_immL32_peep());
14694 %}
14695 
14696 peephole
14697 %{
14698   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14699                 VM_Version::is_intel_cascade_lake());
14700   peepmatch (incL_rReg);
14701   peepprocedure (lea_coalesce_imm);
14702   peepreplace (leaL_rReg_immL32_peep());
14703 %}
14704 
14705 peephole
14706 %{
14707   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14708                 VM_Version::is_intel_cascade_lake());
14709   peepmatch (decL_rReg);
14710   peepprocedure (lea_coalesce_imm);
14711   peepreplace (leaL_rReg_immL32_peep());
14712 %}
14713 
14714 peephole
14715 %{
14716   peeppredicate(VM_Version::supports_fast_2op_lea());
14717   peepmatch (salL_rReg_immI2);
14718   peepprocedure (lea_coalesce_imm);
14719   peepreplace (leaL_rReg_immI2_peep());
14720 %}
14721 
14722 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14723 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14724 
14725 //int variant
14726 peephole
14727 %{
14728   peepmatch (testI_reg);
14729   peepprocedure (test_may_remove);
14730 %}
14731 
14732 //long variant
14733 peephole
14734 %{
14735   peepmatch (testL_reg);
14736   peepprocedure (test_may_remove);
14737 %}
14738 
14739 
14740 //----------SMARTSPILL RULES---------------------------------------------------
14741 // These must follow all instruction definitions as they use the names
14742 // defined in the instructions definitions.