1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 
  442 extern RegMask _ANY_REG_mask;
  443 extern RegMask _PTR_REG_mask;
  444 extern RegMask _PTR_REG_NO_RBP_mask;
  445 extern RegMask _PTR_NO_RAX_REG_mask;
  446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  447 extern RegMask _LONG_REG_mask;
  448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  449 extern RegMask _LONG_NO_RCX_REG_mask;
  450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  451 extern RegMask _INT_REG_mask;
  452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  453 extern RegMask _INT_NO_RCX_REG_mask;
  454 extern RegMask _INT_NO_RBP_R13_REG_mask;
  455 extern RegMask _FLOAT_REG_mask;
  456 
  457 extern RegMask _STACK_OR_PTR_REG_mask;
  458 extern RegMask _STACK_OR_LONG_REG_mask;
  459 extern RegMask _STACK_OR_INT_REG_mask;
  460 
  461 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  462 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  463 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  464 
  465 %}
  466 
  467 source %{
  468 #define   RELOC_IMM64    Assembler::imm_operand
  469 #define   RELOC_DISP32   Assembler::disp32_operand
  470 
  471 #define __ masm->
  472 
  473 RegMask _ANY_REG_mask;
  474 RegMask _PTR_REG_mask;
  475 RegMask _PTR_REG_NO_RBP_mask;
  476 RegMask _PTR_NO_RAX_REG_mask;
  477 RegMask _PTR_NO_RAX_RBX_REG_mask;
  478 RegMask _LONG_REG_mask;
  479 RegMask _LONG_NO_RAX_RDX_REG_mask;
  480 RegMask _LONG_NO_RCX_REG_mask;
  481 RegMask _LONG_NO_RBP_R13_REG_mask;
  482 RegMask _INT_REG_mask;
  483 RegMask _INT_NO_RAX_RDX_REG_mask;
  484 RegMask _INT_NO_RCX_REG_mask;
  485 RegMask _INT_NO_RBP_R13_REG_mask;
  486 RegMask _FLOAT_REG_mask;
  487 RegMask _STACK_OR_PTR_REG_mask;
  488 RegMask _STACK_OR_LONG_REG_mask;
  489 RegMask _STACK_OR_INT_REG_mask;
  490 
  491 static bool need_r12_heapbase() {
  492   return UseCompressedOops;
  493 }
  494 
  495 void reg_mask_init() {
  496   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  497 
  498   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  499   // We derive a number of subsets from it.
  500   _ANY_REG_mask = _ALL_REG_mask;
  501 
  502   if (PreserveFramePointer) {
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  504     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  505   }
  506   if (need_r12_heapbase()) {
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  508     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  509   }
  510 
  511   _PTR_REG_mask = _ANY_REG_mask;
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  515   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  516   if (!UseAPX) {
  517     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  519       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  520     }
  521   }
  522 
  523   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  524   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  525 
  526   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  528   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  529 
  530   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  532   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  533 
  534   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  536   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  537 
  538 
  539   _LONG_REG_mask = _PTR_REG_mask;
  540   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  541   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  542 
  543   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  547   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  548 
  549   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  551   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  552 
  553   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  557   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  558 
  559   _INT_REG_mask = _ALL_INT_REG_mask;
  560   if (!UseAPX) {
  561     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  562       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  563     }
  564   }
  565 
  566   if (PreserveFramePointer) {
  567     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  568   }
  569   if (need_r12_heapbase()) {
  570     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  571   }
  572 
  573   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  574   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  575 
  576   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  578   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  579 
  580   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  581   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  582 
  583   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  585   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  586 
  587   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  588   // from the float_reg_legacy/float_reg_evex register class.
  589   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  590 }
  591 
  592 static bool generate_vzeroupper(Compile* C) {
  593   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  594 }
  595 
  596 static int clear_avx_size() {
  597   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  598 }
  599 
  600 // !!!!! Special hack to get all types of calls to specify the byte offset
  601 //       from the start of the call to the point where the return address
  602 //       will point.
  603 int MachCallStaticJavaNode::ret_addr_offset()
  604 {
  605   int offset = 5; // 5 bytes from start of call to where return address points
  606   offset += clear_avx_size();
  607   return offset;
  608 }
  609 
  610 int MachCallDynamicJavaNode::ret_addr_offset()
  611 {
  612   int offset = 15; // 15 bytes from start of call to where return address points
  613   offset += clear_avx_size();
  614   return offset;
  615 }
  616 
  617 int MachCallRuntimeNode::ret_addr_offset() {
  618   int offset = 13; // movq r10,#addr; callq (r10)
  619   if (this->ideal_Opcode() != Op_CallLeafVector) {
  620     offset += clear_avx_size();
  621   }
  622   return offset;
  623 }
  624 //
  625 // Compute padding required for nodes which need alignment
  626 //
  627 
  628 // The address of the call instruction needs to be 4-byte aligned to
  629 // ensure that it does not span a cache line so that it can be patched.
  630 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  631 {
  632   current_offset += clear_avx_size(); // skip vzeroupper
  633   current_offset += 1; // skip call opcode byte
  634   return align_up(current_offset, alignment_required()) - current_offset;
  635 }
  636 
  637 // The address of the call instruction needs to be 4-byte aligned to
  638 // ensure that it does not span a cache line so that it can be patched.
  639 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  640 {
  641   current_offset += clear_avx_size(); // skip vzeroupper
  642   current_offset += 11; // skip movq instruction + call opcode byte
  643   return align_up(current_offset, alignment_required()) - current_offset;
  644 }
  645 
  646 // This could be in MacroAssembler but it's fairly C2 specific
  647 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  648   Label exit;
  649   __ jccb(Assembler::noParity, exit);
  650   __ pushf();
  651   //
  652   // comiss/ucomiss instructions set ZF,PF,CF flags and
  653   // zero OF,AF,SF for NaN values.
  654   // Fixup flags by zeroing ZF,PF so that compare of NaN
  655   // values returns 'less than' result (CF is set).
  656   // Leave the rest of flags unchanged.
  657   //
  658   //    7 6 5 4 3 2 1 0
  659   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  660   //    0 0 1 0 1 0 1 1   (0x2B)
  661   //
  662   __ andq(Address(rsp, 0), 0xffffff2b);
  663   __ popf();
  664   __ bind(exit);
  665 }
  666 
  667 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  668   Label done;
  669   __ movl(dst, -1);
  670   __ jcc(Assembler::parity, done);
  671   __ jcc(Assembler::below, done);
  672   __ setcc(Assembler::notEqual, dst);
  673   __ bind(done);
  674 }
  675 
  676 // Math.min()    # Math.max()
  677 // --------------------------
  678 // ucomis[s/d]   #
  679 // ja   -> b     # a
  680 // jp   -> NaN   # NaN
  681 // jb   -> a     # b
  682 // je            #
  683 // |-jz -> a | b # a & b
  684 // |    -> a     #
  685 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  686                             XMMRegister a, XMMRegister b,
  687                             XMMRegister xmmt, Register rt,
  688                             bool min, bool single) {
  689 
  690   Label nan, zero, below, above, done;
  691 
  692   if (single)
  693     __ ucomiss(a, b);
  694   else
  695     __ ucomisd(a, b);
  696 
  697   if (dst->encoding() != (min ? b : a)->encoding())
  698     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  699   else
  700     __ jccb(Assembler::above, done);
  701 
  702   __ jccb(Assembler::parity, nan);  // PF=1
  703   __ jccb(Assembler::below, below); // CF=1
  704 
  705   // equal
  706   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  707   if (single) {
  708     __ ucomiss(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movflt(dst, a);
  712     __ jmp(done);
  713   }
  714   else {
  715     __ ucomisd(a, xmmt);
  716     __ jccb(Assembler::equal, zero);
  717 
  718     __ movdbl(dst, a);
  719     __ jmp(done);
  720   }
  721 
  722   __ bind(zero);
  723   if (min)
  724     __ vpor(dst, a, b, Assembler::AVX_128bit);
  725   else
  726     __ vpand(dst, a, b, Assembler::AVX_128bit);
  727 
  728   __ jmp(done);
  729 
  730   __ bind(above);
  731   if (single)
  732     __ movflt(dst, min ? b : a);
  733   else
  734     __ movdbl(dst, min ? b : a);
  735 
  736   __ jmp(done);
  737 
  738   __ bind(nan);
  739   if (single) {
  740     __ movl(rt, 0x7fc00000); // Float.NaN
  741     __ movdl(dst, rt);
  742   }
  743   else {
  744     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  745     __ movdq(dst, rt);
  746   }
  747   __ jmp(done);
  748 
  749   __ bind(below);
  750   if (single)
  751     __ movflt(dst, min ? a : b);
  752   else
  753     __ movdbl(dst, min ? a : b);
  754 
  755   __ bind(done);
  756 }
  757 
  758 //=============================================================================
  759 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  760 
  761 int ConstantTable::calculate_table_base_offset() const {
  762   return 0;  // absolute addressing, no offset
  763 }
  764 
  765 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  766 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  767   ShouldNotReachHere();
  768 }
  769 
  770 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  771   // Empty encoding
  772 }
  773 
  774 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  775   return 0;
  776 }
  777 
  778 #ifndef PRODUCT
  779 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  780   st->print("# MachConstantBaseNode (empty encoding)");
  781 }
  782 #endif
  783 
  784 
  785 //=============================================================================
  786 #ifndef PRODUCT
  787 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  788   Compile* C = ra_->C;
  789 
  790   int framesize = C->output()->frame_size_in_bytes();
  791   int bangsize = C->output()->bang_size_in_bytes();
  792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  793   // Remove wordSize for return addr which is already pushed.
  794   framesize -= wordSize;
  795 
  796   if (C->output()->need_stack_bang(bangsize)) {
  797     framesize -= wordSize;
  798     st->print("# stack bang (%d bytes)", bangsize);
  799     st->print("\n\t");
  800     st->print("pushq   rbp\t# Save rbp");
  801     if (PreserveFramePointer) {
  802         st->print("\n\t");
  803         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  804     }
  805     if (framesize) {
  806       st->print("\n\t");
  807       st->print("subq    rsp, #%d\t# Create frame",framesize);
  808     }
  809   } else {
  810     st->print("subq    rsp, #%d\t# Create frame",framesize);
  811     st->print("\n\t");
  812     framesize -= wordSize;
  813     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  814     if (PreserveFramePointer) {
  815       st->print("\n\t");
  816       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  817       if (framesize > 0) {
  818         st->print("\n\t");
  819         st->print("addq    rbp, #%d", framesize);
  820       }
  821     }
  822   }
  823 
  824   if (VerifyStackAtCalls) {
  825     st->print("\n\t");
  826     framesize -= wordSize;
  827     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  828 #ifdef ASSERT
  829     st->print("\n\t");
  830     st->print("# stack alignment check");
  831 #endif
  832   }
  833   if (C->stub_function() != nullptr) {
  834     st->print("\n\t");
  835     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  836     st->print("\n\t");
  837     st->print("je      fast_entry\t");
  838     st->print("\n\t");
  839     st->print("call    #nmethod_entry_barrier_stub\t");
  840     st->print("\n\tfast_entry:");
  841   }
  842   st->cr();
  843 }
  844 #endif
  845 
  846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  847   Compile* C = ra_->C;
  848 
  849   int framesize = C->output()->frame_size_in_bytes();
  850   int bangsize = C->output()->bang_size_in_bytes();
  851 
  852   if (C->clinit_barrier_on_entry()) {
  853     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  854     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  855 
  856     Label L_skip_barrier;
  857     Register klass = rscratch1;
  858 
  859     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  860     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  861 
  862     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  863 
  864     __ bind(L_skip_barrier);
  865   }
  866 
  867   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  868 
  869   C->output()->set_frame_complete(__ offset());
  870 
  871   if (C->has_mach_constant_base_node()) {
  872     // NOTE: We set the table base offset here because users might be
  873     // emitted before MachConstantBaseNode.
  874     ConstantTable& constant_table = C->output()->constant_table();
  875     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  876   }
  877 }
  878 
  879 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  880 {
  881   return MachNode::size(ra_); // too many variables; just compute it
  882                               // the hard way
  883 }
  884 
  885 int MachPrologNode::reloc() const
  886 {
  887   return 0; // a large enough number
  888 }
  889 
  890 //=============================================================================
  891 #ifndef PRODUCT
  892 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  893 {
  894   Compile* C = ra_->C;
  895   if (generate_vzeroupper(C)) {
  896     st->print("vzeroupper");
  897     st->cr(); st->print("\t");
  898   }
  899 
  900   int framesize = C->output()->frame_size_in_bytes();
  901   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  902   // Remove word for return adr already pushed
  903   // and RBP
  904   framesize -= 2*wordSize;
  905 
  906   if (framesize) {
  907     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  908     st->print("\t");
  909   }
  910 
  911   st->print_cr("popq    rbp");
  912   if (do_polling() && C->is_method_compilation()) {
  913     st->print("\t");
  914     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  915                  "ja      #safepoint_stub\t"
  916                  "# Safepoint: poll for GC");
  917   }
  918 }
  919 #endif
  920 
  921 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  922 {
  923   Compile* C = ra_->C;
  924 
  925   if (generate_vzeroupper(C)) {
  926     // Clear upper bits of YMM registers when current compiled code uses
  927     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  928     __ vzeroupper();
  929   }
  930 
  931   int framesize = C->output()->frame_size_in_bytes();
  932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  933   // Remove word for return adr already pushed
  934   // and RBP
  935   framesize -= 2*wordSize;
  936 
  937   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  938 
  939   if (framesize) {
  940     __ addq(rsp, framesize);
  941   }
  942 
  943   __ popq(rbp);
  944 
  945   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  946     __ reserved_stack_check();
  947   }
  948 
  949   if (do_polling() && C->is_method_compilation()) {
  950     Label dummy_label;
  951     Label* code_stub = &dummy_label;
  952     if (!C->output()->in_scratch_emit_size()) {
  953       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  954       C->output()->add_stub(stub);
  955       code_stub = &stub->entry();
  956     }
  957     __ relocate(relocInfo::poll_return_type);
  958     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  959   }
  960 }
  961 
  962 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  963 {
  964   return MachNode::size(ra_); // too many variables; just compute it
  965                               // the hard way
  966 }
  967 
  968 int MachEpilogNode::reloc() const
  969 {
  970   return 2; // a large enough number
  971 }
  972 
  973 const Pipeline* MachEpilogNode::pipeline() const
  974 {
  975   return MachNode::pipeline_class();
  976 }
  977 
  978 //=============================================================================
  979 
  980 enum RC {
  981   rc_bad,
  982   rc_int,
  983   rc_kreg,
  984   rc_float,
  985   rc_stack
  986 };
  987 
  988 static enum RC rc_class(OptoReg::Name reg)
  989 {
  990   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  991 
  992   if (OptoReg::is_stack(reg)) return rc_stack;
  993 
  994   VMReg r = OptoReg::as_VMReg(reg);
  995 
  996   if (r->is_Register()) return rc_int;
  997 
  998   if (r->is_KRegister()) return rc_kreg;
  999 
 1000   assert(r->is_XMMRegister(), "must be");
 1001   return rc_float;
 1002 }
 1003 
 1004 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1005 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1006                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1007 
 1008 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1009                      int stack_offset, int reg, uint ireg, outputStream* st);
 1010 
 1011 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1012                                       int dst_offset, uint ireg, outputStream* st) {
 1013   if (masm) {
 1014     switch (ireg) {
 1015     case Op_VecS:
 1016       __ movq(Address(rsp, -8), rax);
 1017       __ movl(rax, Address(rsp, src_offset));
 1018       __ movl(Address(rsp, dst_offset), rax);
 1019       __ movq(rax, Address(rsp, -8));
 1020       break;
 1021     case Op_VecD:
 1022       __ pushq(Address(rsp, src_offset));
 1023       __ popq (Address(rsp, dst_offset));
 1024       break;
 1025     case Op_VecX:
 1026       __ pushq(Address(rsp, src_offset));
 1027       __ popq (Address(rsp, dst_offset));
 1028       __ pushq(Address(rsp, src_offset+8));
 1029       __ popq (Address(rsp, dst_offset+8));
 1030       break;
 1031     case Op_VecY:
 1032       __ vmovdqu(Address(rsp, -32), xmm0);
 1033       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1034       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1035       __ vmovdqu(xmm0, Address(rsp, -32));
 1036       break;
 1037     case Op_VecZ:
 1038       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1039       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1040       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1041       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #ifndef PRODUCT
 1047   } else {
 1048     switch (ireg) {
 1049     case Op_VecS:
 1050       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                 "movl    rax, [rsp + #%d]\n\t"
 1052                 "movl    [rsp + #%d], rax\n\t"
 1053                 "movq    rax, [rsp - #8]",
 1054                 src_offset, dst_offset);
 1055       break;
 1056     case Op_VecD:
 1057       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1058                 "popq    [rsp + #%d]",
 1059                 src_offset, dst_offset);
 1060       break;
 1061      case Op_VecX:
 1062       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1063                 "popq    [rsp + #%d]\n\t"
 1064                 "pushq   [rsp + #%d]\n\t"
 1065                 "popq    [rsp + #%d]",
 1066                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1067       break;
 1068     case Op_VecY:
 1069       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1070                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1071                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1072                 "vmovdqu xmm0, [rsp - #32]",
 1073                 src_offset, dst_offset);
 1074       break;
 1075     case Op_VecZ:
 1076       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1077                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1078                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1079                 "vmovdqu xmm0, [rsp - #64]",
 1080                 src_offset, dst_offset);
 1081       break;
 1082     default:
 1083       ShouldNotReachHere();
 1084     }
 1085 #endif
 1086   }
 1087 }
 1088 
 1089 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1090                                        PhaseRegAlloc* ra_,
 1091                                        bool do_size,
 1092                                        outputStream* st) const {
 1093   assert(masm != nullptr || st  != nullptr, "sanity");
 1094   // Get registers to move
 1095   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1096   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1097   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1098   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1099 
 1100   enum RC src_second_rc = rc_class(src_second);
 1101   enum RC src_first_rc = rc_class(src_first);
 1102   enum RC dst_second_rc = rc_class(dst_second);
 1103   enum RC dst_first_rc = rc_class(dst_first);
 1104 
 1105   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1106          "must move at least 1 register" );
 1107 
 1108   if (src_first == dst_first && src_second == dst_second) {
 1109     // Self copy, no move
 1110     return 0;
 1111   }
 1112   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1113     uint ireg = ideal_reg();
 1114     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1115     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1116     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1117       // mem -> mem
 1118       int src_offset = ra_->reg2offset(src_first);
 1119       int dst_offset = ra_->reg2offset(dst_first);
 1120       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1121     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1122       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1123     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1124       int stack_offset = ra_->reg2offset(dst_first);
 1125       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1126     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1127       int stack_offset = ra_->reg2offset(src_first);
 1128       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1129     } else {
 1130       ShouldNotReachHere();
 1131     }
 1132     return 0;
 1133   }
 1134   if (src_first_rc == rc_stack) {
 1135     // mem ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // mem -> mem
 1138       assert(src_second != dst_first, "overlap");
 1139       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1140           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1141         // 64-bit
 1142         int src_offset = ra_->reg2offset(src_first);
 1143         int dst_offset = ra_->reg2offset(dst_first);
 1144         if (masm) {
 1145           __ pushq(Address(rsp, src_offset));
 1146           __ popq (Address(rsp, dst_offset));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1150                     "popq    [rsp + #%d]",
 1151                      src_offset, dst_offset);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         // No pushl/popl, so:
 1159         int src_offset = ra_->reg2offset(src_first);
 1160         int dst_offset = ra_->reg2offset(dst_first);
 1161         if (masm) {
 1162           __ movq(Address(rsp, -8), rax);
 1163           __ movl(rax, Address(rsp, src_offset));
 1164           __ movl(Address(rsp, dst_offset), rax);
 1165           __ movq(rax, Address(rsp, -8));
 1166 #ifndef PRODUCT
 1167         } else {
 1168           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1169                     "movl    rax, [rsp + #%d]\n\t"
 1170                     "movl    [rsp + #%d], rax\n\t"
 1171                     "movq    rax, [rsp - #8]",
 1172                      src_offset, dst_offset);
 1173 #endif
 1174         }
 1175       }
 1176       return 0;
 1177     } else if (dst_first_rc == rc_int) {
 1178       // mem -> gpr
 1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1181         // 64-bit
 1182         int offset = ra_->reg2offset(src_first);
 1183         if (masm) {
 1184           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1185 #ifndef PRODUCT
 1186         } else {
 1187           st->print("movq    %s, [rsp + #%d]\t# spill",
 1188                      Matcher::regName[dst_first],
 1189                      offset);
 1190 #endif
 1191         }
 1192       } else {
 1193         // 32-bit
 1194         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1195         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1196         int offset = ra_->reg2offset(src_first);
 1197         if (masm) {
 1198           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1199 #ifndef PRODUCT
 1200         } else {
 1201           st->print("movl    %s, [rsp + #%d]\t# spill",
 1202                      Matcher::regName[dst_first],
 1203                      offset);
 1204 #endif
 1205         }
 1206       }
 1207       return 0;
 1208     } else if (dst_first_rc == rc_float) {
 1209       // mem-> xmm
 1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1212         // 64-bit
 1213         int offset = ra_->reg2offset(src_first);
 1214         if (masm) {
 1215           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1216 #ifndef PRODUCT
 1217         } else {
 1218           st->print("%s  %s, [rsp + #%d]\t# spill",
 1219                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1220                      Matcher::regName[dst_first],
 1221                      offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         int offset = ra_->reg2offset(src_first);
 1229         if (masm) {
 1230           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("movss   %s, [rsp + #%d]\t# spill",
 1234                      Matcher::regName[dst_first],
 1235                      offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_kreg) {
 1241       // mem -> kreg
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (masm) {
 1247           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1248 #ifndef PRODUCT
 1249         } else {
 1250           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1251                      Matcher::regName[dst_first],
 1252                      offset);
 1253 #endif
 1254         }
 1255       }
 1256       return 0;
 1257     }
 1258   } else if (src_first_rc == rc_int) {
 1259     // gpr ->
 1260     if (dst_first_rc == rc_stack) {
 1261       // gpr -> mem
 1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1264         // 64-bit
 1265         int offset = ra_->reg2offset(dst_first);
 1266         if (masm) {
 1267           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1268 #ifndef PRODUCT
 1269         } else {
 1270           st->print("movq    [rsp + #%d], %s\t# spill",
 1271                      offset,
 1272                      Matcher::regName[src_first]);
 1273 #endif
 1274         }
 1275       } else {
 1276         // 32-bit
 1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1279         int offset = ra_->reg2offset(dst_first);
 1280         if (masm) {
 1281           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("movl    [rsp + #%d], %s\t# spill",
 1285                      offset,
 1286                      Matcher::regName[src_first]);
 1287 #endif
 1288         }
 1289       }
 1290       return 0;
 1291     } else if (dst_first_rc == rc_int) {
 1292       // gpr -> gpr
 1293       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1294           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1295         // 64-bit
 1296         if (masm) {
 1297           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1298                   as_Register(Matcher::_regEncode[src_first]));
 1299 #ifndef PRODUCT
 1300         } else {
 1301           st->print("movq    %s, %s\t# spill",
 1302                      Matcher::regName[dst_first],
 1303                      Matcher::regName[src_first]);
 1304 #endif
 1305         }
 1306         return 0;
 1307       } else {
 1308         // 32-bit
 1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1311         if (masm) {
 1312           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1313                   as_Register(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("movl    %s, %s\t# spill",
 1317                      Matcher::regName[dst_first],
 1318                      Matcher::regName[src_first]);
 1319 #endif
 1320         }
 1321         return 0;
 1322       }
 1323     } else if (dst_first_rc == rc_float) {
 1324       // gpr -> xmm
 1325       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1326           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1327         // 64-bit
 1328         if (masm) {
 1329           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1330 #ifndef PRODUCT
 1331         } else {
 1332           st->print("movdq   %s, %s\t# spill",
 1333                      Matcher::regName[dst_first],
 1334                      Matcher::regName[src_first]);
 1335 #endif
 1336         }
 1337       } else {
 1338         // 32-bit
 1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1341         if (masm) {
 1342           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1343 #ifndef PRODUCT
 1344         } else {
 1345           st->print("movdl   %s, %s\t# spill",
 1346                      Matcher::regName[dst_first],
 1347                      Matcher::regName[src_first]);
 1348 #endif
 1349         }
 1350       }
 1351       return 0;
 1352     } else if (dst_first_rc == rc_kreg) {
 1353       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1354           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1355         // 64-bit
 1356         if (masm) {
 1357           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1358   #ifndef PRODUCT
 1359         } else {
 1360            st->print("kmovq   %s, %s\t# spill",
 1361                        Matcher::regName[dst_first],
 1362                        Matcher::regName[src_first]);
 1363   #endif
 1364         }
 1365       }
 1366       Unimplemented();
 1367       return 0;
 1368     }
 1369   } else if (src_first_rc == rc_float) {
 1370     // xmm ->
 1371     if (dst_first_rc == rc_stack) {
 1372       // xmm -> mem
 1373       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1374           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1375         // 64-bit
 1376         int offset = ra_->reg2offset(dst_first);
 1377         if (masm) {
 1378           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1379 #ifndef PRODUCT
 1380         } else {
 1381           st->print("movsd   [rsp + #%d], %s\t# spill",
 1382                      offset,
 1383                      Matcher::regName[src_first]);
 1384 #endif
 1385         }
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         int offset = ra_->reg2offset(dst_first);
 1391         if (masm) {
 1392           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1393 #ifndef PRODUCT
 1394         } else {
 1395           st->print("movss   [rsp + #%d], %s\t# spill",
 1396                      offset,
 1397                      Matcher::regName[src_first]);
 1398 #endif
 1399         }
 1400       }
 1401       return 0;
 1402     } else if (dst_first_rc == rc_int) {
 1403       // xmm -> gpr
 1404       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1405           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1406         // 64-bit
 1407         if (masm) {
 1408           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       } else {
 1417         // 32-bit
 1418         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1419         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1420         if (masm) {
 1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1422 #ifndef PRODUCT
 1423         } else {
 1424           st->print("movdl   %s, %s\t# spill",
 1425                      Matcher::regName[dst_first],
 1426                      Matcher::regName[src_first]);
 1427 #endif
 1428         }
 1429       }
 1430       return 0;
 1431     } else if (dst_first_rc == rc_float) {
 1432       // xmm -> xmm
 1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1435         // 64-bit
 1436         if (masm) {
 1437           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1438 #ifndef PRODUCT
 1439         } else {
 1440           st->print("%s  %s, %s\t# spill",
 1441                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       } else {
 1447         // 32-bit
 1448         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1449         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1450         if (masm) {
 1451           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1452 #ifndef PRODUCT
 1453         } else {
 1454           st->print("%s  %s, %s\t# spill",
 1455                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1456                      Matcher::regName[dst_first],
 1457                      Matcher::regName[src_first]);
 1458 #endif
 1459         }
 1460       }
 1461       return 0;
 1462     } else if (dst_first_rc == rc_kreg) {
 1463       assert(false, "Illegal spilling");
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_kreg) {
 1467     if (dst_first_rc == rc_stack) {
 1468       // mem -> kreg
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         int offset = ra_->reg2offset(dst_first);
 1473         if (masm) {
 1474           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1478                      offset,
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       }
 1483       return 0;
 1484     } else if (dst_first_rc == rc_int) {
 1485       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1486           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1487         // 64-bit
 1488         if (masm) {
 1489           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1490 #ifndef PRODUCT
 1491         } else {
 1492          st->print("kmovq   %s, %s\t# spill",
 1493                      Matcher::regName[dst_first],
 1494                      Matcher::regName[src_first]);
 1495 #endif
 1496         }
 1497       }
 1498       Unimplemented();
 1499       return 0;
 1500     } else if (dst_first_rc == rc_kreg) {
 1501       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1502           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1503         // 64-bit
 1504         if (masm) {
 1505           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508          st->print("kmovq   %s, %s\t# spill",
 1509                      Matcher::regName[dst_first],
 1510                      Matcher::regName[src_first]);
 1511 #endif
 1512         }
 1513       }
 1514       return 0;
 1515     } else if (dst_first_rc == rc_float) {
 1516       assert(false, "Illegal spill");
 1517       return 0;
 1518     }
 1519   }
 1520 
 1521   assert(0," foo ");
 1522   Unimplemented();
 1523   return 0;
 1524 }
 1525 
 1526 #ifndef PRODUCT
 1527 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1528   implementation(nullptr, ra_, false, st);
 1529 }
 1530 #endif
 1531 
 1532 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1533   implementation(masm, ra_, false, nullptr);
 1534 }
 1535 
 1536 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1537   return MachNode::size(ra_);
 1538 }
 1539 
 1540 //=============================================================================
 1541 #ifndef PRODUCT
 1542 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1543 {
 1544   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1545   int reg = ra_->get_reg_first(this);
 1546   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1547             Matcher::regName[reg], offset);
 1548 }
 1549 #endif
 1550 
 1551 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1552 {
 1553   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1554   int reg = ra_->get_encode(this);
 1555 
 1556   __ lea(as_Register(reg), Address(rsp, offset));
 1557 }
 1558 
 1559 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1560 {
 1561   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1562   if (ra_->get_encode(this) > 15) {
 1563     return (offset < 0x80) ? 6 : 9; // REX2
 1564   } else {
 1565     return (offset < 0x80) ? 5 : 8; // REX
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1590 {
 1591   return MachNode::size(ra_); // too many variables; just compute it
 1592                               // the hard way
 1593 }
 1594 
 1595 
 1596 //=============================================================================
 1597 
 1598 bool Matcher::supports_vector_calling_convention(void) {
 1599   return EnableVectorSupport;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     DEBUG_ONLY(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     DEBUG_ONLY(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     // No relocation needed
 1863     __ mov64(r10, (int64_t) $meth$$method);
 1864     __ call(r10);
 1865     __ post_call_nop();
 1866   %}
 1867 
 1868   enc_class Java_Static_Call(method meth)
 1869   %{
 1870     // JAVA STATIC CALL
 1871     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1872     // determine who we intended to call.
 1873     if (!_method) {
 1874       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1875     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1876       // The NOP here is purely to ensure that eliding a call to
 1877       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1878       __ addr_nop_5();
 1879       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1880     } else {
 1881       int method_index = resolved_method_index(masm);
 1882       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1883                                                   : static_call_Relocation::spec(method_index);
 1884       address mark = __ pc();
 1885       int call_offset = __ offset();
 1886       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1887       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1888         // Calls of the same statically bound method can share
 1889         // a stub to the interpreter.
 1890         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1891       } else {
 1892         // Emit stubs for static call.
 1893         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1894         __ clear_inst_mark();
 1895         if (stub == nullptr) {
 1896           ciEnv::current()->record_failure("CodeCache is full");
 1897           return;
 1898         }
 1899       }
 1900     }
 1901     __ post_call_nop();
 1902   %}
 1903 
 1904   enc_class Java_Dynamic_Call(method meth) %{
 1905     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1906     __ post_call_nop();
 1907   %}
 1908 
 1909 %}
 1910 
 1911 
 1912 
 1913 //----------FRAME--------------------------------------------------------------
 1914 // Definition of frame structure and management information.
 1915 //
 1916 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1917 //                             |   (to get allocators register number
 1918 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1919 //  r   CALLER     |        |
 1920 //  o     |        +--------+      pad to even-align allocators stack-slot
 1921 //  w     V        |  pad0  |        numbers; owned by CALLER
 1922 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1923 //  h     ^        |   in   |  5
 1924 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1925 //  |     |        |        |  3
 1926 //  |     |        +--------+
 1927 //  V     |        | old out|      Empty on Intel, window on Sparc
 1928 //        |    old |preserve|      Must be even aligned.
 1929 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1930 //        |        |   in   |  3   area for Intel ret address
 1931 //     Owned by    |preserve|      Empty on Sparc.
 1932 //       SELF      +--------+
 1933 //        |        |  pad2  |  2   pad to align old SP
 1934 //        |        +--------+  1
 1935 //        |        | locks  |  0
 1936 //        |        +--------+----> OptoReg::stack0(), even aligned
 1937 //        |        |  pad1  | 11   pad to align new SP
 1938 //        |        +--------+
 1939 //        |        |        | 10
 1940 //        |        | spills |  9   spills
 1941 //        V        |        |  8   (pad0 slot for callee)
 1942 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1943 //        ^        |  out   |  7
 1944 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1945 //     Owned by    +--------+
 1946 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1947 //        |    new |preserve|      Must be even-aligned.
 1948 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1949 //        |        |        |
 1950 //
 1951 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1952 //         known from SELF's arguments and the Java calling convention.
 1953 //         Region 6-7 is determined per call site.
 1954 // Note 2: If the calling convention leaves holes in the incoming argument
 1955 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1956 //         are owned by the CALLEE.  Holes should not be necessary in the
 1957 //         incoming area, as the Java calling convention is completely under
 1958 //         the control of the AD file.  Doubles can be sorted and packed to
 1959 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1960 //         varargs C calling conventions.
 1961 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1962 //         even aligned with pad0 as needed.
 1963 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1964 //         region 6-11 is even aligned; it may be padded out more so that
 1965 //         the region from SP to FP meets the minimum stack alignment.
 1966 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1967 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1968 //         SP meets the minimum alignment.
 1969 
 1970 frame
 1971 %{
 1972   // These three registers define part of the calling convention
 1973   // between compiled code and the interpreter.
 1974   inline_cache_reg(RAX);                // Inline Cache Register
 1975 
 1976   // Optional: name the operand used by cisc-spilling to access
 1977   // [stack_pointer + offset]
 1978   cisc_spilling_operand_name(indOffset32);
 1979 
 1980   // Number of stack slots consumed by locking an object
 1981   sync_stack_slots(2);
 1982 
 1983   // Compiled code's Frame Pointer
 1984   frame_pointer(RSP);
 1985 
 1986   // Interpreter stores its frame pointer in a register which is
 1987   // stored to the stack by I2CAdaptors.
 1988   // I2CAdaptors convert from interpreted java to compiled java.
 1989   interpreter_frame_pointer(RBP);
 1990 
 1991   // Stack alignment requirement
 1992   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1993 
 1994   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1995   // for calls to C.  Supports the var-args backing area for register parms.
 1996   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1997 
 1998   // The after-PROLOG location of the return address.  Location of
 1999   // return address specifies a type (REG or STACK) and a number
 2000   // representing the register number (i.e. - use a register name) or
 2001   // stack slot.
 2002   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2003   // Otherwise, it is above the locks and verification slot and alignment word
 2004   return_addr(STACK - 2 +
 2005               align_up((Compile::current()->in_preserve_stack_slots() +
 2006                         Compile::current()->fixed_slots()),
 2007                        stack_alignment_in_slots()));
 2008 
 2009   // Location of compiled Java return values.  Same as C for now.
 2010   return_value
 2011   %{
 2012     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2013            "only return normal values");
 2014 
 2015     static const int lo[Op_RegL + 1] = {
 2016       0,
 2017       0,
 2018       RAX_num,  // Op_RegN
 2019       RAX_num,  // Op_RegI
 2020       RAX_num,  // Op_RegP
 2021       XMM0_num, // Op_RegF
 2022       XMM0_num, // Op_RegD
 2023       RAX_num   // Op_RegL
 2024     };
 2025     static const int hi[Op_RegL + 1] = {
 2026       0,
 2027       0,
 2028       OptoReg::Bad, // Op_RegN
 2029       OptoReg::Bad, // Op_RegI
 2030       RAX_H_num,    // Op_RegP
 2031       OptoReg::Bad, // Op_RegF
 2032       XMM0b_num,    // Op_RegD
 2033       RAX_H_num     // Op_RegL
 2034     };
 2035     // Excluded flags and vector registers.
 2036     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2037     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2038   %}
 2039 %}
 2040 
 2041 //----------ATTRIBUTES---------------------------------------------------------
 2042 //----------Operand Attributes-------------------------------------------------
 2043 op_attrib op_cost(0);        // Required cost attribute
 2044 
 2045 //----------Instruction Attributes---------------------------------------------
 2046 ins_attrib ins_cost(100);       // Required cost attribute
 2047 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2048 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2049                                 // a non-matching short branch variant
 2050                                 // of some long branch?
 2051 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2052                                 // be a power of 2) specifies the
 2053                                 // alignment that some part of the
 2054                                 // instruction (not necessarily the
 2055                                 // start) requires.  If > 1, a
 2056                                 // compute_padding() function must be
 2057                                 // provided for the instruction
 2058 
 2059 //----------OPERANDS-----------------------------------------------------------
 2060 // Operand definitions must precede instruction definitions for correct parsing
 2061 // in the ADLC because operands constitute user defined types which are used in
 2062 // instruction definitions.
 2063 
 2064 //----------Simple Operands----------------------------------------------------
 2065 // Immediate Operands
 2066 // Integer Immediate
 2067 operand immI()
 2068 %{
 2069   match(ConI);
 2070 
 2071   op_cost(10);
 2072   format %{ %}
 2073   interface(CONST_INTER);
 2074 %}
 2075 
 2076 // Constant for test vs zero
 2077 operand immI_0()
 2078 %{
 2079   predicate(n->get_int() == 0);
 2080   match(ConI);
 2081 
 2082   op_cost(0);
 2083   format %{ %}
 2084   interface(CONST_INTER);
 2085 %}
 2086 
 2087 // Constant for increment
 2088 operand immI_1()
 2089 %{
 2090   predicate(n->get_int() == 1);
 2091   match(ConI);
 2092 
 2093   op_cost(0);
 2094   format %{ %}
 2095   interface(CONST_INTER);
 2096 %}
 2097 
 2098 // Constant for decrement
 2099 operand immI_M1()
 2100 %{
 2101   predicate(n->get_int() == -1);
 2102   match(ConI);
 2103 
 2104   op_cost(0);
 2105   format %{ %}
 2106   interface(CONST_INTER);
 2107 %}
 2108 
 2109 operand immI_2()
 2110 %{
 2111   predicate(n->get_int() == 2);
 2112   match(ConI);
 2113 
 2114   op_cost(0);
 2115   format %{ %}
 2116   interface(CONST_INTER);
 2117 %}
 2118 
 2119 operand immI_4()
 2120 %{
 2121   predicate(n->get_int() == 4);
 2122   match(ConI);
 2123 
 2124   op_cost(0);
 2125   format %{ %}
 2126   interface(CONST_INTER);
 2127 %}
 2128 
 2129 operand immI_8()
 2130 %{
 2131   predicate(n->get_int() == 8);
 2132   match(ConI);
 2133 
 2134   op_cost(0);
 2135   format %{ %}
 2136   interface(CONST_INTER);
 2137 %}
 2138 
 2139 // Valid scale values for addressing modes
 2140 operand immI2()
 2141 %{
 2142   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2143   match(ConI);
 2144 
 2145   format %{ %}
 2146   interface(CONST_INTER);
 2147 %}
 2148 
 2149 operand immU7()
 2150 %{
 2151   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2152   match(ConI);
 2153 
 2154   op_cost(5);
 2155   format %{ %}
 2156   interface(CONST_INTER);
 2157 %}
 2158 
 2159 operand immI8()
 2160 %{
 2161   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2162   match(ConI);
 2163 
 2164   op_cost(5);
 2165   format %{ %}
 2166   interface(CONST_INTER);
 2167 %}
 2168 
 2169 operand immU8()
 2170 %{
 2171   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2172   match(ConI);
 2173 
 2174   op_cost(5);
 2175   format %{ %}
 2176   interface(CONST_INTER);
 2177 %}
 2178 
 2179 operand immI16()
 2180 %{
 2181   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2182   match(ConI);
 2183 
 2184   op_cost(10);
 2185   format %{ %}
 2186   interface(CONST_INTER);
 2187 %}
 2188 
 2189 // Int Immediate non-negative
 2190 operand immU31()
 2191 %{
 2192   predicate(n->get_int() >= 0);
 2193   match(ConI);
 2194 
 2195   op_cost(0);
 2196   format %{ %}
 2197   interface(CONST_INTER);
 2198 %}
 2199 
 2200 // Pointer Immediate
 2201 operand immP()
 2202 %{
 2203   match(ConP);
 2204 
 2205   op_cost(10);
 2206   format %{ %}
 2207   interface(CONST_INTER);
 2208 %}
 2209 
 2210 // Null Pointer Immediate
 2211 operand immP0()
 2212 %{
 2213   predicate(n->get_ptr() == 0);
 2214   match(ConP);
 2215 
 2216   op_cost(5);
 2217   format %{ %}
 2218   interface(CONST_INTER);
 2219 %}
 2220 
 2221 // Pointer Immediate
 2222 operand immN() %{
 2223   match(ConN);
 2224 
 2225   op_cost(10);
 2226   format %{ %}
 2227   interface(CONST_INTER);
 2228 %}
 2229 
 2230 operand immNKlass() %{
 2231   match(ConNKlass);
 2232 
 2233   op_cost(10);
 2234   format %{ %}
 2235   interface(CONST_INTER);
 2236 %}
 2237 
 2238 // Null Pointer Immediate
 2239 operand immN0() %{
 2240   predicate(n->get_narrowcon() == 0);
 2241   match(ConN);
 2242 
 2243   op_cost(5);
 2244   format %{ %}
 2245   interface(CONST_INTER);
 2246 %}
 2247 
 2248 operand immP31()
 2249 %{
 2250   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2251             && (n->get_ptr() >> 31) == 0);
 2252   match(ConP);
 2253 
 2254   op_cost(5);
 2255   format %{ %}
 2256   interface(CONST_INTER);
 2257 %}
 2258 
 2259 
 2260 // Long Immediate
 2261 operand immL()
 2262 %{
 2263   match(ConL);
 2264 
 2265   op_cost(20);
 2266   format %{ %}
 2267   interface(CONST_INTER);
 2268 %}
 2269 
 2270 // Long Immediate 8-bit
 2271 operand immL8()
 2272 %{
 2273   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2274   match(ConL);
 2275 
 2276   op_cost(5);
 2277   format %{ %}
 2278   interface(CONST_INTER);
 2279 %}
 2280 
 2281 // Long Immediate 32-bit unsigned
 2282 operand immUL32()
 2283 %{
 2284   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2285   match(ConL);
 2286 
 2287   op_cost(10);
 2288   format %{ %}
 2289   interface(CONST_INTER);
 2290 %}
 2291 
 2292 // Long Immediate 32-bit signed
 2293 operand immL32()
 2294 %{
 2295   predicate(n->get_long() == (int) (n->get_long()));
 2296   match(ConL);
 2297 
 2298   op_cost(15);
 2299   format %{ %}
 2300   interface(CONST_INTER);
 2301 %}
 2302 
 2303 operand immL_Pow2()
 2304 %{
 2305   predicate(is_power_of_2((julong)n->get_long()));
 2306   match(ConL);
 2307 
 2308   op_cost(15);
 2309   format %{ %}
 2310   interface(CONST_INTER);
 2311 %}
 2312 
 2313 operand immL_NotPow2()
 2314 %{
 2315   predicate(is_power_of_2((julong)~n->get_long()));
 2316   match(ConL);
 2317 
 2318   op_cost(15);
 2319   format %{ %}
 2320   interface(CONST_INTER);
 2321 %}
 2322 
 2323 // Long Immediate zero
 2324 operand immL0()
 2325 %{
 2326   predicate(n->get_long() == 0L);
 2327   match(ConL);
 2328 
 2329   op_cost(10);
 2330   format %{ %}
 2331   interface(CONST_INTER);
 2332 %}
 2333 
 2334 // Constant for increment
 2335 operand immL1()
 2336 %{
 2337   predicate(n->get_long() == 1);
 2338   match(ConL);
 2339 
 2340   format %{ %}
 2341   interface(CONST_INTER);
 2342 %}
 2343 
 2344 // Constant for decrement
 2345 operand immL_M1()
 2346 %{
 2347   predicate(n->get_long() == -1);
 2348   match(ConL);
 2349 
 2350   format %{ %}
 2351   interface(CONST_INTER);
 2352 %}
 2353 
 2354 // Long Immediate: low 32-bit mask
 2355 operand immL_32bits()
 2356 %{
 2357   predicate(n->get_long() == 0xFFFFFFFFL);
 2358   match(ConL);
 2359   op_cost(20);
 2360 
 2361   format %{ %}
 2362   interface(CONST_INTER);
 2363 %}
 2364 
 2365 // Int Immediate: 2^n-1, positive
 2366 operand immI_Pow2M1()
 2367 %{
 2368   predicate((n->get_int() > 0)
 2369             && is_power_of_2((juint)n->get_int() + 1));
 2370   match(ConI);
 2371 
 2372   op_cost(20);
 2373   format %{ %}
 2374   interface(CONST_INTER);
 2375 %}
 2376 
 2377 // Float Immediate zero
 2378 operand immF0()
 2379 %{
 2380   predicate(jint_cast(n->getf()) == 0);
 2381   match(ConF);
 2382 
 2383   op_cost(5);
 2384   format %{ %}
 2385   interface(CONST_INTER);
 2386 %}
 2387 
 2388 // Float Immediate
 2389 operand immF()
 2390 %{
 2391   match(ConF);
 2392 
 2393   op_cost(15);
 2394   format %{ %}
 2395   interface(CONST_INTER);
 2396 %}
 2397 
 2398 // Half Float Immediate
 2399 operand immH()
 2400 %{
 2401   match(ConH);
 2402 
 2403   op_cost(15);
 2404   format %{ %}
 2405   interface(CONST_INTER);
 2406 %}
 2407 
 2408 // Double Immediate zero
 2409 operand immD0()
 2410 %{
 2411   predicate(jlong_cast(n->getd()) == 0);
 2412   match(ConD);
 2413 
 2414   op_cost(5);
 2415   format %{ %}
 2416   interface(CONST_INTER);
 2417 %}
 2418 
 2419 // Double Immediate
 2420 operand immD()
 2421 %{
 2422   match(ConD);
 2423 
 2424   op_cost(15);
 2425   format %{ %}
 2426   interface(CONST_INTER);
 2427 %}
 2428 
 2429 // Immediates for special shifts (sign extend)
 2430 
 2431 // Constants for increment
 2432 operand immI_16()
 2433 %{
 2434   predicate(n->get_int() == 16);
 2435   match(ConI);
 2436 
 2437   format %{ %}
 2438   interface(CONST_INTER);
 2439 %}
 2440 
 2441 operand immI_24()
 2442 %{
 2443   predicate(n->get_int() == 24);
 2444   match(ConI);
 2445 
 2446   format %{ %}
 2447   interface(CONST_INTER);
 2448 %}
 2449 
 2450 // Constant for byte-wide masking
 2451 operand immI_255()
 2452 %{
 2453   predicate(n->get_int() == 255);
 2454   match(ConI);
 2455 
 2456   format %{ %}
 2457   interface(CONST_INTER);
 2458 %}
 2459 
 2460 // Constant for short-wide masking
 2461 operand immI_65535()
 2462 %{
 2463   predicate(n->get_int() == 65535);
 2464   match(ConI);
 2465 
 2466   format %{ %}
 2467   interface(CONST_INTER);
 2468 %}
 2469 
 2470 // Constant for byte-wide masking
 2471 operand immL_255()
 2472 %{
 2473   predicate(n->get_long() == 255);
 2474   match(ConL);
 2475 
 2476   format %{ %}
 2477   interface(CONST_INTER);
 2478 %}
 2479 
 2480 // Constant for short-wide masking
 2481 operand immL_65535()
 2482 %{
 2483   predicate(n->get_long() == 65535);
 2484   match(ConL);
 2485 
 2486   format %{ %}
 2487   interface(CONST_INTER);
 2488 %}
 2489 
 2490 operand kReg()
 2491 %{
 2492   constraint(ALLOC_IN_RC(vectmask_reg));
 2493   match(RegVectMask);
 2494   format %{%}
 2495   interface(REG_INTER);
 2496 %}
 2497 
 2498 // Register Operands
 2499 // Integer Register
 2500 operand rRegI()
 2501 %{
 2502   constraint(ALLOC_IN_RC(int_reg));
 2503   match(RegI);
 2504 
 2505   match(rax_RegI);
 2506   match(rbx_RegI);
 2507   match(rcx_RegI);
 2508   match(rdx_RegI);
 2509   match(rdi_RegI);
 2510 
 2511   format %{ %}
 2512   interface(REG_INTER);
 2513 %}
 2514 
 2515 // Special Registers
 2516 operand rax_RegI()
 2517 %{
 2518   constraint(ALLOC_IN_RC(int_rax_reg));
 2519   match(RegI);
 2520   match(rRegI);
 2521 
 2522   format %{ "RAX" %}
 2523   interface(REG_INTER);
 2524 %}
 2525 
 2526 // Special Registers
 2527 operand rbx_RegI()
 2528 %{
 2529   constraint(ALLOC_IN_RC(int_rbx_reg));
 2530   match(RegI);
 2531   match(rRegI);
 2532 
 2533   format %{ "RBX" %}
 2534   interface(REG_INTER);
 2535 %}
 2536 
 2537 operand rcx_RegI()
 2538 %{
 2539   constraint(ALLOC_IN_RC(int_rcx_reg));
 2540   match(RegI);
 2541   match(rRegI);
 2542 
 2543   format %{ "RCX" %}
 2544   interface(REG_INTER);
 2545 %}
 2546 
 2547 operand rdx_RegI()
 2548 %{
 2549   constraint(ALLOC_IN_RC(int_rdx_reg));
 2550   match(RegI);
 2551   match(rRegI);
 2552 
 2553   format %{ "RDX" %}
 2554   interface(REG_INTER);
 2555 %}
 2556 
 2557 operand rdi_RegI()
 2558 %{
 2559   constraint(ALLOC_IN_RC(int_rdi_reg));
 2560   match(RegI);
 2561   match(rRegI);
 2562 
 2563   format %{ "RDI" %}
 2564   interface(REG_INTER);
 2565 %}
 2566 
 2567 operand no_rax_rdx_RegI()
 2568 %{
 2569   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2570   match(RegI);
 2571   match(rbx_RegI);
 2572   match(rcx_RegI);
 2573   match(rdi_RegI);
 2574 
 2575   format %{ %}
 2576   interface(REG_INTER);
 2577 %}
 2578 
 2579 operand no_rbp_r13_RegI()
 2580 %{
 2581   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2582   match(RegI);
 2583   match(rRegI);
 2584   match(rax_RegI);
 2585   match(rbx_RegI);
 2586   match(rcx_RegI);
 2587   match(rdx_RegI);
 2588   match(rdi_RegI);
 2589 
 2590   format %{ %}
 2591   interface(REG_INTER);
 2592 %}
 2593 
 2594 // Pointer Register
 2595 operand any_RegP()
 2596 %{
 2597   constraint(ALLOC_IN_RC(any_reg));
 2598   match(RegP);
 2599   match(rax_RegP);
 2600   match(rbx_RegP);
 2601   match(rdi_RegP);
 2602   match(rsi_RegP);
 2603   match(rbp_RegP);
 2604   match(r15_RegP);
 2605   match(rRegP);
 2606 
 2607   format %{ %}
 2608   interface(REG_INTER);
 2609 %}
 2610 
 2611 operand rRegP()
 2612 %{
 2613   constraint(ALLOC_IN_RC(ptr_reg));
 2614   match(RegP);
 2615   match(rax_RegP);
 2616   match(rbx_RegP);
 2617   match(rdi_RegP);
 2618   match(rsi_RegP);
 2619   match(rbp_RegP);  // See Q&A below about
 2620   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2621 
 2622   format %{ %}
 2623   interface(REG_INTER);
 2624 %}
 2625 
 2626 operand rRegN() %{
 2627   constraint(ALLOC_IN_RC(int_reg));
 2628   match(RegN);
 2629 
 2630   format %{ %}
 2631   interface(REG_INTER);
 2632 %}
 2633 
 2634 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2635 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2636 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2637 // The output of an instruction is controlled by the allocator, which respects
 2638 // register class masks, not match rules.  Unless an instruction mentions
 2639 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2640 // by the allocator as an input.
 2641 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2642 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2643 // result, RBP is not included in the output of the instruction either.
 2644 
 2645 // This operand is not allowed to use RBP even if
 2646 // RBP is not used to hold the frame pointer.
 2647 operand no_rbp_RegP()
 2648 %{
 2649   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2650   match(RegP);
 2651   match(rbx_RegP);
 2652   match(rsi_RegP);
 2653   match(rdi_RegP);
 2654 
 2655   format %{ %}
 2656   interface(REG_INTER);
 2657 %}
 2658 
 2659 // Special Registers
 2660 // Return a pointer value
 2661 operand rax_RegP()
 2662 %{
 2663   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2664   match(RegP);
 2665   match(rRegP);
 2666 
 2667   format %{ %}
 2668   interface(REG_INTER);
 2669 %}
 2670 
 2671 // Special Registers
 2672 // Return a compressed pointer value
 2673 operand rax_RegN()
 2674 %{
 2675   constraint(ALLOC_IN_RC(int_rax_reg));
 2676   match(RegN);
 2677   match(rRegN);
 2678 
 2679   format %{ %}
 2680   interface(REG_INTER);
 2681 %}
 2682 
 2683 // Used in AtomicAdd
 2684 operand rbx_RegP()
 2685 %{
 2686   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2687   match(RegP);
 2688   match(rRegP);
 2689 
 2690   format %{ %}
 2691   interface(REG_INTER);
 2692 %}
 2693 
 2694 operand rsi_RegP()
 2695 %{
 2696   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2697   match(RegP);
 2698   match(rRegP);
 2699 
 2700   format %{ %}
 2701   interface(REG_INTER);
 2702 %}
 2703 
 2704 operand rbp_RegP()
 2705 %{
 2706   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2707   match(RegP);
 2708   match(rRegP);
 2709 
 2710   format %{ %}
 2711   interface(REG_INTER);
 2712 %}
 2713 
 2714 // Used in rep stosq
 2715 operand rdi_RegP()
 2716 %{
 2717   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2718   match(RegP);
 2719   match(rRegP);
 2720 
 2721   format %{ %}
 2722   interface(REG_INTER);
 2723 %}
 2724 
 2725 operand r15_RegP()
 2726 %{
 2727   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2728   match(RegP);
 2729   match(rRegP);
 2730 
 2731   format %{ %}
 2732   interface(REG_INTER);
 2733 %}
 2734 
 2735 operand rRegL()
 2736 %{
 2737   constraint(ALLOC_IN_RC(long_reg));
 2738   match(RegL);
 2739   match(rax_RegL);
 2740   match(rdx_RegL);
 2741 
 2742   format %{ %}
 2743   interface(REG_INTER);
 2744 %}
 2745 
 2746 // Special Registers
 2747 operand no_rax_rdx_RegL()
 2748 %{
 2749   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2750   match(RegL);
 2751   match(rRegL);
 2752 
 2753   format %{ %}
 2754   interface(REG_INTER);
 2755 %}
 2756 
 2757 operand rax_RegL()
 2758 %{
 2759   constraint(ALLOC_IN_RC(long_rax_reg));
 2760   match(RegL);
 2761   match(rRegL);
 2762 
 2763   format %{ "RAX" %}
 2764   interface(REG_INTER);
 2765 %}
 2766 
 2767 operand rcx_RegL()
 2768 %{
 2769   constraint(ALLOC_IN_RC(long_rcx_reg));
 2770   match(RegL);
 2771   match(rRegL);
 2772 
 2773   format %{ %}
 2774   interface(REG_INTER);
 2775 %}
 2776 
 2777 operand rdx_RegL()
 2778 %{
 2779   constraint(ALLOC_IN_RC(long_rdx_reg));
 2780   match(RegL);
 2781   match(rRegL);
 2782 
 2783   format %{ %}
 2784   interface(REG_INTER);
 2785 %}
 2786 
 2787 operand r11_RegL()
 2788 %{
 2789   constraint(ALLOC_IN_RC(long_r11_reg));
 2790   match(RegL);
 2791   match(rRegL);
 2792 
 2793   format %{ %}
 2794   interface(REG_INTER);
 2795 %}
 2796 
 2797 operand no_rbp_r13_RegL()
 2798 %{
 2799   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2800   match(RegL);
 2801   match(rRegL);
 2802   match(rax_RegL);
 2803   match(rcx_RegL);
 2804   match(rdx_RegL);
 2805 
 2806   format %{ %}
 2807   interface(REG_INTER);
 2808 %}
 2809 
 2810 // Flags register, used as output of compare instructions
 2811 operand rFlagsReg()
 2812 %{
 2813   constraint(ALLOC_IN_RC(int_flags));
 2814   match(RegFlags);
 2815 
 2816   format %{ "RFLAGS" %}
 2817   interface(REG_INTER);
 2818 %}
 2819 
 2820 // Flags register, used as output of FLOATING POINT compare instructions
 2821 operand rFlagsRegU()
 2822 %{
 2823   constraint(ALLOC_IN_RC(int_flags));
 2824   match(RegFlags);
 2825 
 2826   format %{ "RFLAGS_U" %}
 2827   interface(REG_INTER);
 2828 %}
 2829 
 2830 operand rFlagsRegUCF() %{
 2831   constraint(ALLOC_IN_RC(int_flags));
 2832   match(RegFlags);
 2833   predicate(false);
 2834 
 2835   format %{ "RFLAGS_U_CF" %}
 2836   interface(REG_INTER);
 2837 %}
 2838 
 2839 // Float register operands
 2840 operand regF() %{
 2841    constraint(ALLOC_IN_RC(float_reg));
 2842    match(RegF);
 2843 
 2844    format %{ %}
 2845    interface(REG_INTER);
 2846 %}
 2847 
 2848 // Float register operands
 2849 operand legRegF() %{
 2850    constraint(ALLOC_IN_RC(float_reg_legacy));
 2851    match(RegF);
 2852 
 2853    format %{ %}
 2854    interface(REG_INTER);
 2855 %}
 2856 
 2857 // Float register operands
 2858 operand vlRegF() %{
 2859    constraint(ALLOC_IN_RC(float_reg_vl));
 2860    match(RegF);
 2861 
 2862    format %{ %}
 2863    interface(REG_INTER);
 2864 %}
 2865 
 2866 // Double register operands
 2867 operand regD() %{
 2868    constraint(ALLOC_IN_RC(double_reg));
 2869    match(RegD);
 2870 
 2871    format %{ %}
 2872    interface(REG_INTER);
 2873 %}
 2874 
 2875 // Double register operands
 2876 operand legRegD() %{
 2877    constraint(ALLOC_IN_RC(double_reg_legacy));
 2878    match(RegD);
 2879 
 2880    format %{ %}
 2881    interface(REG_INTER);
 2882 %}
 2883 
 2884 // Double register operands
 2885 operand vlRegD() %{
 2886    constraint(ALLOC_IN_RC(double_reg_vl));
 2887    match(RegD);
 2888 
 2889    format %{ %}
 2890    interface(REG_INTER);
 2891 %}
 2892 
 2893 //----------Memory Operands----------------------------------------------------
 2894 // Direct Memory Operand
 2895 // operand direct(immP addr)
 2896 // %{
 2897 //   match(addr);
 2898 
 2899 //   format %{ "[$addr]" %}
 2900 //   interface(MEMORY_INTER) %{
 2901 //     base(0xFFFFFFFF);
 2902 //     index(0x4);
 2903 //     scale(0x0);
 2904 //     disp($addr);
 2905 //   %}
 2906 // %}
 2907 
 2908 // Indirect Memory Operand
 2909 operand indirect(any_RegP reg)
 2910 %{
 2911   constraint(ALLOC_IN_RC(ptr_reg));
 2912   match(reg);
 2913 
 2914   format %{ "[$reg]" %}
 2915   interface(MEMORY_INTER) %{
 2916     base($reg);
 2917     index(0x4);
 2918     scale(0x0);
 2919     disp(0x0);
 2920   %}
 2921 %}
 2922 
 2923 // Indirect Memory Plus Short Offset Operand
 2924 operand indOffset8(any_RegP reg, immL8 off)
 2925 %{
 2926   constraint(ALLOC_IN_RC(ptr_reg));
 2927   match(AddP reg off);
 2928 
 2929   format %{ "[$reg + $off (8-bit)]" %}
 2930   interface(MEMORY_INTER) %{
 2931     base($reg);
 2932     index(0x4);
 2933     scale(0x0);
 2934     disp($off);
 2935   %}
 2936 %}
 2937 
 2938 // Indirect Memory Plus Long Offset Operand
 2939 operand indOffset32(any_RegP reg, immL32 off)
 2940 %{
 2941   constraint(ALLOC_IN_RC(ptr_reg));
 2942   match(AddP reg off);
 2943 
 2944   format %{ "[$reg + $off (32-bit)]" %}
 2945   interface(MEMORY_INTER) %{
 2946     base($reg);
 2947     index(0x4);
 2948     scale(0x0);
 2949     disp($off);
 2950   %}
 2951 %}
 2952 
 2953 // Indirect Memory Plus Index Register Plus Offset Operand
 2954 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2955 %{
 2956   constraint(ALLOC_IN_RC(ptr_reg));
 2957   match(AddP (AddP reg lreg) off);
 2958 
 2959   op_cost(10);
 2960   format %{"[$reg + $off + $lreg]" %}
 2961   interface(MEMORY_INTER) %{
 2962     base($reg);
 2963     index($lreg);
 2964     scale(0x0);
 2965     disp($off);
 2966   %}
 2967 %}
 2968 
 2969 // Indirect Memory Plus Index Register Plus Offset Operand
 2970 operand indIndex(any_RegP reg, rRegL lreg)
 2971 %{
 2972   constraint(ALLOC_IN_RC(ptr_reg));
 2973   match(AddP reg lreg);
 2974 
 2975   op_cost(10);
 2976   format %{"[$reg + $lreg]" %}
 2977   interface(MEMORY_INTER) %{
 2978     base($reg);
 2979     index($lreg);
 2980     scale(0x0);
 2981     disp(0x0);
 2982   %}
 2983 %}
 2984 
 2985 // Indirect Memory Times Scale Plus Index Register
 2986 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2987 %{
 2988   constraint(ALLOC_IN_RC(ptr_reg));
 2989   match(AddP reg (LShiftL lreg scale));
 2990 
 2991   op_cost(10);
 2992   format %{"[$reg + $lreg << $scale]" %}
 2993   interface(MEMORY_INTER) %{
 2994     base($reg);
 2995     index($lreg);
 2996     scale($scale);
 2997     disp(0x0);
 2998   %}
 2999 %}
 3000 
 3001 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3002 %{
 3003   constraint(ALLOC_IN_RC(ptr_reg));
 3004   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3005   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3006 
 3007   op_cost(10);
 3008   format %{"[$reg + pos $idx << $scale]" %}
 3009   interface(MEMORY_INTER) %{
 3010     base($reg);
 3011     index($idx);
 3012     scale($scale);
 3013     disp(0x0);
 3014   %}
 3015 %}
 3016 
 3017 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3018 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3019 %{
 3020   constraint(ALLOC_IN_RC(ptr_reg));
 3021   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3022 
 3023   op_cost(10);
 3024   format %{"[$reg + $off + $lreg << $scale]" %}
 3025   interface(MEMORY_INTER) %{
 3026     base($reg);
 3027     index($lreg);
 3028     scale($scale);
 3029     disp($off);
 3030   %}
 3031 %}
 3032 
 3033 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3034 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3035 %{
 3036   constraint(ALLOC_IN_RC(ptr_reg));
 3037   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3038   match(AddP (AddP reg (ConvI2L idx)) off);
 3039 
 3040   op_cost(10);
 3041   format %{"[$reg + $off + $idx]" %}
 3042   interface(MEMORY_INTER) %{
 3043     base($reg);
 3044     index($idx);
 3045     scale(0x0);
 3046     disp($off);
 3047   %}
 3048 %}
 3049 
 3050 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3051 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3052 %{
 3053   constraint(ALLOC_IN_RC(ptr_reg));
 3054   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3055   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3056 
 3057   op_cost(10);
 3058   format %{"[$reg + $off + $idx << $scale]" %}
 3059   interface(MEMORY_INTER) %{
 3060     base($reg);
 3061     index($idx);
 3062     scale($scale);
 3063     disp($off);
 3064   %}
 3065 %}
 3066 
 3067 // Indirect Narrow Oop Plus Offset Operand
 3068 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3069 // we can't free r12 even with CompressedOops::base() == nullptr.
 3070 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3071   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3072   constraint(ALLOC_IN_RC(ptr_reg));
 3073   match(AddP (DecodeN reg) off);
 3074 
 3075   op_cost(10);
 3076   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3077   interface(MEMORY_INTER) %{
 3078     base(0xc); // R12
 3079     index($reg);
 3080     scale(0x3);
 3081     disp($off);
 3082   %}
 3083 %}
 3084 
 3085 // Indirect Memory Operand
 3086 operand indirectNarrow(rRegN reg)
 3087 %{
 3088   predicate(CompressedOops::shift() == 0);
 3089   constraint(ALLOC_IN_RC(ptr_reg));
 3090   match(DecodeN reg);
 3091 
 3092   format %{ "[$reg]" %}
 3093   interface(MEMORY_INTER) %{
 3094     base($reg);
 3095     index(0x4);
 3096     scale(0x0);
 3097     disp(0x0);
 3098   %}
 3099 %}
 3100 
 3101 // Indirect Memory Plus Short Offset Operand
 3102 operand indOffset8Narrow(rRegN reg, immL8 off)
 3103 %{
 3104   predicate(CompressedOops::shift() == 0);
 3105   constraint(ALLOC_IN_RC(ptr_reg));
 3106   match(AddP (DecodeN reg) off);
 3107 
 3108   format %{ "[$reg + $off (8-bit)]" %}
 3109   interface(MEMORY_INTER) %{
 3110     base($reg);
 3111     index(0x4);
 3112     scale(0x0);
 3113     disp($off);
 3114   %}
 3115 %}
 3116 
 3117 // Indirect Memory Plus Long Offset Operand
 3118 operand indOffset32Narrow(rRegN reg, immL32 off)
 3119 %{
 3120   predicate(CompressedOops::shift() == 0);
 3121   constraint(ALLOC_IN_RC(ptr_reg));
 3122   match(AddP (DecodeN reg) off);
 3123 
 3124   format %{ "[$reg + $off (32-bit)]" %}
 3125   interface(MEMORY_INTER) %{
 3126     base($reg);
 3127     index(0x4);
 3128     scale(0x0);
 3129     disp($off);
 3130   %}
 3131 %}
 3132 
 3133 // Indirect Memory Plus Index Register Plus Offset Operand
 3134 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3135 %{
 3136   predicate(CompressedOops::shift() == 0);
 3137   constraint(ALLOC_IN_RC(ptr_reg));
 3138   match(AddP (AddP (DecodeN reg) lreg) off);
 3139 
 3140   op_cost(10);
 3141   format %{"[$reg + $off + $lreg]" %}
 3142   interface(MEMORY_INTER) %{
 3143     base($reg);
 3144     index($lreg);
 3145     scale(0x0);
 3146     disp($off);
 3147   %}
 3148 %}
 3149 
 3150 // Indirect Memory Plus Index Register Plus Offset Operand
 3151 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3152 %{
 3153   predicate(CompressedOops::shift() == 0);
 3154   constraint(ALLOC_IN_RC(ptr_reg));
 3155   match(AddP (DecodeN reg) lreg);
 3156 
 3157   op_cost(10);
 3158   format %{"[$reg + $lreg]" %}
 3159   interface(MEMORY_INTER) %{
 3160     base($reg);
 3161     index($lreg);
 3162     scale(0x0);
 3163     disp(0x0);
 3164   %}
 3165 %}
 3166 
 3167 // Indirect Memory Times Scale Plus Index Register
 3168 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3169 %{
 3170   predicate(CompressedOops::shift() == 0);
 3171   constraint(ALLOC_IN_RC(ptr_reg));
 3172   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3173 
 3174   op_cost(10);
 3175   format %{"[$reg + $lreg << $scale]" %}
 3176   interface(MEMORY_INTER) %{
 3177     base($reg);
 3178     index($lreg);
 3179     scale($scale);
 3180     disp(0x0);
 3181   %}
 3182 %}
 3183 
 3184 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3185 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3186 %{
 3187   predicate(CompressedOops::shift() == 0);
 3188   constraint(ALLOC_IN_RC(ptr_reg));
 3189   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3190 
 3191   op_cost(10);
 3192   format %{"[$reg + $off + $lreg << $scale]" %}
 3193   interface(MEMORY_INTER) %{
 3194     base($reg);
 3195     index($lreg);
 3196     scale($scale);
 3197     disp($off);
 3198   %}
 3199 %}
 3200 
 3201 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3202 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3203 %{
 3204   constraint(ALLOC_IN_RC(ptr_reg));
 3205   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3206   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3207 
 3208   op_cost(10);
 3209   format %{"[$reg + $off + $idx]" %}
 3210   interface(MEMORY_INTER) %{
 3211     base($reg);
 3212     index($idx);
 3213     scale(0x0);
 3214     disp($off);
 3215   %}
 3216 %}
 3217 
 3218 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3219 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3220 %{
 3221   constraint(ALLOC_IN_RC(ptr_reg));
 3222   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3223   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3224 
 3225   op_cost(10);
 3226   format %{"[$reg + $off + $idx << $scale]" %}
 3227   interface(MEMORY_INTER) %{
 3228     base($reg);
 3229     index($idx);
 3230     scale($scale);
 3231     disp($off);
 3232   %}
 3233 %}
 3234 
 3235 //----------Special Memory Operands--------------------------------------------
 3236 // Stack Slot Operand - This operand is used for loading and storing temporary
 3237 //                      values on the stack where a match requires a value to
 3238 //                      flow through memory.
 3239 operand stackSlotP(sRegP reg)
 3240 %{
 3241   constraint(ALLOC_IN_RC(stack_slots));
 3242   // No match rule because this operand is only generated in matching
 3243 
 3244   format %{ "[$reg]" %}
 3245   interface(MEMORY_INTER) %{
 3246     base(0x4);   // RSP
 3247     index(0x4);  // No Index
 3248     scale(0x0);  // No Scale
 3249     disp($reg);  // Stack Offset
 3250   %}
 3251 %}
 3252 
 3253 operand stackSlotI(sRegI reg)
 3254 %{
 3255   constraint(ALLOC_IN_RC(stack_slots));
 3256   // No match rule because this operand is only generated in matching
 3257 
 3258   format %{ "[$reg]" %}
 3259   interface(MEMORY_INTER) %{
 3260     base(0x4);   // RSP
 3261     index(0x4);  // No Index
 3262     scale(0x0);  // No Scale
 3263     disp($reg);  // Stack Offset
 3264   %}
 3265 %}
 3266 
 3267 operand stackSlotF(sRegF reg)
 3268 %{
 3269   constraint(ALLOC_IN_RC(stack_slots));
 3270   // No match rule because this operand is only generated in matching
 3271 
 3272   format %{ "[$reg]" %}
 3273   interface(MEMORY_INTER) %{
 3274     base(0x4);   // RSP
 3275     index(0x4);  // No Index
 3276     scale(0x0);  // No Scale
 3277     disp($reg);  // Stack Offset
 3278   %}
 3279 %}
 3280 
 3281 operand stackSlotD(sRegD reg)
 3282 %{
 3283   constraint(ALLOC_IN_RC(stack_slots));
 3284   // No match rule because this operand is only generated in matching
 3285 
 3286   format %{ "[$reg]" %}
 3287   interface(MEMORY_INTER) %{
 3288     base(0x4);   // RSP
 3289     index(0x4);  // No Index
 3290     scale(0x0);  // No Scale
 3291     disp($reg);  // Stack Offset
 3292   %}
 3293 %}
 3294 operand stackSlotL(sRegL reg)
 3295 %{
 3296   constraint(ALLOC_IN_RC(stack_slots));
 3297   // No match rule because this operand is only generated in matching
 3298 
 3299   format %{ "[$reg]" %}
 3300   interface(MEMORY_INTER) %{
 3301     base(0x4);   // RSP
 3302     index(0x4);  // No Index
 3303     scale(0x0);  // No Scale
 3304     disp($reg);  // Stack Offset
 3305   %}
 3306 %}
 3307 
 3308 //----------Conditional Branch Operands----------------------------------------
 3309 // Comparison Op  - This is the operation of the comparison, and is limited to
 3310 //                  the following set of codes:
 3311 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3312 //
 3313 // Other attributes of the comparison, such as unsignedness, are specified
 3314 // by the comparison instruction that sets a condition code flags register.
 3315 // That result is represented by a flags operand whose subtype is appropriate
 3316 // to the unsignedness (etc.) of the comparison.
 3317 //
 3318 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3319 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3320 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3321 
 3322 // Comparison Code
 3323 operand cmpOp()
 3324 %{
 3325   match(Bool);
 3326 
 3327   format %{ "" %}
 3328   interface(COND_INTER) %{
 3329     equal(0x4, "e");
 3330     not_equal(0x5, "ne");
 3331     less(0xC, "l");
 3332     greater_equal(0xD, "ge");
 3333     less_equal(0xE, "le");
 3334     greater(0xF, "g");
 3335     overflow(0x0, "o");
 3336     no_overflow(0x1, "no");
 3337   %}
 3338 %}
 3339 
 3340 // Comparison Code, unsigned compare.  Used by FP also, with
 3341 // C2 (unordered) turned into GT or LT already.  The other bits
 3342 // C0 and C3 are turned into Carry & Zero flags.
 3343 operand cmpOpU()
 3344 %{
 3345   match(Bool);
 3346 
 3347   format %{ "" %}
 3348   interface(COND_INTER) %{
 3349     equal(0x4, "e");
 3350     not_equal(0x5, "ne");
 3351     less(0x2, "b");
 3352     greater_equal(0x3, "ae");
 3353     less_equal(0x6, "be");
 3354     greater(0x7, "a");
 3355     overflow(0x0, "o");
 3356     no_overflow(0x1, "no");
 3357   %}
 3358 %}
 3359 
 3360 
 3361 // Floating comparisons that don't require any fixup for the unordered case,
 3362 // If both inputs of the comparison are the same, ZF is always set so we
 3363 // don't need to use cmpOpUCF2 for eq/ne
 3364 operand cmpOpUCF() %{
 3365   match(Bool);
 3366   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3367             n->as_Bool()->_test._test == BoolTest::ge ||
 3368             n->as_Bool()->_test._test == BoolTest::le ||
 3369             n->as_Bool()->_test._test == BoolTest::gt ||
 3370             n->in(1)->in(1) == n->in(1)->in(2));
 3371   format %{ "" %}
 3372   interface(COND_INTER) %{
 3373     equal(0xb, "np");
 3374     not_equal(0xa, "p");
 3375     less(0x2, "b");
 3376     greater_equal(0x3, "ae");
 3377     less_equal(0x6, "be");
 3378     greater(0x7, "a");
 3379     overflow(0x0, "o");
 3380     no_overflow(0x1, "no");
 3381   %}
 3382 %}
 3383 
 3384 
 3385 // Floating comparisons that can be fixed up with extra conditional jumps
 3386 operand cmpOpUCF2() %{
 3387   match(Bool);
 3388   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3389              n->as_Bool()->_test._test == BoolTest::eq) &&
 3390             n->in(1)->in(1) != n->in(1)->in(2));
 3391   format %{ "" %}
 3392   interface(COND_INTER) %{
 3393     equal(0x4, "e");
 3394     not_equal(0x5, "ne");
 3395     less(0x2, "b");
 3396     greater_equal(0x3, "ae");
 3397     less_equal(0x6, "be");
 3398     greater(0x7, "a");
 3399     overflow(0x0, "o");
 3400     no_overflow(0x1, "no");
 3401   %}
 3402 %}
 3403 
 3404 //----------OPERAND CLASSES----------------------------------------------------
 3405 // Operand Classes are groups of operands that are used as to simplify
 3406 // instruction definitions by not requiring the AD writer to specify separate
 3407 // instructions for every form of operand when the instruction accepts
 3408 // multiple operand types with the same basic encoding and format.  The classic
 3409 // case of this is memory operands.
 3410 
 3411 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3412                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3413                indCompressedOopOffset,
 3414                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3415                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3416                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3417 
 3418 //----------PIPELINE-----------------------------------------------------------
 3419 // Rules which define the behavior of the target architectures pipeline.
 3420 pipeline %{
 3421 
 3422 //----------ATTRIBUTES---------------------------------------------------------
 3423 attributes %{
 3424   variable_size_instructions;        // Fixed size instructions
 3425   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3426   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3427   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3428   instruction_fetch_units = 1;       // of 16 bytes
 3429 
 3430   // List of nop instructions
 3431   nops( MachNop );
 3432 %}
 3433 
 3434 //----------RESOURCES----------------------------------------------------------
 3435 // Resources are the functional units available to the machine
 3436 
 3437 // Generic P2/P3 pipeline
 3438 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3439 // 3 instructions decoded per cycle.
 3440 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3441 // 3 ALU op, only ALU0 handles mul instructions.
 3442 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3443            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3444            BR, FPU,
 3445            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3446 
 3447 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3448 // Pipeline Description specifies the stages in the machine's pipeline
 3449 
 3450 // Generic P2/P3 pipeline
 3451 pipe_desc(S0, S1, S2, S3, S4, S5);
 3452 
 3453 //----------PIPELINE CLASSES---------------------------------------------------
 3454 // Pipeline Classes describe the stages in which input and output are
 3455 // referenced by the hardware pipeline.
 3456 
 3457 // Naming convention: ialu or fpu
 3458 // Then: _reg
 3459 // Then: _reg if there is a 2nd register
 3460 // Then: _long if it's a pair of instructions implementing a long
 3461 // Then: _fat if it requires the big decoder
 3462 //   Or: _mem if it requires the big decoder and a memory unit.
 3463 
 3464 // Integer ALU reg operation
 3465 pipe_class ialu_reg(rRegI dst)
 3466 %{
 3467     single_instruction;
 3468     dst    : S4(write);
 3469     dst    : S3(read);
 3470     DECODE : S0;        // any decoder
 3471     ALU    : S3;        // any alu
 3472 %}
 3473 
 3474 // Long ALU reg operation
 3475 pipe_class ialu_reg_long(rRegL dst)
 3476 %{
 3477     instruction_count(2);
 3478     dst    : S4(write);
 3479     dst    : S3(read);
 3480     DECODE : S0(2);     // any 2 decoders
 3481     ALU    : S3(2);     // both alus
 3482 %}
 3483 
 3484 // Integer ALU reg operation using big decoder
 3485 pipe_class ialu_reg_fat(rRegI dst)
 3486 %{
 3487     single_instruction;
 3488     dst    : S4(write);
 3489     dst    : S3(read);
 3490     D0     : S0;        // big decoder only
 3491     ALU    : S3;        // any alu
 3492 %}
 3493 
 3494 // Integer ALU reg-reg operation
 3495 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3496 %{
 3497     single_instruction;
 3498     dst    : S4(write);
 3499     src    : S3(read);
 3500     DECODE : S0;        // any decoder
 3501     ALU    : S3;        // any alu
 3502 %}
 3503 
 3504 // Integer ALU reg-reg operation
 3505 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3506 %{
 3507     single_instruction;
 3508     dst    : S4(write);
 3509     src    : S3(read);
 3510     D0     : S0;        // big decoder only
 3511     ALU    : S3;        // any alu
 3512 %}
 3513 
 3514 // Integer ALU reg-mem operation
 3515 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3516 %{
 3517     single_instruction;
 3518     dst    : S5(write);
 3519     mem    : S3(read);
 3520     D0     : S0;        // big decoder only
 3521     ALU    : S4;        // any alu
 3522     MEM    : S3;        // any mem
 3523 %}
 3524 
 3525 // Integer mem operation (prefetch)
 3526 pipe_class ialu_mem(memory mem)
 3527 %{
 3528     single_instruction;
 3529     mem    : S3(read);
 3530     D0     : S0;        // big decoder only
 3531     MEM    : S3;        // any mem
 3532 %}
 3533 
 3534 // Integer Store to Memory
 3535 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3536 %{
 3537     single_instruction;
 3538     mem    : S3(read);
 3539     src    : S5(read);
 3540     D0     : S0;        // big decoder only
 3541     ALU    : S4;        // any alu
 3542     MEM    : S3;
 3543 %}
 3544 
 3545 // // Long Store to Memory
 3546 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3547 // %{
 3548 //     instruction_count(2);
 3549 //     mem    : S3(read);
 3550 //     src    : S5(read);
 3551 //     D0     : S0(2);          // big decoder only; twice
 3552 //     ALU    : S4(2);     // any 2 alus
 3553 //     MEM    : S3(2);  // Both mems
 3554 // %}
 3555 
 3556 // Integer Store to Memory
 3557 pipe_class ialu_mem_imm(memory mem)
 3558 %{
 3559     single_instruction;
 3560     mem    : S3(read);
 3561     D0     : S0;        // big decoder only
 3562     ALU    : S4;        // any alu
 3563     MEM    : S3;
 3564 %}
 3565 
 3566 // Integer ALU0 reg-reg operation
 3567 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3568 %{
 3569     single_instruction;
 3570     dst    : S4(write);
 3571     src    : S3(read);
 3572     D0     : S0;        // Big decoder only
 3573     ALU0   : S3;        // only alu0
 3574 %}
 3575 
 3576 // Integer ALU0 reg-mem operation
 3577 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3578 %{
 3579     single_instruction;
 3580     dst    : S5(write);
 3581     mem    : S3(read);
 3582     D0     : S0;        // big decoder only
 3583     ALU0   : S4;        // ALU0 only
 3584     MEM    : S3;        // any mem
 3585 %}
 3586 
 3587 // Integer ALU reg-reg operation
 3588 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3589 %{
 3590     single_instruction;
 3591     cr     : S4(write);
 3592     src1   : S3(read);
 3593     src2   : S3(read);
 3594     DECODE : S0;        // any decoder
 3595     ALU    : S3;        // any alu
 3596 %}
 3597 
 3598 // Integer ALU reg-imm operation
 3599 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3600 %{
 3601     single_instruction;
 3602     cr     : S4(write);
 3603     src1   : S3(read);
 3604     DECODE : S0;        // any decoder
 3605     ALU    : S3;        // any alu
 3606 %}
 3607 
 3608 // Integer ALU reg-mem operation
 3609 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3610 %{
 3611     single_instruction;
 3612     cr     : S4(write);
 3613     src1   : S3(read);
 3614     src2   : S3(read);
 3615     D0     : S0;        // big decoder only
 3616     ALU    : S4;        // any alu
 3617     MEM    : S3;
 3618 %}
 3619 
 3620 // Conditional move reg-reg
 3621 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3622 %{
 3623     instruction_count(4);
 3624     y      : S4(read);
 3625     q      : S3(read);
 3626     p      : S3(read);
 3627     DECODE : S0(4);     // any decoder
 3628 %}
 3629 
 3630 // Conditional move reg-reg
 3631 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3632 %{
 3633     single_instruction;
 3634     dst    : S4(write);
 3635     src    : S3(read);
 3636     cr     : S3(read);
 3637     DECODE : S0;        // any decoder
 3638 %}
 3639 
 3640 // Conditional move reg-mem
 3641 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3642 %{
 3643     single_instruction;
 3644     dst    : S4(write);
 3645     src    : S3(read);
 3646     cr     : S3(read);
 3647     DECODE : S0;        // any decoder
 3648     MEM    : S3;
 3649 %}
 3650 
 3651 // Conditional move reg-reg long
 3652 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3653 %{
 3654     single_instruction;
 3655     dst    : S4(write);
 3656     src    : S3(read);
 3657     cr     : S3(read);
 3658     DECODE : S0(2);     // any 2 decoders
 3659 %}
 3660 
 3661 // Float reg-reg operation
 3662 pipe_class fpu_reg(regD dst)
 3663 %{
 3664     instruction_count(2);
 3665     dst    : S3(read);
 3666     DECODE : S0(2);     // any 2 decoders
 3667     FPU    : S3;
 3668 %}
 3669 
 3670 // Float reg-reg operation
 3671 pipe_class fpu_reg_reg(regD dst, regD src)
 3672 %{
 3673     instruction_count(2);
 3674     dst    : S4(write);
 3675     src    : S3(read);
 3676     DECODE : S0(2);     // any 2 decoders
 3677     FPU    : S3;
 3678 %}
 3679 
 3680 // Float reg-reg operation
 3681 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3682 %{
 3683     instruction_count(3);
 3684     dst    : S4(write);
 3685     src1   : S3(read);
 3686     src2   : S3(read);
 3687     DECODE : S0(3);     // any 3 decoders
 3688     FPU    : S3(2);
 3689 %}
 3690 
 3691 // Float reg-reg operation
 3692 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3693 %{
 3694     instruction_count(4);
 3695     dst    : S4(write);
 3696     src1   : S3(read);
 3697     src2   : S3(read);
 3698     src3   : S3(read);
 3699     DECODE : S0(4);     // any 3 decoders
 3700     FPU    : S3(2);
 3701 %}
 3702 
 3703 // Float reg-reg operation
 3704 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3705 %{
 3706     instruction_count(4);
 3707     dst    : S4(write);
 3708     src1   : S3(read);
 3709     src2   : S3(read);
 3710     src3   : S3(read);
 3711     DECODE : S1(3);     // any 3 decoders
 3712     D0     : S0;        // Big decoder only
 3713     FPU    : S3(2);
 3714     MEM    : S3;
 3715 %}
 3716 
 3717 // Float reg-mem operation
 3718 pipe_class fpu_reg_mem(regD dst, memory mem)
 3719 %{
 3720     instruction_count(2);
 3721     dst    : S5(write);
 3722     mem    : S3(read);
 3723     D0     : S0;        // big decoder only
 3724     DECODE : S1;        // any decoder for FPU POP
 3725     FPU    : S4;
 3726     MEM    : S3;        // any mem
 3727 %}
 3728 
 3729 // Float reg-mem operation
 3730 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3731 %{
 3732     instruction_count(3);
 3733     dst    : S5(write);
 3734     src1   : S3(read);
 3735     mem    : S3(read);
 3736     D0     : S0;        // big decoder only
 3737     DECODE : S1(2);     // any decoder for FPU POP
 3738     FPU    : S4;
 3739     MEM    : S3;        // any mem
 3740 %}
 3741 
 3742 // Float mem-reg operation
 3743 pipe_class fpu_mem_reg(memory mem, regD src)
 3744 %{
 3745     instruction_count(2);
 3746     src    : S5(read);
 3747     mem    : S3(read);
 3748     DECODE : S0;        // any decoder for FPU PUSH
 3749     D0     : S1;        // big decoder only
 3750     FPU    : S4;
 3751     MEM    : S3;        // any mem
 3752 %}
 3753 
 3754 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3755 %{
 3756     instruction_count(3);
 3757     src1   : S3(read);
 3758     src2   : S3(read);
 3759     mem    : S3(read);
 3760     DECODE : S0(2);     // any decoder for FPU PUSH
 3761     D0     : S1;        // big decoder only
 3762     FPU    : S4;
 3763     MEM    : S3;        // any mem
 3764 %}
 3765 
 3766 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3767 %{
 3768     instruction_count(3);
 3769     src1   : S3(read);
 3770     src2   : S3(read);
 3771     mem    : S4(read);
 3772     DECODE : S0;        // any decoder for FPU PUSH
 3773     D0     : S0(2);     // big decoder only
 3774     FPU    : S4;
 3775     MEM    : S3(2);     // any mem
 3776 %}
 3777 
 3778 pipe_class fpu_mem_mem(memory dst, memory src1)
 3779 %{
 3780     instruction_count(2);
 3781     src1   : S3(read);
 3782     dst    : S4(read);
 3783     D0     : S0(2);     // big decoder only
 3784     MEM    : S3(2);     // any mem
 3785 %}
 3786 
 3787 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3788 %{
 3789     instruction_count(3);
 3790     src1   : S3(read);
 3791     src2   : S3(read);
 3792     dst    : S4(read);
 3793     D0     : S0(3);     // big decoder only
 3794     FPU    : S4;
 3795     MEM    : S3(3);     // any mem
 3796 %}
 3797 
 3798 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3799 %{
 3800     instruction_count(3);
 3801     src1   : S4(read);
 3802     mem    : S4(read);
 3803     DECODE : S0;        // any decoder for FPU PUSH
 3804     D0     : S0(2);     // big decoder only
 3805     FPU    : S4;
 3806     MEM    : S3(2);     // any mem
 3807 %}
 3808 
 3809 // Float load constant
 3810 pipe_class fpu_reg_con(regD dst)
 3811 %{
 3812     instruction_count(2);
 3813     dst    : S5(write);
 3814     D0     : S0;        // big decoder only for the load
 3815     DECODE : S1;        // any decoder for FPU POP
 3816     FPU    : S4;
 3817     MEM    : S3;        // any mem
 3818 %}
 3819 
 3820 // Float load constant
 3821 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3822 %{
 3823     instruction_count(3);
 3824     dst    : S5(write);
 3825     src    : S3(read);
 3826     D0     : S0;        // big decoder only for the load
 3827     DECODE : S1(2);     // any decoder for FPU POP
 3828     FPU    : S4;
 3829     MEM    : S3;        // any mem
 3830 %}
 3831 
 3832 // UnConditional branch
 3833 pipe_class pipe_jmp(label labl)
 3834 %{
 3835     single_instruction;
 3836     BR   : S3;
 3837 %}
 3838 
 3839 // Conditional branch
 3840 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3841 %{
 3842     single_instruction;
 3843     cr    : S1(read);
 3844     BR    : S3;
 3845 %}
 3846 
 3847 // Allocation idiom
 3848 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3849 %{
 3850     instruction_count(1); force_serialization;
 3851     fixed_latency(6);
 3852     heap_ptr : S3(read);
 3853     DECODE   : S0(3);
 3854     D0       : S2;
 3855     MEM      : S3;
 3856     ALU      : S3(2);
 3857     dst      : S5(write);
 3858     BR       : S5;
 3859 %}
 3860 
 3861 // Generic big/slow expanded idiom
 3862 pipe_class pipe_slow()
 3863 %{
 3864     instruction_count(10); multiple_bundles; force_serialization;
 3865     fixed_latency(100);
 3866     D0  : S0(2);
 3867     MEM : S3(2);
 3868 %}
 3869 
 3870 // The real do-nothing guy
 3871 pipe_class empty()
 3872 %{
 3873     instruction_count(0);
 3874 %}
 3875 
 3876 // Define the class for the Nop node
 3877 define
 3878 %{
 3879    MachNop = empty;
 3880 %}
 3881 
 3882 %}
 3883 
 3884 //----------INSTRUCTIONS-------------------------------------------------------
 3885 //
 3886 // match      -- States which machine-independent subtree may be replaced
 3887 //               by this instruction.
 3888 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3889 //               selection to identify a minimum cost tree of machine
 3890 //               instructions that matches a tree of machine-independent
 3891 //               instructions.
 3892 // format     -- A string providing the disassembly for this instruction.
 3893 //               The value of an instruction's operand may be inserted
 3894 //               by referring to it with a '$' prefix.
 3895 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3896 //               to within an encode class as $primary, $secondary, and $tertiary
 3897 //               rrspectively.  The primary opcode is commonly used to
 3898 //               indicate the type of machine instruction, while secondary
 3899 //               and tertiary are often used for prefix options or addressing
 3900 //               modes.
 3901 // ins_encode -- A list of encode classes with parameters. The encode class
 3902 //               name must have been defined in an 'enc_class' specification
 3903 //               in the encode section of the architecture description.
 3904 
 3905 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3906 // Load Float
 3907 instruct MoveF2VL(vlRegF dst, regF src) %{
 3908   match(Set dst src);
 3909   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3910   ins_encode %{
 3911     ShouldNotReachHere();
 3912   %}
 3913   ins_pipe( fpu_reg_reg );
 3914 %}
 3915 
 3916 // Load Float
 3917 instruct MoveF2LEG(legRegF dst, regF src) %{
 3918   match(Set dst src);
 3919   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3920   ins_encode %{
 3921     ShouldNotReachHere();
 3922   %}
 3923   ins_pipe( fpu_reg_reg );
 3924 %}
 3925 
 3926 // Load Float
 3927 instruct MoveVL2F(regF dst, vlRegF src) %{
 3928   match(Set dst src);
 3929   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3930   ins_encode %{
 3931     ShouldNotReachHere();
 3932   %}
 3933   ins_pipe( fpu_reg_reg );
 3934 %}
 3935 
 3936 // Load Float
 3937 instruct MoveLEG2F(regF dst, legRegF src) %{
 3938   match(Set dst src);
 3939   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3940   ins_encode %{
 3941     ShouldNotReachHere();
 3942   %}
 3943   ins_pipe( fpu_reg_reg );
 3944 %}
 3945 
 3946 // Load Double
 3947 instruct MoveD2VL(vlRegD dst, regD src) %{
 3948   match(Set dst src);
 3949   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3950   ins_encode %{
 3951     ShouldNotReachHere();
 3952   %}
 3953   ins_pipe( fpu_reg_reg );
 3954 %}
 3955 
 3956 // Load Double
 3957 instruct MoveD2LEG(legRegD dst, regD src) %{
 3958   match(Set dst src);
 3959   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3960   ins_encode %{
 3961     ShouldNotReachHere();
 3962   %}
 3963   ins_pipe( fpu_reg_reg );
 3964 %}
 3965 
 3966 // Load Double
 3967 instruct MoveVL2D(regD dst, vlRegD src) %{
 3968   match(Set dst src);
 3969   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3970   ins_encode %{
 3971     ShouldNotReachHere();
 3972   %}
 3973   ins_pipe( fpu_reg_reg );
 3974 %}
 3975 
 3976 // Load Double
 3977 instruct MoveLEG2D(regD dst, legRegD src) %{
 3978   match(Set dst src);
 3979   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3980   ins_encode %{
 3981     ShouldNotReachHere();
 3982   %}
 3983   ins_pipe( fpu_reg_reg );
 3984 %}
 3985 
 3986 //----------Load/Store/Move Instructions---------------------------------------
 3987 //----------Load Instructions--------------------------------------------------
 3988 
 3989 // Load Byte (8 bit signed)
 3990 instruct loadB(rRegI dst, memory mem)
 3991 %{
 3992   match(Set dst (LoadB mem));
 3993 
 3994   ins_cost(125);
 3995   format %{ "movsbl  $dst, $mem\t# byte" %}
 3996 
 3997   ins_encode %{
 3998     __ movsbl($dst$$Register, $mem$$Address);
 3999   %}
 4000 
 4001   ins_pipe(ialu_reg_mem);
 4002 %}
 4003 
 4004 // Load Byte (8 bit signed) into Long Register
 4005 instruct loadB2L(rRegL dst, memory mem)
 4006 %{
 4007   match(Set dst (ConvI2L (LoadB mem)));
 4008 
 4009   ins_cost(125);
 4010   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4011 
 4012   ins_encode %{
 4013     __ movsbq($dst$$Register, $mem$$Address);
 4014   %}
 4015 
 4016   ins_pipe(ialu_reg_mem);
 4017 %}
 4018 
 4019 // Load Unsigned Byte (8 bit UNsigned)
 4020 instruct loadUB(rRegI dst, memory mem)
 4021 %{
 4022   match(Set dst (LoadUB mem));
 4023 
 4024   ins_cost(125);
 4025   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4026 
 4027   ins_encode %{
 4028     __ movzbl($dst$$Register, $mem$$Address);
 4029   %}
 4030 
 4031   ins_pipe(ialu_reg_mem);
 4032 %}
 4033 
 4034 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4035 instruct loadUB2L(rRegL dst, memory mem)
 4036 %{
 4037   match(Set dst (ConvI2L (LoadUB mem)));
 4038 
 4039   ins_cost(125);
 4040   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4041 
 4042   ins_encode %{
 4043     __ movzbq($dst$$Register, $mem$$Address);
 4044   %}
 4045 
 4046   ins_pipe(ialu_reg_mem);
 4047 %}
 4048 
 4049 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4050 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4051   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4052   effect(KILL cr);
 4053 
 4054   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4055             "andl    $dst, right_n_bits($mask, 8)" %}
 4056   ins_encode %{
 4057     Register Rdst = $dst$$Register;
 4058     __ movzbq(Rdst, $mem$$Address);
 4059     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4060   %}
 4061   ins_pipe(ialu_reg_mem);
 4062 %}
 4063 
 4064 // Load Short (16 bit signed)
 4065 instruct loadS(rRegI dst, memory mem)
 4066 %{
 4067   match(Set dst (LoadS mem));
 4068 
 4069   ins_cost(125);
 4070   format %{ "movswl $dst, $mem\t# short" %}
 4071 
 4072   ins_encode %{
 4073     __ movswl($dst$$Register, $mem$$Address);
 4074   %}
 4075 
 4076   ins_pipe(ialu_reg_mem);
 4077 %}
 4078 
 4079 // Load Short (16 bit signed) to Byte (8 bit signed)
 4080 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4081   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4082 
 4083   ins_cost(125);
 4084   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4085   ins_encode %{
 4086     __ movsbl($dst$$Register, $mem$$Address);
 4087   %}
 4088   ins_pipe(ialu_reg_mem);
 4089 %}
 4090 
 4091 // Load Short (16 bit signed) into Long Register
 4092 instruct loadS2L(rRegL dst, memory mem)
 4093 %{
 4094   match(Set dst (ConvI2L (LoadS mem)));
 4095 
 4096   ins_cost(125);
 4097   format %{ "movswq $dst, $mem\t# short -> long" %}
 4098 
 4099   ins_encode %{
 4100     __ movswq($dst$$Register, $mem$$Address);
 4101   %}
 4102 
 4103   ins_pipe(ialu_reg_mem);
 4104 %}
 4105 
 4106 // Load Unsigned Short/Char (16 bit UNsigned)
 4107 instruct loadUS(rRegI dst, memory mem)
 4108 %{
 4109   match(Set dst (LoadUS mem));
 4110 
 4111   ins_cost(125);
 4112   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4113 
 4114   ins_encode %{
 4115     __ movzwl($dst$$Register, $mem$$Address);
 4116   %}
 4117 
 4118   ins_pipe(ialu_reg_mem);
 4119 %}
 4120 
 4121 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4122 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4123   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4124 
 4125   ins_cost(125);
 4126   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4127   ins_encode %{
 4128     __ movsbl($dst$$Register, $mem$$Address);
 4129   %}
 4130   ins_pipe(ialu_reg_mem);
 4131 %}
 4132 
 4133 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4134 instruct loadUS2L(rRegL dst, memory mem)
 4135 %{
 4136   match(Set dst (ConvI2L (LoadUS mem)));
 4137 
 4138   ins_cost(125);
 4139   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4140 
 4141   ins_encode %{
 4142     __ movzwq($dst$$Register, $mem$$Address);
 4143   %}
 4144 
 4145   ins_pipe(ialu_reg_mem);
 4146 %}
 4147 
 4148 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4149 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4150   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4151 
 4152   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4153   ins_encode %{
 4154     __ movzbq($dst$$Register, $mem$$Address);
 4155   %}
 4156   ins_pipe(ialu_reg_mem);
 4157 %}
 4158 
 4159 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4160 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4161   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4162   effect(KILL cr);
 4163 
 4164   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4165             "andl    $dst, right_n_bits($mask, 16)" %}
 4166   ins_encode %{
 4167     Register Rdst = $dst$$Register;
 4168     __ movzwq(Rdst, $mem$$Address);
 4169     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4170   %}
 4171   ins_pipe(ialu_reg_mem);
 4172 %}
 4173 
 4174 // Load Integer
 4175 instruct loadI(rRegI dst, memory mem)
 4176 %{
 4177   match(Set dst (LoadI mem));
 4178 
 4179   ins_cost(125);
 4180   format %{ "movl    $dst, $mem\t# int" %}
 4181 
 4182   ins_encode %{
 4183     __ movl($dst$$Register, $mem$$Address);
 4184   %}
 4185 
 4186   ins_pipe(ialu_reg_mem);
 4187 %}
 4188 
 4189 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4190 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4191   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4192 
 4193   ins_cost(125);
 4194   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4195   ins_encode %{
 4196     __ movsbl($dst$$Register, $mem$$Address);
 4197   %}
 4198   ins_pipe(ialu_reg_mem);
 4199 %}
 4200 
 4201 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4202 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4203   match(Set dst (AndI (LoadI mem) mask));
 4204 
 4205   ins_cost(125);
 4206   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4207   ins_encode %{
 4208     __ movzbl($dst$$Register, $mem$$Address);
 4209   %}
 4210   ins_pipe(ialu_reg_mem);
 4211 %}
 4212 
 4213 // Load Integer (32 bit signed) to Short (16 bit signed)
 4214 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4215   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4216 
 4217   ins_cost(125);
 4218   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4219   ins_encode %{
 4220     __ movswl($dst$$Register, $mem$$Address);
 4221   %}
 4222   ins_pipe(ialu_reg_mem);
 4223 %}
 4224 
 4225 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4226 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4227   match(Set dst (AndI (LoadI mem) mask));
 4228 
 4229   ins_cost(125);
 4230   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4231   ins_encode %{
 4232     __ movzwl($dst$$Register, $mem$$Address);
 4233   %}
 4234   ins_pipe(ialu_reg_mem);
 4235 %}
 4236 
 4237 // Load Integer into Long Register
 4238 instruct loadI2L(rRegL dst, memory mem)
 4239 %{
 4240   match(Set dst (ConvI2L (LoadI mem)));
 4241 
 4242   ins_cost(125);
 4243   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4244 
 4245   ins_encode %{
 4246     __ movslq($dst$$Register, $mem$$Address);
 4247   %}
 4248 
 4249   ins_pipe(ialu_reg_mem);
 4250 %}
 4251 
 4252 // Load Integer with mask 0xFF into Long Register
 4253 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4254   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4255 
 4256   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4257   ins_encode %{
 4258     __ movzbq($dst$$Register, $mem$$Address);
 4259   %}
 4260   ins_pipe(ialu_reg_mem);
 4261 %}
 4262 
 4263 // Load Integer with mask 0xFFFF into Long Register
 4264 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4265   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4266 
 4267   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4268   ins_encode %{
 4269     __ movzwq($dst$$Register, $mem$$Address);
 4270   %}
 4271   ins_pipe(ialu_reg_mem);
 4272 %}
 4273 
 4274 // Load Integer with a 31-bit mask into Long Register
 4275 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4276   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4277   effect(KILL cr);
 4278 
 4279   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4280             "andl    $dst, $mask" %}
 4281   ins_encode %{
 4282     Register Rdst = $dst$$Register;
 4283     __ movl(Rdst, $mem$$Address);
 4284     __ andl(Rdst, $mask$$constant);
 4285   %}
 4286   ins_pipe(ialu_reg_mem);
 4287 %}
 4288 
 4289 // Load Unsigned Integer into Long Register
 4290 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4291 %{
 4292   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4293 
 4294   ins_cost(125);
 4295   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4296 
 4297   ins_encode %{
 4298     __ movl($dst$$Register, $mem$$Address);
 4299   %}
 4300 
 4301   ins_pipe(ialu_reg_mem);
 4302 %}
 4303 
 4304 // Load Long
 4305 instruct loadL(rRegL dst, memory mem)
 4306 %{
 4307   match(Set dst (LoadL mem));
 4308 
 4309   ins_cost(125);
 4310   format %{ "movq    $dst, $mem\t# long" %}
 4311 
 4312   ins_encode %{
 4313     __ movq($dst$$Register, $mem$$Address);
 4314   %}
 4315 
 4316   ins_pipe(ialu_reg_mem); // XXX
 4317 %}
 4318 
 4319 // Load Range
 4320 instruct loadRange(rRegI dst, memory mem)
 4321 %{
 4322   match(Set dst (LoadRange mem));
 4323 
 4324   ins_cost(125); // XXX
 4325   format %{ "movl    $dst, $mem\t# range" %}
 4326   ins_encode %{
 4327     __ movl($dst$$Register, $mem$$Address);
 4328   %}
 4329   ins_pipe(ialu_reg_mem);
 4330 %}
 4331 
 4332 // Load Pointer
 4333 instruct loadP(rRegP dst, memory mem)
 4334 %{
 4335   match(Set dst (LoadP mem));
 4336   predicate(n->as_Load()->barrier_data() == 0);
 4337 
 4338   ins_cost(125); // XXX
 4339   format %{ "movq    $dst, $mem\t# ptr" %}
 4340   ins_encode %{
 4341     __ movq($dst$$Register, $mem$$Address);
 4342   %}
 4343   ins_pipe(ialu_reg_mem); // XXX
 4344 %}
 4345 
 4346 // Load Compressed Pointer
 4347 instruct loadN(rRegN dst, memory mem)
 4348 %{
 4349    predicate(n->as_Load()->barrier_data() == 0);
 4350    match(Set dst (LoadN mem));
 4351 
 4352    ins_cost(125); // XXX
 4353    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4354    ins_encode %{
 4355      __ movl($dst$$Register, $mem$$Address);
 4356    %}
 4357    ins_pipe(ialu_reg_mem); // XXX
 4358 %}
 4359 
 4360 
 4361 // Load Klass Pointer
 4362 instruct loadKlass(rRegP dst, memory mem)
 4363 %{
 4364   match(Set dst (LoadKlass mem));
 4365 
 4366   ins_cost(125); // XXX
 4367   format %{ "movq    $dst, $mem\t# class" %}
 4368   ins_encode %{
 4369     __ movq($dst$$Register, $mem$$Address);
 4370   %}
 4371   ins_pipe(ialu_reg_mem); // XXX
 4372 %}
 4373 
 4374 // Load narrow Klass Pointer
 4375 instruct loadNKlass(rRegN dst, memory mem)
 4376 %{
 4377   predicate(!UseCompactObjectHeaders);
 4378   match(Set dst (LoadNKlass mem));
 4379 
 4380   ins_cost(125); // XXX
 4381   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4382   ins_encode %{
 4383     __ movl($dst$$Register, $mem$$Address);
 4384   %}
 4385   ins_pipe(ialu_reg_mem); // XXX
 4386 %}
 4387 
 4388 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4389 %{
 4390   predicate(UseCompactObjectHeaders);
 4391   match(Set dst (LoadNKlass mem));
 4392   effect(KILL cr);
 4393   ins_cost(125);
 4394   format %{
 4395     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4396     "shrl    $dst, markWord::klass_shift"
 4397   %}
 4398   ins_encode %{
 4399     // The incoming address is pointing into obj-start + Type::klass_offset(). We need to extract
 4400     // obj-start, so that we can load from the object's mark-word instead.
 4401     Register d = $dst$$Register;
 4402     Address  s = ($mem$$Address).plus_disp(-Type::klass_offset());
 4403     if (UseAPX) {
 4404       __ eshrl(d, s, markWord::klass_shift, false);
 4405     } else {
 4406       __ movl(d, s);
 4407       __ shrl(d, markWord::klass_shift);
 4408     }
 4409   %}
 4410   ins_pipe(ialu_reg_mem);
 4411 %}
 4412 
 4413 // Load Float
 4414 instruct loadF(regF dst, memory mem)
 4415 %{
 4416   match(Set dst (LoadF mem));
 4417 
 4418   ins_cost(145); // XXX
 4419   format %{ "movss   $dst, $mem\t# float" %}
 4420   ins_encode %{
 4421     __ movflt($dst$$XMMRegister, $mem$$Address);
 4422   %}
 4423   ins_pipe(pipe_slow); // XXX
 4424 %}
 4425 
 4426 // Load Double
 4427 instruct loadD_partial(regD dst, memory mem)
 4428 %{
 4429   predicate(!UseXmmLoadAndClearUpper);
 4430   match(Set dst (LoadD mem));
 4431 
 4432   ins_cost(145); // XXX
 4433   format %{ "movlpd  $dst, $mem\t# double" %}
 4434   ins_encode %{
 4435     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4436   %}
 4437   ins_pipe(pipe_slow); // XXX
 4438 %}
 4439 
 4440 instruct loadD(regD dst, memory mem)
 4441 %{
 4442   predicate(UseXmmLoadAndClearUpper);
 4443   match(Set dst (LoadD mem));
 4444 
 4445   ins_cost(145); // XXX
 4446   format %{ "movsd   $dst, $mem\t# double" %}
 4447   ins_encode %{
 4448     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4449   %}
 4450   ins_pipe(pipe_slow); // XXX
 4451 %}
 4452 
 4453 // max = java.lang.Math.max(float a, float b)
 4454 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4455   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4456   match(Set dst (MaxF a b));
 4457   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4458   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4459   ins_encode %{
 4460     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4461   %}
 4462   ins_pipe( pipe_slow );
 4463 %}
 4464 
 4465 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4466   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4467   match(Set dst (MaxF a b));
 4468   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4469 
 4470   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4471   ins_encode %{
 4472     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4473                     false /*min*/, true /*single*/);
 4474   %}
 4475   ins_pipe( pipe_slow );
 4476 %}
 4477 
 4478 // max = java.lang.Math.max(double a, double b)
 4479 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4480   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4481   match(Set dst (MaxD a b));
 4482   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4483   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4484   ins_encode %{
 4485     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4486   %}
 4487   ins_pipe( pipe_slow );
 4488 %}
 4489 
 4490 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4491   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4492   match(Set dst (MaxD a b));
 4493   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4494 
 4495   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4496   ins_encode %{
 4497     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4498                     false /*min*/, false /*single*/);
 4499   %}
 4500   ins_pipe( pipe_slow );
 4501 %}
 4502 
 4503 // min = java.lang.Math.min(float a, float b)
 4504 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4505   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4506   match(Set dst (MinF a b));
 4507   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4508   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4509   ins_encode %{
 4510     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4511   %}
 4512   ins_pipe( pipe_slow );
 4513 %}
 4514 
 4515 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4516   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4517   match(Set dst (MinF a b));
 4518   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4519 
 4520   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4521   ins_encode %{
 4522     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4523                     true /*min*/, true /*single*/);
 4524   %}
 4525   ins_pipe( pipe_slow );
 4526 %}
 4527 
 4528 // min = java.lang.Math.min(double a, double b)
 4529 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4530   predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4531   match(Set dst (MinD a b));
 4532   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4533     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4534   ins_encode %{
 4535     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4536   %}
 4537   ins_pipe( pipe_slow );
 4538 %}
 4539 
 4540 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4541   predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
 4542   match(Set dst (MinD a b));
 4543   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4544 
 4545   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4546   ins_encode %{
 4547     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4548                     true /*min*/, false /*single*/);
 4549   %}
 4550   ins_pipe( pipe_slow );
 4551 %}
 4552 
 4553 // Load Effective Address
 4554 instruct leaP8(rRegP dst, indOffset8 mem)
 4555 %{
 4556   match(Set dst mem);
 4557 
 4558   ins_cost(110); // XXX
 4559   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4560   ins_encode %{
 4561     __ leaq($dst$$Register, $mem$$Address);
 4562   %}
 4563   ins_pipe(ialu_reg_reg_fat);
 4564 %}
 4565 
 4566 instruct leaP32(rRegP dst, indOffset32 mem)
 4567 %{
 4568   match(Set dst mem);
 4569 
 4570   ins_cost(110);
 4571   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4572   ins_encode %{
 4573     __ leaq($dst$$Register, $mem$$Address);
 4574   %}
 4575   ins_pipe(ialu_reg_reg_fat);
 4576 %}
 4577 
 4578 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4579 %{
 4580   match(Set dst mem);
 4581 
 4582   ins_cost(110);
 4583   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4584   ins_encode %{
 4585     __ leaq($dst$$Register, $mem$$Address);
 4586   %}
 4587   ins_pipe(ialu_reg_reg_fat);
 4588 %}
 4589 
 4590 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4591 %{
 4592   match(Set dst mem);
 4593 
 4594   ins_cost(110);
 4595   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4596   ins_encode %{
 4597     __ leaq($dst$$Register, $mem$$Address);
 4598   %}
 4599   ins_pipe(ialu_reg_reg_fat);
 4600 %}
 4601 
 4602 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4603 %{
 4604   match(Set dst mem);
 4605 
 4606   ins_cost(110);
 4607   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4608   ins_encode %{
 4609     __ leaq($dst$$Register, $mem$$Address);
 4610   %}
 4611   ins_pipe(ialu_reg_reg_fat);
 4612 %}
 4613 
 4614 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4615 %{
 4616   match(Set dst mem);
 4617 
 4618   ins_cost(110);
 4619   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4620   ins_encode %{
 4621     __ leaq($dst$$Register, $mem$$Address);
 4622   %}
 4623   ins_pipe(ialu_reg_reg_fat);
 4624 %}
 4625 
 4626 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4627 %{
 4628   match(Set dst mem);
 4629 
 4630   ins_cost(110);
 4631   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4632   ins_encode %{
 4633     __ leaq($dst$$Register, $mem$$Address);
 4634   %}
 4635   ins_pipe(ialu_reg_reg_fat);
 4636 %}
 4637 
 4638 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4639 %{
 4640   match(Set dst mem);
 4641 
 4642   ins_cost(110);
 4643   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4644   ins_encode %{
 4645     __ leaq($dst$$Register, $mem$$Address);
 4646   %}
 4647   ins_pipe(ialu_reg_reg_fat);
 4648 %}
 4649 
 4650 // Load Effective Address which uses Narrow (32-bits) oop
 4651 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4652 %{
 4653   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4654   match(Set dst mem);
 4655 
 4656   ins_cost(110);
 4657   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4658   ins_encode %{
 4659     __ leaq($dst$$Register, $mem$$Address);
 4660   %}
 4661   ins_pipe(ialu_reg_reg_fat);
 4662 %}
 4663 
 4664 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4665 %{
 4666   predicate(CompressedOops::shift() == 0);
 4667   match(Set dst mem);
 4668 
 4669   ins_cost(110); // XXX
 4670   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4671   ins_encode %{
 4672     __ leaq($dst$$Register, $mem$$Address);
 4673   %}
 4674   ins_pipe(ialu_reg_reg_fat);
 4675 %}
 4676 
 4677 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4678 %{
 4679   predicate(CompressedOops::shift() == 0);
 4680   match(Set dst mem);
 4681 
 4682   ins_cost(110);
 4683   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4684   ins_encode %{
 4685     __ leaq($dst$$Register, $mem$$Address);
 4686   %}
 4687   ins_pipe(ialu_reg_reg_fat);
 4688 %}
 4689 
 4690 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4691 %{
 4692   predicate(CompressedOops::shift() == 0);
 4693   match(Set dst mem);
 4694 
 4695   ins_cost(110);
 4696   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4697   ins_encode %{
 4698     __ leaq($dst$$Register, $mem$$Address);
 4699   %}
 4700   ins_pipe(ialu_reg_reg_fat);
 4701 %}
 4702 
 4703 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4704 %{
 4705   predicate(CompressedOops::shift() == 0);
 4706   match(Set dst mem);
 4707 
 4708   ins_cost(110);
 4709   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4710   ins_encode %{
 4711     __ leaq($dst$$Register, $mem$$Address);
 4712   %}
 4713   ins_pipe(ialu_reg_reg_fat);
 4714 %}
 4715 
 4716 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4717 %{
 4718   predicate(CompressedOops::shift() == 0);
 4719   match(Set dst mem);
 4720 
 4721   ins_cost(110);
 4722   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4723   ins_encode %{
 4724     __ leaq($dst$$Register, $mem$$Address);
 4725   %}
 4726   ins_pipe(ialu_reg_reg_fat);
 4727 %}
 4728 
 4729 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4730 %{
 4731   predicate(CompressedOops::shift() == 0);
 4732   match(Set dst mem);
 4733 
 4734   ins_cost(110);
 4735   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4736   ins_encode %{
 4737     __ leaq($dst$$Register, $mem$$Address);
 4738   %}
 4739   ins_pipe(ialu_reg_reg_fat);
 4740 %}
 4741 
 4742 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4743 %{
 4744   predicate(CompressedOops::shift() == 0);
 4745   match(Set dst mem);
 4746 
 4747   ins_cost(110);
 4748   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4749   ins_encode %{
 4750     __ leaq($dst$$Register, $mem$$Address);
 4751   %}
 4752   ins_pipe(ialu_reg_reg_fat);
 4753 %}
 4754 
 4755 instruct loadConI(rRegI dst, immI src)
 4756 %{
 4757   match(Set dst src);
 4758 
 4759   format %{ "movl    $dst, $src\t# int" %}
 4760   ins_encode %{
 4761     __ movl($dst$$Register, $src$$constant);
 4762   %}
 4763   ins_pipe(ialu_reg_fat); // XXX
 4764 %}
 4765 
 4766 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4767 %{
 4768   match(Set dst src);
 4769   effect(KILL cr);
 4770 
 4771   ins_cost(50);
 4772   format %{ "xorl    $dst, $dst\t# int" %}
 4773   ins_encode %{
 4774     __ xorl($dst$$Register, $dst$$Register);
 4775   %}
 4776   ins_pipe(ialu_reg);
 4777 %}
 4778 
 4779 instruct loadConL(rRegL dst, immL src)
 4780 %{
 4781   match(Set dst src);
 4782 
 4783   ins_cost(150);
 4784   format %{ "movq    $dst, $src\t# long" %}
 4785   ins_encode %{
 4786     __ mov64($dst$$Register, $src$$constant);
 4787   %}
 4788   ins_pipe(ialu_reg);
 4789 %}
 4790 
 4791 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4792 %{
 4793   match(Set dst src);
 4794   effect(KILL cr);
 4795 
 4796   ins_cost(50);
 4797   format %{ "xorl    $dst, $dst\t# long" %}
 4798   ins_encode %{
 4799     __ xorl($dst$$Register, $dst$$Register);
 4800   %}
 4801   ins_pipe(ialu_reg); // XXX
 4802 %}
 4803 
 4804 instruct loadConUL32(rRegL dst, immUL32 src)
 4805 %{
 4806   match(Set dst src);
 4807 
 4808   ins_cost(60);
 4809   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4810   ins_encode %{
 4811     __ movl($dst$$Register, $src$$constant);
 4812   %}
 4813   ins_pipe(ialu_reg);
 4814 %}
 4815 
 4816 instruct loadConL32(rRegL dst, immL32 src)
 4817 %{
 4818   match(Set dst src);
 4819 
 4820   ins_cost(70);
 4821   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4822   ins_encode %{
 4823     __ movq($dst$$Register, $src$$constant);
 4824   %}
 4825   ins_pipe(ialu_reg);
 4826 %}
 4827 
 4828 instruct loadConP(rRegP dst, immP con) %{
 4829   match(Set dst con);
 4830 
 4831   format %{ "movq    $dst, $con\t# ptr" %}
 4832   ins_encode %{
 4833     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4834   %}
 4835   ins_pipe(ialu_reg_fat); // XXX
 4836 %}
 4837 
 4838 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4839 %{
 4840   match(Set dst src);
 4841   effect(KILL cr);
 4842 
 4843   ins_cost(50);
 4844   format %{ "xorl    $dst, $dst\t# ptr" %}
 4845   ins_encode %{
 4846     __ xorl($dst$$Register, $dst$$Register);
 4847   %}
 4848   ins_pipe(ialu_reg);
 4849 %}
 4850 
 4851 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4852 %{
 4853   match(Set dst src);
 4854   effect(KILL cr);
 4855 
 4856   ins_cost(60);
 4857   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4858   ins_encode %{
 4859     __ movl($dst$$Register, $src$$constant);
 4860   %}
 4861   ins_pipe(ialu_reg);
 4862 %}
 4863 
 4864 instruct loadConF(regF dst, immF con) %{
 4865   match(Set dst con);
 4866   ins_cost(125);
 4867   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4868   ins_encode %{
 4869     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4870   %}
 4871   ins_pipe(pipe_slow);
 4872 %}
 4873 
 4874 instruct loadConH(regF dst, immH con) %{
 4875   match(Set dst con);
 4876   ins_cost(125);
 4877   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4878   ins_encode %{
 4879     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4880   %}
 4881   ins_pipe(pipe_slow);
 4882 %}
 4883 
 4884 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4885   match(Set dst src);
 4886   effect(KILL cr);
 4887   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4888   ins_encode %{
 4889     __ xorq($dst$$Register, $dst$$Register);
 4890   %}
 4891   ins_pipe(ialu_reg);
 4892 %}
 4893 
 4894 instruct loadConN(rRegN dst, immN src) %{
 4895   match(Set dst src);
 4896 
 4897   ins_cost(125);
 4898   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4899   ins_encode %{
 4900     address con = (address)$src$$constant;
 4901     if (con == nullptr) {
 4902       ShouldNotReachHere();
 4903     } else {
 4904       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4905     }
 4906   %}
 4907   ins_pipe(ialu_reg_fat); // XXX
 4908 %}
 4909 
 4910 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4911   match(Set dst src);
 4912 
 4913   ins_cost(125);
 4914   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4915   ins_encode %{
 4916     address con = (address)$src$$constant;
 4917     if (con == nullptr) {
 4918       ShouldNotReachHere();
 4919     } else {
 4920       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4921     }
 4922   %}
 4923   ins_pipe(ialu_reg_fat); // XXX
 4924 %}
 4925 
 4926 instruct loadConF0(regF dst, immF0 src)
 4927 %{
 4928   match(Set dst src);
 4929   ins_cost(100);
 4930 
 4931   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4932   ins_encode %{
 4933     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4934   %}
 4935   ins_pipe(pipe_slow);
 4936 %}
 4937 
 4938 // Use the same format since predicate() can not be used here.
 4939 instruct loadConD(regD dst, immD con) %{
 4940   match(Set dst con);
 4941   ins_cost(125);
 4942   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4943   ins_encode %{
 4944     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4945   %}
 4946   ins_pipe(pipe_slow);
 4947 %}
 4948 
 4949 instruct loadConD0(regD dst, immD0 src)
 4950 %{
 4951   match(Set dst src);
 4952   ins_cost(100);
 4953 
 4954   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4955   ins_encode %{
 4956     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4957   %}
 4958   ins_pipe(pipe_slow);
 4959 %}
 4960 
 4961 instruct loadSSI(rRegI dst, stackSlotI src)
 4962 %{
 4963   match(Set dst src);
 4964 
 4965   ins_cost(125);
 4966   format %{ "movl    $dst, $src\t# int stk" %}
 4967   ins_encode %{
 4968     __ movl($dst$$Register, $src$$Address);
 4969   %}
 4970   ins_pipe(ialu_reg_mem);
 4971 %}
 4972 
 4973 instruct loadSSL(rRegL dst, stackSlotL src)
 4974 %{
 4975   match(Set dst src);
 4976 
 4977   ins_cost(125);
 4978   format %{ "movq    $dst, $src\t# long stk" %}
 4979   ins_encode %{
 4980     __ movq($dst$$Register, $src$$Address);
 4981   %}
 4982   ins_pipe(ialu_reg_mem);
 4983 %}
 4984 
 4985 instruct loadSSP(rRegP dst, stackSlotP src)
 4986 %{
 4987   match(Set dst src);
 4988 
 4989   ins_cost(125);
 4990   format %{ "movq    $dst, $src\t# ptr stk" %}
 4991   ins_encode %{
 4992     __ movq($dst$$Register, $src$$Address);
 4993   %}
 4994   ins_pipe(ialu_reg_mem);
 4995 %}
 4996 
 4997 instruct loadSSF(regF dst, stackSlotF src)
 4998 %{
 4999   match(Set dst src);
 5000 
 5001   ins_cost(125);
 5002   format %{ "movss   $dst, $src\t# float stk" %}
 5003   ins_encode %{
 5004     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5005   %}
 5006   ins_pipe(pipe_slow); // XXX
 5007 %}
 5008 
 5009 // Use the same format since predicate() can not be used here.
 5010 instruct loadSSD(regD dst, stackSlotD src)
 5011 %{
 5012   match(Set dst src);
 5013 
 5014   ins_cost(125);
 5015   format %{ "movsd   $dst, $src\t# double stk" %}
 5016   ins_encode  %{
 5017     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5018   %}
 5019   ins_pipe(pipe_slow); // XXX
 5020 %}
 5021 
 5022 // Prefetch instructions for allocation.
 5023 // Must be safe to execute with invalid address (cannot fault).
 5024 
 5025 instruct prefetchAlloc( memory mem ) %{
 5026   predicate(AllocatePrefetchInstr==3);
 5027   match(PrefetchAllocation mem);
 5028   ins_cost(125);
 5029 
 5030   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5031   ins_encode %{
 5032     __ prefetchw($mem$$Address);
 5033   %}
 5034   ins_pipe(ialu_mem);
 5035 %}
 5036 
 5037 instruct prefetchAllocNTA( memory mem ) %{
 5038   predicate(AllocatePrefetchInstr==0);
 5039   match(PrefetchAllocation mem);
 5040   ins_cost(125);
 5041 
 5042   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5043   ins_encode %{
 5044     __ prefetchnta($mem$$Address);
 5045   %}
 5046   ins_pipe(ialu_mem);
 5047 %}
 5048 
 5049 instruct prefetchAllocT0( memory mem ) %{
 5050   predicate(AllocatePrefetchInstr==1);
 5051   match(PrefetchAllocation mem);
 5052   ins_cost(125);
 5053 
 5054   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5055   ins_encode %{
 5056     __ prefetcht0($mem$$Address);
 5057   %}
 5058   ins_pipe(ialu_mem);
 5059 %}
 5060 
 5061 instruct prefetchAllocT2( memory mem ) %{
 5062   predicate(AllocatePrefetchInstr==2);
 5063   match(PrefetchAllocation mem);
 5064   ins_cost(125);
 5065 
 5066   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5067   ins_encode %{
 5068     __ prefetcht2($mem$$Address);
 5069   %}
 5070   ins_pipe(ialu_mem);
 5071 %}
 5072 
 5073 //----------Store Instructions-------------------------------------------------
 5074 
 5075 // Store Byte
 5076 instruct storeB(memory mem, rRegI src)
 5077 %{
 5078   match(Set mem (StoreB mem src));
 5079 
 5080   ins_cost(125); // XXX
 5081   format %{ "movb    $mem, $src\t# byte" %}
 5082   ins_encode %{
 5083     __ movb($mem$$Address, $src$$Register);
 5084   %}
 5085   ins_pipe(ialu_mem_reg);
 5086 %}
 5087 
 5088 // Store Char/Short
 5089 instruct storeC(memory mem, rRegI src)
 5090 %{
 5091   match(Set mem (StoreC mem src));
 5092 
 5093   ins_cost(125); // XXX
 5094   format %{ "movw    $mem, $src\t# char/short" %}
 5095   ins_encode %{
 5096     __ movw($mem$$Address, $src$$Register);
 5097   %}
 5098   ins_pipe(ialu_mem_reg);
 5099 %}
 5100 
 5101 // Store Integer
 5102 instruct storeI(memory mem, rRegI src)
 5103 %{
 5104   match(Set mem (StoreI mem src));
 5105 
 5106   ins_cost(125); // XXX
 5107   format %{ "movl    $mem, $src\t# int" %}
 5108   ins_encode %{
 5109     __ movl($mem$$Address, $src$$Register);
 5110   %}
 5111   ins_pipe(ialu_mem_reg);
 5112 %}
 5113 
 5114 // Store Long
 5115 instruct storeL(memory mem, rRegL src)
 5116 %{
 5117   match(Set mem (StoreL mem src));
 5118 
 5119   ins_cost(125); // XXX
 5120   format %{ "movq    $mem, $src\t# long" %}
 5121   ins_encode %{
 5122     __ movq($mem$$Address, $src$$Register);
 5123   %}
 5124   ins_pipe(ialu_mem_reg); // XXX
 5125 %}
 5126 
 5127 // Store Pointer
 5128 instruct storeP(memory mem, any_RegP src)
 5129 %{
 5130   predicate(n->as_Store()->barrier_data() == 0);
 5131   match(Set mem (StoreP mem src));
 5132 
 5133   ins_cost(125); // XXX
 5134   format %{ "movq    $mem, $src\t# ptr" %}
 5135   ins_encode %{
 5136     __ movq($mem$$Address, $src$$Register);
 5137   %}
 5138   ins_pipe(ialu_mem_reg);
 5139 %}
 5140 
 5141 instruct storeImmP0(memory mem, immP0 zero)
 5142 %{
 5143   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5144   match(Set mem (StoreP mem zero));
 5145 
 5146   ins_cost(125); // XXX
 5147   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5148   ins_encode %{
 5149     __ movq($mem$$Address, r12);
 5150   %}
 5151   ins_pipe(ialu_mem_reg);
 5152 %}
 5153 
 5154 // Store Null Pointer, mark word, or other simple pointer constant.
 5155 instruct storeImmP(memory mem, immP31 src)
 5156 %{
 5157   predicate(n->as_Store()->barrier_data() == 0);
 5158   match(Set mem (StoreP mem src));
 5159 
 5160   ins_cost(150); // XXX
 5161   format %{ "movq    $mem, $src\t# ptr" %}
 5162   ins_encode %{
 5163     __ movq($mem$$Address, $src$$constant);
 5164   %}
 5165   ins_pipe(ialu_mem_imm);
 5166 %}
 5167 
 5168 // Store Compressed Pointer
 5169 instruct storeN(memory mem, rRegN src)
 5170 %{
 5171   predicate(n->as_Store()->barrier_data() == 0);
 5172   match(Set mem (StoreN mem src));
 5173 
 5174   ins_cost(125); // XXX
 5175   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5176   ins_encode %{
 5177     __ movl($mem$$Address, $src$$Register);
 5178   %}
 5179   ins_pipe(ialu_mem_reg);
 5180 %}
 5181 
 5182 instruct storeNKlass(memory mem, rRegN src)
 5183 %{
 5184   match(Set mem (StoreNKlass mem src));
 5185 
 5186   ins_cost(125); // XXX
 5187   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5188   ins_encode %{
 5189     __ movl($mem$$Address, $src$$Register);
 5190   %}
 5191   ins_pipe(ialu_mem_reg);
 5192 %}
 5193 
 5194 instruct storeImmN0(memory mem, immN0 zero)
 5195 %{
 5196   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5197   match(Set mem (StoreN mem zero));
 5198 
 5199   ins_cost(125); // XXX
 5200   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5201   ins_encode %{
 5202     __ movl($mem$$Address, r12);
 5203   %}
 5204   ins_pipe(ialu_mem_reg);
 5205 %}
 5206 
 5207 instruct storeImmN(memory mem, immN src)
 5208 %{
 5209   predicate(n->as_Store()->barrier_data() == 0);
 5210   match(Set mem (StoreN mem src));
 5211 
 5212   ins_cost(150); // XXX
 5213   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5214   ins_encode %{
 5215     address con = (address)$src$$constant;
 5216     if (con == nullptr) {
 5217       __ movl($mem$$Address, 0);
 5218     } else {
 5219       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5220     }
 5221   %}
 5222   ins_pipe(ialu_mem_imm);
 5223 %}
 5224 
 5225 instruct storeImmNKlass(memory mem, immNKlass src)
 5226 %{
 5227   match(Set mem (StoreNKlass mem src));
 5228 
 5229   ins_cost(150); // XXX
 5230   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5231   ins_encode %{
 5232     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5233   %}
 5234   ins_pipe(ialu_mem_imm);
 5235 %}
 5236 
 5237 // Store Integer Immediate
 5238 instruct storeImmI0(memory mem, immI_0 zero)
 5239 %{
 5240   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5241   match(Set mem (StoreI mem zero));
 5242 
 5243   ins_cost(125); // XXX
 5244   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5245   ins_encode %{
 5246     __ movl($mem$$Address, r12);
 5247   %}
 5248   ins_pipe(ialu_mem_reg);
 5249 %}
 5250 
 5251 instruct storeImmI(memory mem, immI src)
 5252 %{
 5253   match(Set mem (StoreI mem src));
 5254 
 5255   ins_cost(150);
 5256   format %{ "movl    $mem, $src\t# int" %}
 5257   ins_encode %{
 5258     __ movl($mem$$Address, $src$$constant);
 5259   %}
 5260   ins_pipe(ialu_mem_imm);
 5261 %}
 5262 
 5263 // Store Long Immediate
 5264 instruct storeImmL0(memory mem, immL0 zero)
 5265 %{
 5266   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5267   match(Set mem (StoreL mem zero));
 5268 
 5269   ins_cost(125); // XXX
 5270   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5271   ins_encode %{
 5272     __ movq($mem$$Address, r12);
 5273   %}
 5274   ins_pipe(ialu_mem_reg);
 5275 %}
 5276 
 5277 instruct storeImmL(memory mem, immL32 src)
 5278 %{
 5279   match(Set mem (StoreL mem src));
 5280 
 5281   ins_cost(150);
 5282   format %{ "movq    $mem, $src\t# long" %}
 5283   ins_encode %{
 5284     __ movq($mem$$Address, $src$$constant);
 5285   %}
 5286   ins_pipe(ialu_mem_imm);
 5287 %}
 5288 
 5289 // Store Short/Char Immediate
 5290 instruct storeImmC0(memory mem, immI_0 zero)
 5291 %{
 5292   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5293   match(Set mem (StoreC mem zero));
 5294 
 5295   ins_cost(125); // XXX
 5296   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5297   ins_encode %{
 5298     __ movw($mem$$Address, r12);
 5299   %}
 5300   ins_pipe(ialu_mem_reg);
 5301 %}
 5302 
 5303 instruct storeImmI16(memory mem, immI16 src)
 5304 %{
 5305   predicate(UseStoreImmI16);
 5306   match(Set mem (StoreC mem src));
 5307 
 5308   ins_cost(150);
 5309   format %{ "movw    $mem, $src\t# short/char" %}
 5310   ins_encode %{
 5311     __ movw($mem$$Address, $src$$constant);
 5312   %}
 5313   ins_pipe(ialu_mem_imm);
 5314 %}
 5315 
 5316 // Store Byte Immediate
 5317 instruct storeImmB0(memory mem, immI_0 zero)
 5318 %{
 5319   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5320   match(Set mem (StoreB mem zero));
 5321 
 5322   ins_cost(125); // XXX
 5323   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5324   ins_encode %{
 5325     __ movb($mem$$Address, r12);
 5326   %}
 5327   ins_pipe(ialu_mem_reg);
 5328 %}
 5329 
 5330 instruct storeImmB(memory mem, immI8 src)
 5331 %{
 5332   match(Set mem (StoreB mem src));
 5333 
 5334   ins_cost(150); // XXX
 5335   format %{ "movb    $mem, $src\t# byte" %}
 5336   ins_encode %{
 5337     __ movb($mem$$Address, $src$$constant);
 5338   %}
 5339   ins_pipe(ialu_mem_imm);
 5340 %}
 5341 
 5342 // Store Float
 5343 instruct storeF(memory mem, regF src)
 5344 %{
 5345   match(Set mem (StoreF mem src));
 5346 
 5347   ins_cost(95); // XXX
 5348   format %{ "movss   $mem, $src\t# float" %}
 5349   ins_encode %{
 5350     __ movflt($mem$$Address, $src$$XMMRegister);
 5351   %}
 5352   ins_pipe(pipe_slow); // XXX
 5353 %}
 5354 
 5355 // Store immediate Float value (it is faster than store from XMM register)
 5356 instruct storeF0(memory mem, immF0 zero)
 5357 %{
 5358   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5359   match(Set mem (StoreF mem zero));
 5360 
 5361   ins_cost(25); // XXX
 5362   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5363   ins_encode %{
 5364     __ movl($mem$$Address, r12);
 5365   %}
 5366   ins_pipe(ialu_mem_reg);
 5367 %}
 5368 
 5369 instruct storeF_imm(memory mem, immF src)
 5370 %{
 5371   match(Set mem (StoreF mem src));
 5372 
 5373   ins_cost(50);
 5374   format %{ "movl    $mem, $src\t# float" %}
 5375   ins_encode %{
 5376     __ movl($mem$$Address, jint_cast($src$$constant));
 5377   %}
 5378   ins_pipe(ialu_mem_imm);
 5379 %}
 5380 
 5381 // Store Double
 5382 instruct storeD(memory mem, regD src)
 5383 %{
 5384   match(Set mem (StoreD mem src));
 5385 
 5386   ins_cost(95); // XXX
 5387   format %{ "movsd   $mem, $src\t# double" %}
 5388   ins_encode %{
 5389     __ movdbl($mem$$Address, $src$$XMMRegister);
 5390   %}
 5391   ins_pipe(pipe_slow); // XXX
 5392 %}
 5393 
 5394 // Store immediate double 0.0 (it is faster than store from XMM register)
 5395 instruct storeD0_imm(memory mem, immD0 src)
 5396 %{
 5397   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5398   match(Set mem (StoreD mem src));
 5399 
 5400   ins_cost(50);
 5401   format %{ "movq    $mem, $src\t# double 0." %}
 5402   ins_encode %{
 5403     __ movq($mem$$Address, $src$$constant);
 5404   %}
 5405   ins_pipe(ialu_mem_imm);
 5406 %}
 5407 
 5408 instruct storeD0(memory mem, immD0 zero)
 5409 %{
 5410   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5411   match(Set mem (StoreD mem zero));
 5412 
 5413   ins_cost(25); // XXX
 5414   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5415   ins_encode %{
 5416     __ movq($mem$$Address, r12);
 5417   %}
 5418   ins_pipe(ialu_mem_reg);
 5419 %}
 5420 
 5421 instruct storeSSI(stackSlotI dst, rRegI src)
 5422 %{
 5423   match(Set dst src);
 5424 
 5425   ins_cost(100);
 5426   format %{ "movl    $dst, $src\t# int stk" %}
 5427   ins_encode %{
 5428     __ movl($dst$$Address, $src$$Register);
 5429   %}
 5430   ins_pipe( ialu_mem_reg );
 5431 %}
 5432 
 5433 instruct storeSSL(stackSlotL dst, rRegL src)
 5434 %{
 5435   match(Set dst src);
 5436 
 5437   ins_cost(100);
 5438   format %{ "movq    $dst, $src\t# long stk" %}
 5439   ins_encode %{
 5440     __ movq($dst$$Address, $src$$Register);
 5441   %}
 5442   ins_pipe(ialu_mem_reg);
 5443 %}
 5444 
 5445 instruct storeSSP(stackSlotP dst, rRegP src)
 5446 %{
 5447   match(Set dst src);
 5448 
 5449   ins_cost(100);
 5450   format %{ "movq    $dst, $src\t# ptr stk" %}
 5451   ins_encode %{
 5452     __ movq($dst$$Address, $src$$Register);
 5453   %}
 5454   ins_pipe(ialu_mem_reg);
 5455 %}
 5456 
 5457 instruct storeSSF(stackSlotF dst, regF src)
 5458 %{
 5459   match(Set dst src);
 5460 
 5461   ins_cost(95); // XXX
 5462   format %{ "movss   $dst, $src\t# float stk" %}
 5463   ins_encode %{
 5464     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5465   %}
 5466   ins_pipe(pipe_slow); // XXX
 5467 %}
 5468 
 5469 instruct storeSSD(stackSlotD dst, regD src)
 5470 %{
 5471   match(Set dst src);
 5472 
 5473   ins_cost(95); // XXX
 5474   format %{ "movsd   $dst, $src\t# double stk" %}
 5475   ins_encode %{
 5476     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5477   %}
 5478   ins_pipe(pipe_slow); // XXX
 5479 %}
 5480 
 5481 instruct cacheWB(indirect addr)
 5482 %{
 5483   predicate(VM_Version::supports_data_cache_line_flush());
 5484   match(CacheWB addr);
 5485 
 5486   ins_cost(100);
 5487   format %{"cache wb $addr" %}
 5488   ins_encode %{
 5489     assert($addr->index_position() < 0, "should be");
 5490     assert($addr$$disp == 0, "should be");
 5491     __ cache_wb(Address($addr$$base$$Register, 0));
 5492   %}
 5493   ins_pipe(pipe_slow); // XXX
 5494 %}
 5495 
 5496 instruct cacheWBPreSync()
 5497 %{
 5498   predicate(VM_Version::supports_data_cache_line_flush());
 5499   match(CacheWBPreSync);
 5500 
 5501   ins_cost(100);
 5502   format %{"cache wb presync" %}
 5503   ins_encode %{
 5504     __ cache_wbsync(true);
 5505   %}
 5506   ins_pipe(pipe_slow); // XXX
 5507 %}
 5508 
 5509 instruct cacheWBPostSync()
 5510 %{
 5511   predicate(VM_Version::supports_data_cache_line_flush());
 5512   match(CacheWBPostSync);
 5513 
 5514   ins_cost(100);
 5515   format %{"cache wb postsync" %}
 5516   ins_encode %{
 5517     __ cache_wbsync(false);
 5518   %}
 5519   ins_pipe(pipe_slow); // XXX
 5520 %}
 5521 
 5522 //----------BSWAP Instructions-------------------------------------------------
 5523 instruct bytes_reverse_int(rRegI dst) %{
 5524   match(Set dst (ReverseBytesI dst));
 5525 
 5526   format %{ "bswapl  $dst" %}
 5527   ins_encode %{
 5528     __ bswapl($dst$$Register);
 5529   %}
 5530   ins_pipe( ialu_reg );
 5531 %}
 5532 
 5533 instruct bytes_reverse_long(rRegL dst) %{
 5534   match(Set dst (ReverseBytesL dst));
 5535 
 5536   format %{ "bswapq  $dst" %}
 5537   ins_encode %{
 5538     __ bswapq($dst$$Register);
 5539   %}
 5540   ins_pipe( ialu_reg);
 5541 %}
 5542 
 5543 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5544   match(Set dst (ReverseBytesUS dst));
 5545   effect(KILL cr);
 5546 
 5547   format %{ "bswapl  $dst\n\t"
 5548             "shrl    $dst,16\n\t" %}
 5549   ins_encode %{
 5550     __ bswapl($dst$$Register);
 5551     __ shrl($dst$$Register, 16);
 5552   %}
 5553   ins_pipe( ialu_reg );
 5554 %}
 5555 
 5556 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5557   match(Set dst (ReverseBytesS dst));
 5558   effect(KILL cr);
 5559 
 5560   format %{ "bswapl  $dst\n\t"
 5561             "sar     $dst,16\n\t" %}
 5562   ins_encode %{
 5563     __ bswapl($dst$$Register);
 5564     __ sarl($dst$$Register, 16);
 5565   %}
 5566   ins_pipe( ialu_reg );
 5567 %}
 5568 
 5569 //---------- Zeros Count Instructions ------------------------------------------
 5570 
 5571 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5572   predicate(UseCountLeadingZerosInstruction);
 5573   match(Set dst (CountLeadingZerosI src));
 5574   effect(KILL cr);
 5575 
 5576   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5577   ins_encode %{
 5578     __ lzcntl($dst$$Register, $src$$Register);
 5579   %}
 5580   ins_pipe(ialu_reg);
 5581 %}
 5582 
 5583 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5584   predicate(UseCountLeadingZerosInstruction);
 5585   match(Set dst (CountLeadingZerosI (LoadI src)));
 5586   effect(KILL cr);
 5587   ins_cost(175);
 5588   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5589   ins_encode %{
 5590     __ lzcntl($dst$$Register, $src$$Address);
 5591   %}
 5592   ins_pipe(ialu_reg_mem);
 5593 %}
 5594 
 5595 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5596   predicate(!UseCountLeadingZerosInstruction);
 5597   match(Set dst (CountLeadingZerosI src));
 5598   effect(KILL cr);
 5599 
 5600   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5601             "jnz     skip\n\t"
 5602             "movl    $dst, -1\n"
 5603       "skip:\n\t"
 5604             "negl    $dst\n\t"
 5605             "addl    $dst, 31" %}
 5606   ins_encode %{
 5607     Register Rdst = $dst$$Register;
 5608     Register Rsrc = $src$$Register;
 5609     Label skip;
 5610     __ bsrl(Rdst, Rsrc);
 5611     __ jccb(Assembler::notZero, skip);
 5612     __ movl(Rdst, -1);
 5613     __ bind(skip);
 5614     __ negl(Rdst);
 5615     __ addl(Rdst, BitsPerInt - 1);
 5616   %}
 5617   ins_pipe(ialu_reg);
 5618 %}
 5619 
 5620 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5621   predicate(UseCountLeadingZerosInstruction);
 5622   match(Set dst (CountLeadingZerosL src));
 5623   effect(KILL cr);
 5624 
 5625   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5626   ins_encode %{
 5627     __ lzcntq($dst$$Register, $src$$Register);
 5628   %}
 5629   ins_pipe(ialu_reg);
 5630 %}
 5631 
 5632 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5633   predicate(UseCountLeadingZerosInstruction);
 5634   match(Set dst (CountLeadingZerosL (LoadL src)));
 5635   effect(KILL cr);
 5636   ins_cost(175);
 5637   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5638   ins_encode %{
 5639     __ lzcntq($dst$$Register, $src$$Address);
 5640   %}
 5641   ins_pipe(ialu_reg_mem);
 5642 %}
 5643 
 5644 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5645   predicate(!UseCountLeadingZerosInstruction);
 5646   match(Set dst (CountLeadingZerosL src));
 5647   effect(KILL cr);
 5648 
 5649   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5650             "jnz     skip\n\t"
 5651             "movl    $dst, -1\n"
 5652       "skip:\n\t"
 5653             "negl    $dst\n\t"
 5654             "addl    $dst, 63" %}
 5655   ins_encode %{
 5656     Register Rdst = $dst$$Register;
 5657     Register Rsrc = $src$$Register;
 5658     Label skip;
 5659     __ bsrq(Rdst, Rsrc);
 5660     __ jccb(Assembler::notZero, skip);
 5661     __ movl(Rdst, -1);
 5662     __ bind(skip);
 5663     __ negl(Rdst);
 5664     __ addl(Rdst, BitsPerLong - 1);
 5665   %}
 5666   ins_pipe(ialu_reg);
 5667 %}
 5668 
 5669 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5670   predicate(UseCountTrailingZerosInstruction);
 5671   match(Set dst (CountTrailingZerosI src));
 5672   effect(KILL cr);
 5673 
 5674   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5675   ins_encode %{
 5676     __ tzcntl($dst$$Register, $src$$Register);
 5677   %}
 5678   ins_pipe(ialu_reg);
 5679 %}
 5680 
 5681 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5682   predicate(UseCountTrailingZerosInstruction);
 5683   match(Set dst (CountTrailingZerosI (LoadI src)));
 5684   effect(KILL cr);
 5685   ins_cost(175);
 5686   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5687   ins_encode %{
 5688     __ tzcntl($dst$$Register, $src$$Address);
 5689   %}
 5690   ins_pipe(ialu_reg_mem);
 5691 %}
 5692 
 5693 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5694   predicate(!UseCountTrailingZerosInstruction);
 5695   match(Set dst (CountTrailingZerosI src));
 5696   effect(KILL cr);
 5697 
 5698   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5699             "jnz     done\n\t"
 5700             "movl    $dst, 32\n"
 5701       "done:" %}
 5702   ins_encode %{
 5703     Register Rdst = $dst$$Register;
 5704     Label done;
 5705     __ bsfl(Rdst, $src$$Register);
 5706     __ jccb(Assembler::notZero, done);
 5707     __ movl(Rdst, BitsPerInt);
 5708     __ bind(done);
 5709   %}
 5710   ins_pipe(ialu_reg);
 5711 %}
 5712 
 5713 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5714   predicate(UseCountTrailingZerosInstruction);
 5715   match(Set dst (CountTrailingZerosL src));
 5716   effect(KILL cr);
 5717 
 5718   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5719   ins_encode %{
 5720     __ tzcntq($dst$$Register, $src$$Register);
 5721   %}
 5722   ins_pipe(ialu_reg);
 5723 %}
 5724 
 5725 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5726   predicate(UseCountTrailingZerosInstruction);
 5727   match(Set dst (CountTrailingZerosL (LoadL src)));
 5728   effect(KILL cr);
 5729   ins_cost(175);
 5730   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5731   ins_encode %{
 5732     __ tzcntq($dst$$Register, $src$$Address);
 5733   %}
 5734   ins_pipe(ialu_reg_mem);
 5735 %}
 5736 
 5737 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5738   predicate(!UseCountTrailingZerosInstruction);
 5739   match(Set dst (CountTrailingZerosL src));
 5740   effect(KILL cr);
 5741 
 5742   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5743             "jnz     done\n\t"
 5744             "movl    $dst, 64\n"
 5745       "done:" %}
 5746   ins_encode %{
 5747     Register Rdst = $dst$$Register;
 5748     Label done;
 5749     __ bsfq(Rdst, $src$$Register);
 5750     __ jccb(Assembler::notZero, done);
 5751     __ movl(Rdst, BitsPerLong);
 5752     __ bind(done);
 5753   %}
 5754   ins_pipe(ialu_reg);
 5755 %}
 5756 
 5757 //--------------- Reverse Operation Instructions ----------------
 5758 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5759   predicate(!VM_Version::supports_gfni());
 5760   match(Set dst (ReverseI src));
 5761   effect(TEMP dst, TEMP rtmp, KILL cr);
 5762   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5763   ins_encode %{
 5764     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5765   %}
 5766   ins_pipe( ialu_reg );
 5767 %}
 5768 
 5769 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5770   predicate(VM_Version::supports_gfni());
 5771   match(Set dst (ReverseI src));
 5772   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5773   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5774   ins_encode %{
 5775     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5776   %}
 5777   ins_pipe( ialu_reg );
 5778 %}
 5779 
 5780 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5781   predicate(!VM_Version::supports_gfni());
 5782   match(Set dst (ReverseL src));
 5783   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5784   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5785   ins_encode %{
 5786     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5787   %}
 5788   ins_pipe( ialu_reg );
 5789 %}
 5790 
 5791 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5792   predicate(VM_Version::supports_gfni());
 5793   match(Set dst (ReverseL src));
 5794   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5795   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5796   ins_encode %{
 5797     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5798   %}
 5799   ins_pipe( ialu_reg );
 5800 %}
 5801 
 5802 //---------- Population Count Instructions -------------------------------------
 5803 
 5804 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5805   predicate(UsePopCountInstruction);
 5806   match(Set dst (PopCountI src));
 5807   effect(KILL cr);
 5808 
 5809   format %{ "popcnt  $dst, $src" %}
 5810   ins_encode %{
 5811     __ popcntl($dst$$Register, $src$$Register);
 5812   %}
 5813   ins_pipe(ialu_reg);
 5814 %}
 5815 
 5816 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5817   predicate(UsePopCountInstruction);
 5818   match(Set dst (PopCountI (LoadI mem)));
 5819   effect(KILL cr);
 5820 
 5821   format %{ "popcnt  $dst, $mem" %}
 5822   ins_encode %{
 5823     __ popcntl($dst$$Register, $mem$$Address);
 5824   %}
 5825   ins_pipe(ialu_reg);
 5826 %}
 5827 
 5828 // Note: Long.bitCount(long) returns an int.
 5829 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5830   predicate(UsePopCountInstruction);
 5831   match(Set dst (PopCountL src));
 5832   effect(KILL cr);
 5833 
 5834   format %{ "popcnt  $dst, $src" %}
 5835   ins_encode %{
 5836     __ popcntq($dst$$Register, $src$$Register);
 5837   %}
 5838   ins_pipe(ialu_reg);
 5839 %}
 5840 
 5841 // Note: Long.bitCount(long) returns an int.
 5842 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5843   predicate(UsePopCountInstruction);
 5844   match(Set dst (PopCountL (LoadL mem)));
 5845   effect(KILL cr);
 5846 
 5847   format %{ "popcnt  $dst, $mem" %}
 5848   ins_encode %{
 5849     __ popcntq($dst$$Register, $mem$$Address);
 5850   %}
 5851   ins_pipe(ialu_reg);
 5852 %}
 5853 
 5854 
 5855 //----------MemBar Instructions-----------------------------------------------
 5856 // Memory barrier flavors
 5857 
 5858 instruct membar_acquire()
 5859 %{
 5860   match(MemBarAcquire);
 5861   match(LoadFence);
 5862   ins_cost(0);
 5863 
 5864   size(0);
 5865   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5866   ins_encode();
 5867   ins_pipe(empty);
 5868 %}
 5869 
 5870 instruct membar_acquire_lock()
 5871 %{
 5872   match(MemBarAcquireLock);
 5873   ins_cost(0);
 5874 
 5875   size(0);
 5876   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5877   ins_encode();
 5878   ins_pipe(empty);
 5879 %}
 5880 
 5881 instruct membar_release()
 5882 %{
 5883   match(MemBarRelease);
 5884   match(StoreFence);
 5885   ins_cost(0);
 5886 
 5887   size(0);
 5888   format %{ "MEMBAR-release ! (empty encoding)" %}
 5889   ins_encode();
 5890   ins_pipe(empty);
 5891 %}
 5892 
 5893 instruct membar_release_lock()
 5894 %{
 5895   match(MemBarReleaseLock);
 5896   ins_cost(0);
 5897 
 5898   size(0);
 5899   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5900   ins_encode();
 5901   ins_pipe(empty);
 5902 %}
 5903 
 5904 instruct membar_volatile(rFlagsReg cr) %{
 5905   match(MemBarVolatile);
 5906   effect(KILL cr);
 5907   ins_cost(400);
 5908 
 5909   format %{
 5910     $$template
 5911     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5912   %}
 5913   ins_encode %{
 5914     __ membar(Assembler::StoreLoad);
 5915   %}
 5916   ins_pipe(pipe_slow);
 5917 %}
 5918 
 5919 instruct unnecessary_membar_volatile()
 5920 %{
 5921   match(MemBarVolatile);
 5922   predicate(Matcher::post_store_load_barrier(n));
 5923   ins_cost(0);
 5924 
 5925   size(0);
 5926   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5927   ins_encode();
 5928   ins_pipe(empty);
 5929 %}
 5930 
 5931 instruct membar_storestore() %{
 5932   match(MemBarStoreStore);
 5933   match(StoreStoreFence);
 5934   ins_cost(0);
 5935 
 5936   size(0);
 5937   format %{ "MEMBAR-storestore (empty encoding)" %}
 5938   ins_encode( );
 5939   ins_pipe(empty);
 5940 %}
 5941 
 5942 //----------Move Instructions--------------------------------------------------
 5943 
 5944 instruct castX2P(rRegP dst, rRegL src)
 5945 %{
 5946   match(Set dst (CastX2P src));
 5947 
 5948   format %{ "movq    $dst, $src\t# long->ptr" %}
 5949   ins_encode %{
 5950     if ($dst$$reg != $src$$reg) {
 5951       __ movptr($dst$$Register, $src$$Register);
 5952     }
 5953   %}
 5954   ins_pipe(ialu_reg_reg); // XXX
 5955 %}
 5956 
 5957 instruct castP2X(rRegL dst, rRegP src)
 5958 %{
 5959   match(Set dst (CastP2X src));
 5960 
 5961   format %{ "movq    $dst, $src\t# ptr -> long" %}
 5962   ins_encode %{
 5963     if ($dst$$reg != $src$$reg) {
 5964       __ movptr($dst$$Register, $src$$Register);
 5965     }
 5966   %}
 5967   ins_pipe(ialu_reg_reg); // XXX
 5968 %}
 5969 
 5970 // Convert oop into int for vectors alignment masking
 5971 instruct convP2I(rRegI dst, rRegP src)
 5972 %{
 5973   match(Set dst (ConvL2I (CastP2X src)));
 5974 
 5975   format %{ "movl    $dst, $src\t# ptr -> int" %}
 5976   ins_encode %{
 5977     __ movl($dst$$Register, $src$$Register);
 5978   %}
 5979   ins_pipe(ialu_reg_reg); // XXX
 5980 %}
 5981 
 5982 // Convert compressed oop into int for vectors alignment masking
 5983 // in case of 32bit oops (heap < 4Gb).
 5984 instruct convN2I(rRegI dst, rRegN src)
 5985 %{
 5986   predicate(CompressedOops::shift() == 0);
 5987   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 5988 
 5989   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 5990   ins_encode %{
 5991     __ movl($dst$$Register, $src$$Register);
 5992   %}
 5993   ins_pipe(ialu_reg_reg); // XXX
 5994 %}
 5995 
 5996 // Convert oop pointer into compressed form
 5997 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 5998   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 5999   match(Set dst (EncodeP src));
 6000   effect(KILL cr);
 6001   format %{ "encode_heap_oop $dst,$src" %}
 6002   ins_encode %{
 6003     Register s = $src$$Register;
 6004     Register d = $dst$$Register;
 6005     if (s != d) {
 6006       __ movq(d, s);
 6007     }
 6008     __ encode_heap_oop(d);
 6009   %}
 6010   ins_pipe(ialu_reg_long);
 6011 %}
 6012 
 6013 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6014   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6015   match(Set dst (EncodeP src));
 6016   effect(KILL cr);
 6017   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6018   ins_encode %{
 6019     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6020   %}
 6021   ins_pipe(ialu_reg_long);
 6022 %}
 6023 
 6024 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6025   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6026             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6027   match(Set dst (DecodeN src));
 6028   effect(KILL cr);
 6029   format %{ "decode_heap_oop $dst,$src" %}
 6030   ins_encode %{
 6031     Register s = $src$$Register;
 6032     Register d = $dst$$Register;
 6033     if (s != d) {
 6034       __ movq(d, s);
 6035     }
 6036     __ decode_heap_oop(d);
 6037   %}
 6038   ins_pipe(ialu_reg_long);
 6039 %}
 6040 
 6041 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6042   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6043             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6044   match(Set dst (DecodeN src));
 6045   effect(KILL cr);
 6046   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6047   ins_encode %{
 6048     Register s = $src$$Register;
 6049     Register d = $dst$$Register;
 6050     if (s != d) {
 6051       __ decode_heap_oop_not_null(d, s);
 6052     } else {
 6053       __ decode_heap_oop_not_null(d);
 6054     }
 6055   %}
 6056   ins_pipe(ialu_reg_long);
 6057 %}
 6058 
 6059 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6060   match(Set dst (EncodePKlass src));
 6061   effect(TEMP dst, KILL cr);
 6062   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6063   ins_encode %{
 6064     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6065   %}
 6066   ins_pipe(ialu_reg_long);
 6067 %}
 6068 
 6069 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6070   match(Set dst (DecodeNKlass src));
 6071   effect(TEMP dst, KILL cr);
 6072   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6073   ins_encode %{
 6074     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6075   %}
 6076   ins_pipe(ialu_reg_long);
 6077 %}
 6078 
 6079 //----------Conditional Move---------------------------------------------------
 6080 // Jump
 6081 // dummy instruction for generating temp registers
 6082 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6083   match(Jump (LShiftL switch_val shift));
 6084   ins_cost(350);
 6085   predicate(false);
 6086   effect(TEMP dest);
 6087 
 6088   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6089             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6090   ins_encode %{
 6091     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6092     // to do that and the compiler is using that register as one it can allocate.
 6093     // So we build it all by hand.
 6094     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6095     // ArrayAddress dispatch(table, index);
 6096     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6097     __ lea($dest$$Register, $constantaddress);
 6098     __ jmp(dispatch);
 6099   %}
 6100   ins_pipe(pipe_jmp);
 6101 %}
 6102 
 6103 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6104   match(Jump (AddL (LShiftL switch_val shift) offset));
 6105   ins_cost(350);
 6106   effect(TEMP dest);
 6107 
 6108   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6109             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6110   ins_encode %{
 6111     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6112     // to do that and the compiler is using that register as one it can allocate.
 6113     // So we build it all by hand.
 6114     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6115     // ArrayAddress dispatch(table, index);
 6116     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6117     __ lea($dest$$Register, $constantaddress);
 6118     __ jmp(dispatch);
 6119   %}
 6120   ins_pipe(pipe_jmp);
 6121 %}
 6122 
 6123 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6124   match(Jump switch_val);
 6125   ins_cost(350);
 6126   effect(TEMP dest);
 6127 
 6128   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6129             "jmp     [$dest + $switch_val]\n\t" %}
 6130   ins_encode %{
 6131     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6132     // to do that and the compiler is using that register as one it can allocate.
 6133     // So we build it all by hand.
 6134     // Address index(noreg, switch_reg, Address::times_1);
 6135     // ArrayAddress dispatch(table, index);
 6136     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6137     __ lea($dest$$Register, $constantaddress);
 6138     __ jmp(dispatch);
 6139   %}
 6140   ins_pipe(pipe_jmp);
 6141 %}
 6142 
 6143 // Conditional move
 6144 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6145 %{
 6146   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6147   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6148 
 6149   ins_cost(100); // XXX
 6150   format %{ "setbn$cop $dst\t# signed, int" %}
 6151   ins_encode %{
 6152     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6153     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6154   %}
 6155   ins_pipe(ialu_reg);
 6156 %}
 6157 
 6158 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6159 %{
 6160   predicate(!UseAPX);
 6161   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6162 
 6163   ins_cost(200); // XXX
 6164   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6165   ins_encode %{
 6166     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6167   %}
 6168   ins_pipe(pipe_cmov_reg);
 6169 %}
 6170 
 6171 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6172 %{
 6173   predicate(UseAPX);
 6174   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6175 
 6176   ins_cost(200);
 6177   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6178   ins_encode %{
 6179     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6180   %}
 6181   ins_pipe(pipe_cmov_reg);
 6182 %}
 6183 
 6184 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6185 %{
 6186   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6187   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6188 
 6189   ins_cost(100); // XXX
 6190   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6191   ins_encode %{
 6192     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6193     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6194   %}
 6195   ins_pipe(ialu_reg);
 6196 %}
 6197 
 6198 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6199   predicate(!UseAPX);
 6200   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6201 
 6202   ins_cost(200); // XXX
 6203   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6204   ins_encode %{
 6205     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6206   %}
 6207   ins_pipe(pipe_cmov_reg);
 6208 %}
 6209 
 6210 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6211   predicate(UseAPX);
 6212   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6213 
 6214   ins_cost(200);
 6215   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6216   ins_encode %{
 6217     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6218   %}
 6219   ins_pipe(pipe_cmov_reg);
 6220 %}
 6221 
 6222 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6223 %{
 6224   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6225   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6226 
 6227   ins_cost(100); // XXX
 6228   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6229   ins_encode %{
 6230     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6231     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6232   %}
 6233   ins_pipe(ialu_reg);
 6234 %}
 6235 
 6236 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6237   predicate(!UseAPX);
 6238   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6239   ins_cost(200);
 6240   expand %{
 6241     cmovI_regU(cop, cr, dst, src);
 6242   %}
 6243 %}
 6244 
 6245 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6246   predicate(UseAPX);
 6247   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6248   ins_cost(200);
 6249   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6250   ins_encode %{
 6251     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6252   %}
 6253   ins_pipe(pipe_cmov_reg);
 6254 %}
 6255 
 6256 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6257   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6258   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6259 
 6260   ins_cost(200); // XXX
 6261   format %{ "cmovpl  $dst, $src\n\t"
 6262             "cmovnel $dst, $src" %}
 6263   ins_encode %{
 6264     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6265     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6266   %}
 6267   ins_pipe(pipe_cmov_reg);
 6268 %}
 6269 
 6270 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6271   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6272   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6273   effect(TEMP dst);
 6274 
 6275   ins_cost(200);
 6276   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6277             "cmovnel  $dst, $src2" %}
 6278   ins_encode %{
 6279     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6280     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6281   %}
 6282   ins_pipe(pipe_cmov_reg);
 6283 %}
 6284 
 6285 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6286 // inputs of the CMove
 6287 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6288   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6289   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6290   effect(TEMP dst);
 6291 
 6292   ins_cost(200); // XXX
 6293   format %{ "cmovpl  $dst, $src\n\t"
 6294             "cmovnel $dst, $src" %}
 6295   ins_encode %{
 6296     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6297     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6298   %}
 6299   ins_pipe(pipe_cmov_reg);
 6300 %}
 6301 
 6302 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6303 // and parity flag bit is set if any of the operand is a NaN.
 6304 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6305   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6306   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6307   effect(TEMP dst);
 6308 
 6309   ins_cost(200);
 6310   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6311             "cmovnel  $dst, $src2" %}
 6312   ins_encode %{
 6313     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6314     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6315   %}
 6316   ins_pipe(pipe_cmov_reg);
 6317 %}
 6318 
 6319 // Conditional move
 6320 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6321   predicate(!UseAPX);
 6322   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6323 
 6324   ins_cost(250); // XXX
 6325   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6326   ins_encode %{
 6327     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6328   %}
 6329   ins_pipe(pipe_cmov_mem);
 6330 %}
 6331 
 6332 // Conditional move
 6333 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6334 %{
 6335   predicate(UseAPX);
 6336   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6337 
 6338   ins_cost(250);
 6339   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6340   ins_encode %{
 6341     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6342   %}
 6343   ins_pipe(pipe_cmov_mem);
 6344 %}
 6345 
 6346 // Conditional move
 6347 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6348 %{
 6349   predicate(!UseAPX);
 6350   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6351 
 6352   ins_cost(250); // XXX
 6353   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6354   ins_encode %{
 6355     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6356   %}
 6357   ins_pipe(pipe_cmov_mem);
 6358 %}
 6359 
 6360 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6361   predicate(!UseAPX);
 6362   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6363   ins_cost(250);
 6364   expand %{
 6365     cmovI_memU(cop, cr, dst, src);
 6366   %}
 6367 %}
 6368 
 6369 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6370 %{
 6371   predicate(UseAPX);
 6372   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6373 
 6374   ins_cost(250);
 6375   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6376   ins_encode %{
 6377     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6378   %}
 6379   ins_pipe(pipe_cmov_mem);
 6380 %}
 6381 
 6382 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2) 
 6383 %{
 6384   predicate(UseAPX);
 6385   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6386   ins_cost(250);
 6387   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6388   ins_encode %{
 6389     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6390   %}
 6391   ins_pipe(pipe_cmov_mem);
 6392 %}
 6393 
 6394 // Conditional move
 6395 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6396 %{
 6397   predicate(!UseAPX);
 6398   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6399 
 6400   ins_cost(200); // XXX
 6401   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6402   ins_encode %{
 6403     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6404   %}
 6405   ins_pipe(pipe_cmov_reg);
 6406 %}
 6407 
 6408 // Conditional move ndd
 6409 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6410 %{
 6411   predicate(UseAPX);
 6412   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6413 
 6414   ins_cost(200);
 6415   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6416   ins_encode %{
 6417     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6418   %}
 6419   ins_pipe(pipe_cmov_reg);
 6420 %}
 6421 
 6422 // Conditional move
 6423 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6424 %{
 6425   predicate(!UseAPX);
 6426   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6427 
 6428   ins_cost(200); // XXX
 6429   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6430   ins_encode %{
 6431     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6432   %}
 6433   ins_pipe(pipe_cmov_reg);
 6434 %}
 6435 
 6436 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6437   predicate(!UseAPX);
 6438   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6439   ins_cost(200);
 6440   expand %{
 6441     cmovN_regU(cop, cr, dst, src);
 6442   %}
 6443 %}
 6444 
 6445 // Conditional move ndd
 6446 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6447 %{
 6448   predicate(UseAPX);
 6449   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6450 
 6451   ins_cost(200);
 6452   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6453   ins_encode %{
 6454     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6455   %}
 6456   ins_pipe(pipe_cmov_reg);
 6457 %}
 6458 
 6459 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6460   predicate(UseAPX);
 6461   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6462   ins_cost(200);
 6463   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6464   ins_encode %{
 6465     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6466   %}
 6467   ins_pipe(pipe_cmov_reg);
 6468 %}
 6469 
 6470 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6471   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6472   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6473 
 6474   ins_cost(200); // XXX
 6475   format %{ "cmovpl  $dst, $src\n\t"
 6476             "cmovnel $dst, $src" %}
 6477   ins_encode %{
 6478     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6479     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6480   %}
 6481   ins_pipe(pipe_cmov_reg);
 6482 %}
 6483 
 6484 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6485 // inputs of the CMove
 6486 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6487   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6488   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6489 
 6490   ins_cost(200); // XXX
 6491   format %{ "cmovpl  $dst, $src\n\t"
 6492             "cmovnel $dst, $src" %}
 6493   ins_encode %{
 6494     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6495     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6496   %}
 6497   ins_pipe(pipe_cmov_reg);
 6498 %}
 6499 
 6500 // Conditional move
 6501 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6502 %{
 6503   predicate(!UseAPX);
 6504   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6505 
 6506   ins_cost(200); // XXX
 6507   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6508   ins_encode %{
 6509     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6510   %}
 6511   ins_pipe(pipe_cmov_reg);  // XXX
 6512 %}
 6513 
 6514 // Conditional move ndd
 6515 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6516 %{
 6517   predicate(UseAPX);
 6518   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6519 
 6520   ins_cost(200);
 6521   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6522   ins_encode %{
 6523     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6524   %}
 6525   ins_pipe(pipe_cmov_reg);
 6526 %}
 6527 
 6528 // Conditional move
 6529 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6530 %{
 6531   predicate(!UseAPX);
 6532   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6533 
 6534   ins_cost(200); // XXX
 6535   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6536   ins_encode %{
 6537     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6538   %}
 6539   ins_pipe(pipe_cmov_reg); // XXX
 6540 %}
 6541 
 6542 // Conditional move ndd
 6543 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6544 %{
 6545   predicate(UseAPX);
 6546   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6547 
 6548   ins_cost(200);
 6549   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6550   ins_encode %{
 6551     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6552   %}
 6553   ins_pipe(pipe_cmov_reg);
 6554 %}
 6555 
 6556 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6557   predicate(!UseAPX);
 6558   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6559   ins_cost(200);
 6560   expand %{
 6561     cmovP_regU(cop, cr, dst, src);
 6562   %}
 6563 %}
 6564 
 6565 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6566   predicate(UseAPX);
 6567   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6568   ins_cost(200);
 6569   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6570   ins_encode %{
 6571     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6572   %}
 6573   ins_pipe(pipe_cmov_reg);
 6574 %}
 6575 
 6576 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6577   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6578   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6579 
 6580   ins_cost(200); // XXX
 6581   format %{ "cmovpq  $dst, $src\n\t"
 6582             "cmovneq $dst, $src" %}
 6583   ins_encode %{
 6584     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6585     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6586   %}
 6587   ins_pipe(pipe_cmov_reg);
 6588 %}
 6589 
 6590 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6591   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6592   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6593   effect(TEMP dst);
 6594 
 6595   ins_cost(200);
 6596   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6597             "cmovneq  $dst, $src2" %}
 6598   ins_encode %{
 6599     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6600     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6601   %}
 6602   ins_pipe(pipe_cmov_reg);
 6603 %}
 6604 
 6605 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6606 // inputs of the CMove
 6607 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6608   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6609   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6610 
 6611   ins_cost(200); // XXX
 6612   format %{ "cmovpq  $dst, $src\n\t"
 6613             "cmovneq $dst, $src" %}
 6614   ins_encode %{
 6615     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6616     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6617   %}
 6618   ins_pipe(pipe_cmov_reg);
 6619 %}
 6620 
 6621 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6622   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6623   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6624   effect(TEMP dst);
 6625 
 6626   ins_cost(200);
 6627   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6628             "cmovneq  $dst, $src2" %}
 6629   ins_encode %{
 6630     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6631     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6632   %}
 6633   ins_pipe(pipe_cmov_reg);
 6634 %}
 6635 
 6636 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6637 %{
 6638   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6639   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6640 
 6641   ins_cost(100); // XXX
 6642   format %{ "setbn$cop $dst\t# signed, long" %}
 6643   ins_encode %{
 6644     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6645     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6646   %}
 6647   ins_pipe(ialu_reg);
 6648 %}
 6649 
 6650 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6651 %{
 6652   predicate(!UseAPX);
 6653   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6654 
 6655   ins_cost(200); // XXX
 6656   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6657   ins_encode %{
 6658     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6659   %}
 6660   ins_pipe(pipe_cmov_reg);  // XXX
 6661 %}
 6662 
 6663 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6664 %{
 6665   predicate(UseAPX);
 6666   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6667 
 6668   ins_cost(200);
 6669   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6670   ins_encode %{
 6671     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6672   %}
 6673   ins_pipe(pipe_cmov_reg);
 6674 %}
 6675 
 6676 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6677 %{
 6678   predicate(!UseAPX);
 6679   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6680 
 6681   ins_cost(200); // XXX
 6682   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6683   ins_encode %{
 6684     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6685   %}
 6686   ins_pipe(pipe_cmov_mem);  // XXX
 6687 %}
 6688 
 6689 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6690 %{
 6691   predicate(UseAPX);
 6692   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6693 
 6694   ins_cost(200);
 6695   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6696   ins_encode %{
 6697     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6698   %}
 6699   ins_pipe(pipe_cmov_mem);
 6700 %}
 6701 
 6702 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6703 %{
 6704   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6705   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6706 
 6707   ins_cost(100); // XXX
 6708   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6709   ins_encode %{
 6710     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6711     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6712   %}
 6713   ins_pipe(ialu_reg);
 6714 %}
 6715 
 6716 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6717 %{
 6718   predicate(!UseAPX);
 6719   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6720 
 6721   ins_cost(200); // XXX
 6722   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6723   ins_encode %{
 6724     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6725   %}
 6726   ins_pipe(pipe_cmov_reg); // XXX
 6727 %}
 6728 
 6729 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6730 %{
 6731   predicate(UseAPX);
 6732   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6733 
 6734   ins_cost(200);
 6735   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6736   ins_encode %{
 6737     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6738   %}
 6739   ins_pipe(pipe_cmov_reg);
 6740 %}
 6741 
 6742 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6743 %{
 6744   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6745   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6746 
 6747   ins_cost(100); // XXX
 6748   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6749   ins_encode %{
 6750     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6751     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6752   %}
 6753   ins_pipe(ialu_reg);
 6754 %}
 6755 
 6756 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6757   predicate(!UseAPX);
 6758   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6759   ins_cost(200);
 6760   expand %{
 6761     cmovL_regU(cop, cr, dst, src);
 6762   %}
 6763 %}
 6764 
 6765 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2) 
 6766 %{
 6767   predicate(UseAPX);
 6768   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6769   ins_cost(200);
 6770   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6771   ins_encode %{
 6772     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6773   %}
 6774   ins_pipe(pipe_cmov_reg);
 6775 %}
 6776 
 6777 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6778   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6779   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6780 
 6781   ins_cost(200); // XXX
 6782   format %{ "cmovpq  $dst, $src\n\t"
 6783             "cmovneq $dst, $src" %}
 6784   ins_encode %{
 6785     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6786     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6787   %}
 6788   ins_pipe(pipe_cmov_reg);
 6789 %}
 6790 
 6791 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6792   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6793   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6794   effect(TEMP dst);
 6795 
 6796   ins_cost(200);
 6797   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6798             "cmovneq  $dst, $src2" %}
 6799   ins_encode %{
 6800     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6801     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6802   %}
 6803   ins_pipe(pipe_cmov_reg);
 6804 %}
 6805 
 6806 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6807 // inputs of the CMove
 6808 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6809   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6810   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6811 
 6812   ins_cost(200); // XXX
 6813   format %{ "cmovpq  $dst, $src\n\t"
 6814             "cmovneq $dst, $src" %}
 6815   ins_encode %{
 6816     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6817     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6818   %}
 6819   ins_pipe(pipe_cmov_reg);
 6820 %}
 6821 
 6822 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6823   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6824   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6825   effect(TEMP dst);
 6826 
 6827   ins_cost(200);
 6828   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6829             "cmovneq $dst, $src2" %}
 6830   ins_encode %{
 6831     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6832     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6833   %}
 6834   ins_pipe(pipe_cmov_reg);
 6835 %}
 6836 
 6837 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6838 %{
 6839   predicate(!UseAPX);
 6840   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6841 
 6842   ins_cost(200); // XXX
 6843   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6844   ins_encode %{
 6845     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6846   %}
 6847   ins_pipe(pipe_cmov_mem); // XXX
 6848 %}
 6849 
 6850 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6851   predicate(!UseAPX);
 6852   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6853   ins_cost(200);
 6854   expand %{
 6855     cmovL_memU(cop, cr, dst, src);
 6856   %}
 6857 %}
 6858 
 6859 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6860 %{
 6861   predicate(UseAPX);
 6862   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6863 
 6864   ins_cost(200);
 6865   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6866   ins_encode %{
 6867     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6868   %}
 6869   ins_pipe(pipe_cmov_mem);
 6870 %}
 6871 
 6872 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2) 
 6873 %{
 6874   predicate(UseAPX);
 6875   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6876   ins_cost(200);
 6877   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6878   ins_encode %{
 6879     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6880   %}
 6881   ins_pipe(pipe_cmov_mem);
 6882 %}
 6883 
 6884 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6885 %{
 6886   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6887 
 6888   ins_cost(200); // XXX
 6889   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6890             "movss     $dst, $src\n"
 6891     "skip:" %}
 6892   ins_encode %{
 6893     Label Lskip;
 6894     // Invert sense of branch from sense of CMOV
 6895     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6896     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6897     __ bind(Lskip);
 6898   %}
 6899   ins_pipe(pipe_slow);
 6900 %}
 6901 
 6902 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6903 %{
 6904   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6905 
 6906   ins_cost(200); // XXX
 6907   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6908             "movss     $dst, $src\n"
 6909     "skip:" %}
 6910   ins_encode %{
 6911     Label Lskip;
 6912     // Invert sense of branch from sense of CMOV
 6913     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6914     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6915     __ bind(Lskip);
 6916   %}
 6917   ins_pipe(pipe_slow);
 6918 %}
 6919 
 6920 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6921   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6922   ins_cost(200);
 6923   expand %{
 6924     cmovF_regU(cop, cr, dst, src);
 6925   %}
 6926 %}
 6927 
 6928 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6929 %{
 6930   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6931 
 6932   ins_cost(200); // XXX
 6933   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6934             "movsd     $dst, $src\n"
 6935     "skip:" %}
 6936   ins_encode %{
 6937     Label Lskip;
 6938     // Invert sense of branch from sense of CMOV
 6939     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6940     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6941     __ bind(Lskip);
 6942   %}
 6943   ins_pipe(pipe_slow);
 6944 %}
 6945 
 6946 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6947 %{
 6948   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6949 
 6950   ins_cost(200); // XXX
 6951   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6952             "movsd     $dst, $src\n"
 6953     "skip:" %}
 6954   ins_encode %{
 6955     Label Lskip;
 6956     // Invert sense of branch from sense of CMOV
 6957     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6958     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6959     __ bind(Lskip);
 6960   %}
 6961   ins_pipe(pipe_slow);
 6962 %}
 6963 
 6964 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 6965   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6966   ins_cost(200);
 6967   expand %{
 6968     cmovD_regU(cop, cr, dst, src);
 6969   %}
 6970 %}
 6971 
 6972 //----------Arithmetic Instructions--------------------------------------------
 6973 //----------Addition Instructions----------------------------------------------
 6974 
 6975 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 6976 %{
 6977   predicate(!UseAPX);
 6978   match(Set dst (AddI dst src));
 6979   effect(KILL cr);
 6980   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6981   format %{ "addl    $dst, $src\t# int" %}
 6982   ins_encode %{
 6983     __ addl($dst$$Register, $src$$Register);
 6984   %}
 6985   ins_pipe(ialu_reg_reg);
 6986 %}
 6987 
 6988 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 6989 %{
 6990   predicate(UseAPX);
 6991   match(Set dst (AddI src1 src2));
 6992   effect(KILL cr);
 6993   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 6994 
 6995   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 6996   ins_encode %{
 6997     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 6998   %}
 6999   ins_pipe(ialu_reg_reg);
 7000 %}
 7001 
 7002 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7003 %{
 7004   predicate(!UseAPX);
 7005   match(Set dst (AddI dst src));
 7006   effect(KILL cr);
 7007   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7008 
 7009   format %{ "addl    $dst, $src\t# int" %}
 7010   ins_encode %{
 7011     __ addl($dst$$Register, $src$$constant);
 7012   %}
 7013   ins_pipe( ialu_reg );
 7014 %}
 7015 
 7016 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7017 %{
 7018   predicate(UseAPX);
 7019   match(Set dst (AddI src1 src2));
 7020   effect(KILL cr);
 7021   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7022 
 7023   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7024   ins_encode %{
 7025     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7026   %}
 7027   ins_pipe( ialu_reg );
 7028 %}
 7029 
 7030 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7031 %{
 7032   predicate(UseAPX);
 7033   match(Set dst (AddI (LoadI src1) src2));
 7034   effect(KILL cr);
 7035   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7036 
 7037   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7038   ins_encode %{
 7039     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7040   %}
 7041   ins_pipe( ialu_reg );
 7042 %}
 7043 
 7044 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7045 %{
 7046   predicate(!UseAPX);
 7047   match(Set dst (AddI dst (LoadI src)));
 7048   effect(KILL cr);
 7049   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7050 
 7051   ins_cost(150); // XXX
 7052   format %{ "addl    $dst, $src\t# int" %}
 7053   ins_encode %{
 7054     __ addl($dst$$Register, $src$$Address);
 7055   %}
 7056   ins_pipe(ialu_reg_mem);
 7057 %}
 7058 
 7059 instruct addI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 7060 %{
 7061   predicate(UseAPX);
 7062   match(Set dst (AddI (LoadI src1) src2));
 7063   effect(KILL cr);
 7064   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7065 
 7066   ins_cost(150);
 7067   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7068   ins_encode %{
 7069     __ eaddl($dst$$Register, $src1$$Address, $src2$$Register, false);
 7070   %}
 7071   ins_pipe(ialu_reg_mem);
 7072 %}
 7073 
 7074 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7075 %{
 7076   predicate(UseAPX);
 7077   match(Set dst (AddI src1 (LoadI src2)));
 7078   effect(KILL cr);
 7079   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7080 
 7081   ins_cost(150);
 7082   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7083   ins_encode %{
 7084     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7085   %}
 7086   ins_pipe(ialu_reg_mem);
 7087 %}
 7088 
 7089 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7090 %{
 7091   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7092   effect(KILL cr);
 7093   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7094 
 7095   ins_cost(150); // XXX
 7096   format %{ "addl    $dst, $src\t# int" %}
 7097   ins_encode %{
 7098     __ addl($dst$$Address, $src$$Register);
 7099   %}
 7100   ins_pipe(ialu_mem_reg);
 7101 %}
 7102 
 7103 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7104 %{
 7105   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7106   effect(KILL cr);
 7107   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7108 
 7109 
 7110   ins_cost(125); // XXX
 7111   format %{ "addl    $dst, $src\t# int" %}
 7112   ins_encode %{
 7113     __ addl($dst$$Address, $src$$constant);
 7114   %}
 7115   ins_pipe(ialu_mem_imm);
 7116 %}
 7117 
 7118 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7119 %{
 7120   predicate(!UseAPX && UseIncDec);
 7121   match(Set dst (AddI dst src));
 7122   effect(KILL cr);
 7123 
 7124   format %{ "incl    $dst\t# int" %}
 7125   ins_encode %{
 7126     __ incrementl($dst$$Register);
 7127   %}
 7128   ins_pipe(ialu_reg);
 7129 %}
 7130 
 7131 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7132 %{
 7133   predicate(UseAPX && UseIncDec);
 7134   match(Set dst (AddI src val));
 7135   effect(KILL cr);
 7136 
 7137   format %{ "eincl    $dst, $src\t# int ndd" %}
 7138   ins_encode %{
 7139     __ eincl($dst$$Register, $src$$Register, false);
 7140   %}
 7141   ins_pipe(ialu_reg);
 7142 %}
 7143 
 7144 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7145 %{
 7146   predicate(UseAPX && UseIncDec);
 7147   match(Set dst (AddI (LoadI src) val));
 7148   effect(KILL cr);
 7149 
 7150   format %{ "eincl    $dst, $src\t# int ndd" %}
 7151   ins_encode %{
 7152     __ eincl($dst$$Register, $src$$Address, false);
 7153   %}
 7154   ins_pipe(ialu_reg);
 7155 %}
 7156 
 7157 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7158 %{
 7159   predicate(UseIncDec);
 7160   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7161   effect(KILL cr);
 7162 
 7163   ins_cost(125); // XXX
 7164   format %{ "incl    $dst\t# int" %}
 7165   ins_encode %{
 7166     __ incrementl($dst$$Address);
 7167   %}
 7168   ins_pipe(ialu_mem_imm);
 7169 %}
 7170 
 7171 // XXX why does that use AddI
 7172 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7173 %{
 7174   predicate(!UseAPX && UseIncDec);
 7175   match(Set dst (AddI dst src));
 7176   effect(KILL cr);
 7177 
 7178   format %{ "decl    $dst\t# int" %}
 7179   ins_encode %{
 7180     __ decrementl($dst$$Register);
 7181   %}
 7182   ins_pipe(ialu_reg);
 7183 %}
 7184 
 7185 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7186 %{
 7187   predicate(UseAPX && UseIncDec);
 7188   match(Set dst (AddI src val));
 7189   effect(KILL cr);
 7190 
 7191   format %{ "edecl    $dst, $src\t# int ndd" %}
 7192   ins_encode %{
 7193     __ edecl($dst$$Register, $src$$Register, false);
 7194   %}
 7195   ins_pipe(ialu_reg);
 7196 %}
 7197 
 7198 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7199 %{
 7200   predicate(UseAPX && UseIncDec);
 7201   match(Set dst (AddI (LoadI src) val));
 7202   effect(KILL cr);
 7203 
 7204   format %{ "edecl    $dst, $src\t# int ndd" %}
 7205   ins_encode %{
 7206     __ edecl($dst$$Register, $src$$Address, false);
 7207   %}
 7208   ins_pipe(ialu_reg);
 7209 %}
 7210 
 7211 // XXX why does that use AddI
 7212 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7213 %{
 7214   predicate(UseIncDec);
 7215   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7216   effect(KILL cr);
 7217 
 7218   ins_cost(125); // XXX
 7219   format %{ "decl    $dst\t# int" %}
 7220   ins_encode %{
 7221     __ decrementl($dst$$Address);
 7222   %}
 7223   ins_pipe(ialu_mem_imm);
 7224 %}
 7225 
 7226 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7227 %{
 7228   predicate(VM_Version::supports_fast_2op_lea());
 7229   match(Set dst (AddI (LShiftI index scale) disp));
 7230 
 7231   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7232   ins_encode %{
 7233     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7234     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7235   %}
 7236   ins_pipe(ialu_reg_reg);
 7237 %}
 7238 
 7239 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7240 %{
 7241   predicate(VM_Version::supports_fast_3op_lea());
 7242   match(Set dst (AddI (AddI base index) disp));
 7243 
 7244   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7245   ins_encode %{
 7246     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7247   %}
 7248   ins_pipe(ialu_reg_reg);
 7249 %}
 7250 
 7251 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7252 %{
 7253   predicate(VM_Version::supports_fast_2op_lea());
 7254   match(Set dst (AddI base (LShiftI index scale)));
 7255 
 7256   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7257   ins_encode %{
 7258     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7259     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7260   %}
 7261   ins_pipe(ialu_reg_reg);
 7262 %}
 7263 
 7264 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7265 %{
 7266   predicate(VM_Version::supports_fast_3op_lea());
 7267   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7268 
 7269   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7270   ins_encode %{
 7271     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7272     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7273   %}
 7274   ins_pipe(ialu_reg_reg);
 7275 %}
 7276 
 7277 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7278 %{
 7279   predicate(!UseAPX);
 7280   match(Set dst (AddL dst src));
 7281   effect(KILL cr);
 7282   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7283 
 7284   format %{ "addq    $dst, $src\t# long" %}
 7285   ins_encode %{
 7286     __ addq($dst$$Register, $src$$Register);
 7287   %}
 7288   ins_pipe(ialu_reg_reg);
 7289 %}
 7290 
 7291 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7292 %{
 7293   predicate(UseAPX);
 7294   match(Set dst (AddL src1 src2));
 7295   effect(KILL cr);
 7296   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7297 
 7298   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7299   ins_encode %{
 7300     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7301   %}
 7302   ins_pipe(ialu_reg_reg);
 7303 %}
 7304 
 7305 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7306 %{
 7307   predicate(!UseAPX);
 7308   match(Set dst (AddL dst src));
 7309   effect(KILL cr);
 7310   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7311 
 7312   format %{ "addq    $dst, $src\t# long" %}
 7313   ins_encode %{
 7314     __ addq($dst$$Register, $src$$constant);
 7315   %}
 7316   ins_pipe( ialu_reg );
 7317 %}
 7318 
 7319 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7320 %{
 7321   predicate(UseAPX);
 7322   match(Set dst (AddL src1 src2));
 7323   effect(KILL cr);
 7324   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7325 
 7326   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7327   ins_encode %{
 7328     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7329   %}
 7330   ins_pipe( ialu_reg );
 7331 %}
 7332 
 7333 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7334 %{
 7335   predicate(UseAPX);
 7336   match(Set dst (AddL (LoadL src1) src2));
 7337   effect(KILL cr);
 7338   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7339 
 7340   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7341   ins_encode %{
 7342     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7343   %}
 7344   ins_pipe( ialu_reg );
 7345 %}
 7346 
 7347 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7348 %{
 7349   predicate(!UseAPX);
 7350   match(Set dst (AddL dst (LoadL src)));
 7351   effect(KILL cr);
 7352   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7353 
 7354   ins_cost(150); // XXX
 7355   format %{ "addq    $dst, $src\t# long" %}
 7356   ins_encode %{
 7357     __ addq($dst$$Register, $src$$Address);
 7358   %}
 7359   ins_pipe(ialu_reg_mem);
 7360 %}
 7361 
 7362 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7363 %{
 7364   predicate(UseAPX);
 7365   match(Set dst (AddL src1 (LoadL src2)));
 7366   effect(KILL cr);
 7367   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7368 
 7369   ins_cost(150);
 7370   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7371   ins_encode %{
 7372     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7373   %}
 7374   ins_pipe(ialu_reg_mem);
 7375 %}
 7376 
 7377 instruct addL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 7378 %{
 7379   predicate(UseAPX);
 7380   match(Set dst (AddL (LoadL src1) src2));
 7381   effect(KILL cr);
 7382   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7383 
 7384   ins_cost(150);
 7385   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7386   ins_encode %{
 7387     __ eaddq($dst$$Register, $src1$$Address, $src2$$Register, false);
 7388   %}
 7389   ins_pipe(ialu_reg_mem);
 7390 %}
 7391 
 7392 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7393 %{
 7394   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7395   effect(KILL cr);
 7396   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7397 
 7398   ins_cost(150); // XXX
 7399   format %{ "addq    $dst, $src\t# long" %}
 7400   ins_encode %{
 7401     __ addq($dst$$Address, $src$$Register);
 7402   %}
 7403   ins_pipe(ialu_mem_reg);
 7404 %}
 7405 
 7406 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7407 %{
 7408   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7409   effect(KILL cr);
 7410   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7411 
 7412   ins_cost(125); // XXX
 7413   format %{ "addq    $dst, $src\t# long" %}
 7414   ins_encode %{
 7415     __ addq($dst$$Address, $src$$constant);
 7416   %}
 7417   ins_pipe(ialu_mem_imm);
 7418 %}
 7419 
 7420 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7421 %{
 7422   predicate(!UseAPX && UseIncDec);
 7423   match(Set dst (AddL dst src));
 7424   effect(KILL cr);
 7425 
 7426   format %{ "incq    $dst\t# long" %}
 7427   ins_encode %{
 7428     __ incrementq($dst$$Register);
 7429   %}
 7430   ins_pipe(ialu_reg);
 7431 %}
 7432 
 7433 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7434 %{
 7435   predicate(UseAPX && UseIncDec);
 7436   match(Set dst (AddL src val));
 7437   effect(KILL cr);
 7438 
 7439   format %{ "eincq    $dst, $src\t# long ndd" %}
 7440   ins_encode %{
 7441     __ eincq($dst$$Register, $src$$Register, false);
 7442   %}
 7443   ins_pipe(ialu_reg);
 7444 %}
 7445 
 7446 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7447 %{
 7448   predicate(UseAPX && UseIncDec);
 7449   match(Set dst (AddL (LoadL src) val));
 7450   effect(KILL cr);
 7451 
 7452   format %{ "eincq    $dst, $src\t# long ndd" %}
 7453   ins_encode %{
 7454     __ eincq($dst$$Register, $src$$Address, false);
 7455   %}
 7456   ins_pipe(ialu_reg);
 7457 %}
 7458 
 7459 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7460 %{
 7461   predicate(UseIncDec);
 7462   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7463   effect(KILL cr);
 7464 
 7465   ins_cost(125); // XXX
 7466   format %{ "incq    $dst\t# long" %}
 7467   ins_encode %{
 7468     __ incrementq($dst$$Address);
 7469   %}
 7470   ins_pipe(ialu_mem_imm);
 7471 %}
 7472 
 7473 // XXX why does that use AddL
 7474 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7475 %{
 7476   predicate(!UseAPX && UseIncDec);
 7477   match(Set dst (AddL dst src));
 7478   effect(KILL cr);
 7479 
 7480   format %{ "decq    $dst\t# long" %}
 7481   ins_encode %{
 7482     __ decrementq($dst$$Register);
 7483   %}
 7484   ins_pipe(ialu_reg);
 7485 %}
 7486 
 7487 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7488 %{
 7489   predicate(UseAPX && UseIncDec);
 7490   match(Set dst (AddL src val));
 7491   effect(KILL cr);
 7492 
 7493   format %{ "edecq    $dst, $src\t# long ndd" %}
 7494   ins_encode %{
 7495     __ edecq($dst$$Register, $src$$Register, false);
 7496   %}
 7497   ins_pipe(ialu_reg);
 7498 %}
 7499 
 7500 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7501 %{
 7502   predicate(UseAPX && UseIncDec);
 7503   match(Set dst (AddL (LoadL src) val));
 7504   effect(KILL cr);
 7505 
 7506   format %{ "edecq    $dst, $src\t# long ndd" %}
 7507   ins_encode %{
 7508     __ edecq($dst$$Register, $src$$Address, false);
 7509   %}
 7510   ins_pipe(ialu_reg);
 7511 %}
 7512 
 7513 // XXX why does that use AddL
 7514 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7515 %{
 7516   predicate(UseIncDec);
 7517   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7518   effect(KILL cr);
 7519 
 7520   ins_cost(125); // XXX
 7521   format %{ "decq    $dst\t# long" %}
 7522   ins_encode %{
 7523     __ decrementq($dst$$Address);
 7524   %}
 7525   ins_pipe(ialu_mem_imm);
 7526 %}
 7527 
 7528 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7529 %{
 7530   predicate(VM_Version::supports_fast_2op_lea());
 7531   match(Set dst (AddL (LShiftL index scale) disp));
 7532 
 7533   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7534   ins_encode %{
 7535     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7536     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7537   %}
 7538   ins_pipe(ialu_reg_reg);
 7539 %}
 7540 
 7541 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7542 %{
 7543   predicate(VM_Version::supports_fast_3op_lea());
 7544   match(Set dst (AddL (AddL base index) disp));
 7545 
 7546   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7547   ins_encode %{
 7548     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7549   %}
 7550   ins_pipe(ialu_reg_reg);
 7551 %}
 7552 
 7553 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7554 %{
 7555   predicate(VM_Version::supports_fast_2op_lea());
 7556   match(Set dst (AddL base (LShiftL index scale)));
 7557 
 7558   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7559   ins_encode %{
 7560     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7561     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7562   %}
 7563   ins_pipe(ialu_reg_reg);
 7564 %}
 7565 
 7566 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7567 %{
 7568   predicate(VM_Version::supports_fast_3op_lea());
 7569   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7570 
 7571   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7572   ins_encode %{
 7573     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7574     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7575   %}
 7576   ins_pipe(ialu_reg_reg);
 7577 %}
 7578 
 7579 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7580 %{
 7581   match(Set dst (AddP dst src));
 7582   effect(KILL cr);
 7583   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7584 
 7585   format %{ "addq    $dst, $src\t# ptr" %}
 7586   ins_encode %{
 7587     __ addq($dst$$Register, $src$$Register);
 7588   %}
 7589   ins_pipe(ialu_reg_reg);
 7590 %}
 7591 
 7592 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7593 %{
 7594   match(Set dst (AddP dst src));
 7595   effect(KILL cr);
 7596   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7597 
 7598   format %{ "addq    $dst, $src\t# ptr" %}
 7599   ins_encode %{
 7600     __ addq($dst$$Register, $src$$constant);
 7601   %}
 7602   ins_pipe( ialu_reg );
 7603 %}
 7604 
 7605 // XXX addP mem ops ????
 7606 
 7607 instruct checkCastPP(rRegP dst)
 7608 %{
 7609   match(Set dst (CheckCastPP dst));
 7610 
 7611   size(0);
 7612   format %{ "# checkcastPP of $dst" %}
 7613   ins_encode(/* empty encoding */);
 7614   ins_pipe(empty);
 7615 %}
 7616 
 7617 instruct castPP(rRegP dst)
 7618 %{
 7619   match(Set dst (CastPP dst));
 7620 
 7621   size(0);
 7622   format %{ "# castPP of $dst" %}
 7623   ins_encode(/* empty encoding */);
 7624   ins_pipe(empty);
 7625 %}
 7626 
 7627 instruct castII(rRegI dst)
 7628 %{
 7629   predicate(VerifyConstraintCasts == 0);
 7630   match(Set dst (CastII dst));
 7631 
 7632   size(0);
 7633   format %{ "# castII of $dst" %}
 7634   ins_encode(/* empty encoding */);
 7635   ins_cost(0);
 7636   ins_pipe(empty);
 7637 %}
 7638 
 7639 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7640 %{
 7641   predicate(VerifyConstraintCasts > 0);
 7642   match(Set dst (CastII dst));
 7643 
 7644   effect(KILL cr);
 7645   format %{ "# cast_checked_II $dst" %}
 7646   ins_encode %{
 7647     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7648   %}
 7649   ins_pipe(pipe_slow);
 7650 %}
 7651 
 7652 instruct castLL(rRegL dst)
 7653 %{
 7654   predicate(VerifyConstraintCasts == 0);
 7655   match(Set dst (CastLL dst));
 7656 
 7657   size(0);
 7658   format %{ "# castLL of $dst" %}
 7659   ins_encode(/* empty encoding */);
 7660   ins_cost(0);
 7661   ins_pipe(empty);
 7662 %}
 7663 
 7664 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7665 %{
 7666   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7667   match(Set dst (CastLL dst));
 7668 
 7669   effect(KILL cr);
 7670   format %{ "# cast_checked_LL $dst" %}
 7671   ins_encode %{
 7672     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7673   %}
 7674   ins_pipe(pipe_slow);
 7675 %}
 7676 
 7677 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7678 %{
 7679   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7680   match(Set dst (CastLL dst));
 7681 
 7682   effect(KILL cr, TEMP tmp);
 7683   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7684   ins_encode %{
 7685     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7686   %}
 7687   ins_pipe(pipe_slow);
 7688 %}
 7689 
 7690 instruct castFF(regF dst)
 7691 %{
 7692   match(Set dst (CastFF dst));
 7693 
 7694   size(0);
 7695   format %{ "# castFF of $dst" %}
 7696   ins_encode(/* empty encoding */);
 7697   ins_cost(0);
 7698   ins_pipe(empty);
 7699 %}
 7700 
 7701 instruct castHH(regF dst)
 7702 %{
 7703   match(Set dst (CastHH dst));
 7704 
 7705   size(0);
 7706   format %{ "# castHH of $dst" %}
 7707   ins_encode(/* empty encoding */);
 7708   ins_cost(0);
 7709   ins_pipe(empty);
 7710 %}
 7711 
 7712 instruct castDD(regD dst)
 7713 %{
 7714   match(Set dst (CastDD dst));
 7715 
 7716   size(0);
 7717   format %{ "# castDD of $dst" %}
 7718   ins_encode(/* empty encoding */);
 7719   ins_cost(0);
 7720   ins_pipe(empty);
 7721 %}
 7722 
 7723 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7724 instruct compareAndSwapP(rRegI res,
 7725                          memory mem_ptr,
 7726                          rax_RegP oldval, rRegP newval,
 7727                          rFlagsReg cr)
 7728 %{
 7729   predicate(n->as_LoadStore()->barrier_data() == 0);
 7730   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7731   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7732   effect(KILL cr, KILL oldval);
 7733 
 7734   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7735             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7736             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7737   ins_encode %{
 7738     __ lock();
 7739     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7740     __ setcc(Assembler::equal, $res$$Register);
 7741   %}
 7742   ins_pipe( pipe_cmpxchg );
 7743 %}
 7744 
 7745 instruct compareAndSwapL(rRegI res,
 7746                          memory mem_ptr,
 7747                          rax_RegL oldval, rRegL newval,
 7748                          rFlagsReg cr)
 7749 %{
 7750   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7751   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7752   effect(KILL cr, KILL oldval);
 7753 
 7754   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7755             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7756             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7757   ins_encode %{
 7758     __ lock();
 7759     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7760     __ setcc(Assembler::equal, $res$$Register);
 7761   %}
 7762   ins_pipe( pipe_cmpxchg );
 7763 %}
 7764 
 7765 instruct compareAndSwapI(rRegI res,
 7766                          memory mem_ptr,
 7767                          rax_RegI oldval, rRegI newval,
 7768                          rFlagsReg cr)
 7769 %{
 7770   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7771   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7772   effect(KILL cr, KILL oldval);
 7773 
 7774   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7775             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7776             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7777   ins_encode %{
 7778     __ lock();
 7779     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7780     __ setcc(Assembler::equal, $res$$Register);
 7781   %}
 7782   ins_pipe( pipe_cmpxchg );
 7783 %}
 7784 
 7785 instruct compareAndSwapB(rRegI res,
 7786                          memory mem_ptr,
 7787                          rax_RegI oldval, rRegI newval,
 7788                          rFlagsReg cr)
 7789 %{
 7790   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7791   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7792   effect(KILL cr, KILL oldval);
 7793 
 7794   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7795             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7796             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7797   ins_encode %{
 7798     __ lock();
 7799     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7800     __ setcc(Assembler::equal, $res$$Register);
 7801   %}
 7802   ins_pipe( pipe_cmpxchg );
 7803 %}
 7804 
 7805 instruct compareAndSwapS(rRegI res,
 7806                          memory mem_ptr,
 7807                          rax_RegI oldval, rRegI newval,
 7808                          rFlagsReg cr)
 7809 %{
 7810   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7811   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7812   effect(KILL cr, KILL oldval);
 7813 
 7814   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7815             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7816             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7817   ins_encode %{
 7818     __ lock();
 7819     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7820     __ setcc(Assembler::equal, $res$$Register);
 7821   %}
 7822   ins_pipe( pipe_cmpxchg );
 7823 %}
 7824 
 7825 instruct compareAndSwapN(rRegI res,
 7826                           memory mem_ptr,
 7827                           rax_RegN oldval, rRegN newval,
 7828                           rFlagsReg cr) %{
 7829   predicate(n->as_LoadStore()->barrier_data() == 0);
 7830   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7831   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7832   effect(KILL cr, KILL oldval);
 7833 
 7834   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7835             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7836             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7837   ins_encode %{
 7838     __ lock();
 7839     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7840     __ setcc(Assembler::equal, $res$$Register);
 7841   %}
 7842   ins_pipe( pipe_cmpxchg );
 7843 %}
 7844 
 7845 instruct compareAndExchangeB(
 7846                          memory mem_ptr,
 7847                          rax_RegI oldval, rRegI newval,
 7848                          rFlagsReg cr)
 7849 %{
 7850   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7851   effect(KILL cr);
 7852 
 7853   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7854             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7855   ins_encode %{
 7856     __ lock();
 7857     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7858   %}
 7859   ins_pipe( pipe_cmpxchg );
 7860 %}
 7861 
 7862 instruct compareAndExchangeS(
 7863                          memory mem_ptr,
 7864                          rax_RegI oldval, rRegI newval,
 7865                          rFlagsReg cr)
 7866 %{
 7867   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7868   effect(KILL cr);
 7869 
 7870   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7871             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7872   ins_encode %{
 7873     __ lock();
 7874     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7875   %}
 7876   ins_pipe( pipe_cmpxchg );
 7877 %}
 7878 
 7879 instruct compareAndExchangeI(
 7880                          memory mem_ptr,
 7881                          rax_RegI oldval, rRegI newval,
 7882                          rFlagsReg cr)
 7883 %{
 7884   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7885   effect(KILL cr);
 7886 
 7887   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7888             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7889   ins_encode %{
 7890     __ lock();
 7891     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7892   %}
 7893   ins_pipe( pipe_cmpxchg );
 7894 %}
 7895 
 7896 instruct compareAndExchangeL(
 7897                          memory mem_ptr,
 7898                          rax_RegL oldval, rRegL newval,
 7899                          rFlagsReg cr)
 7900 %{
 7901   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7902   effect(KILL cr);
 7903 
 7904   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7905             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7906   ins_encode %{
 7907     __ lock();
 7908     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7909   %}
 7910   ins_pipe( pipe_cmpxchg );
 7911 %}
 7912 
 7913 instruct compareAndExchangeN(
 7914                           memory mem_ptr,
 7915                           rax_RegN oldval, rRegN newval,
 7916                           rFlagsReg cr) %{
 7917   predicate(n->as_LoadStore()->barrier_data() == 0);
 7918   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7919   effect(KILL cr);
 7920 
 7921   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7922             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7923   ins_encode %{
 7924     __ lock();
 7925     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7926   %}
 7927   ins_pipe( pipe_cmpxchg );
 7928 %}
 7929 
 7930 instruct compareAndExchangeP(
 7931                          memory mem_ptr,
 7932                          rax_RegP oldval, rRegP newval,
 7933                          rFlagsReg cr)
 7934 %{
 7935   predicate(n->as_LoadStore()->barrier_data() == 0);
 7936   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7937   effect(KILL cr);
 7938 
 7939   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7940             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7941   ins_encode %{
 7942     __ lock();
 7943     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7944   %}
 7945   ins_pipe( pipe_cmpxchg );
 7946 %}
 7947 
 7948 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7949   predicate(n->as_LoadStore()->result_not_used());
 7950   match(Set dummy (GetAndAddB mem add));
 7951   effect(KILL cr);
 7952   format %{ "addb_lock   $mem, $add" %}
 7953   ins_encode %{
 7954     __ lock();
 7955     __ addb($mem$$Address, $add$$Register);
 7956   %}
 7957   ins_pipe(pipe_cmpxchg);
 7958 %}
 7959 
 7960 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7961   predicate(n->as_LoadStore()->result_not_used());
 7962   match(Set dummy (GetAndAddB mem add));
 7963   effect(KILL cr);
 7964   format %{ "addb_lock   $mem, $add" %}
 7965   ins_encode %{
 7966     __ lock();
 7967     __ addb($mem$$Address, $add$$constant);
 7968   %}
 7969   ins_pipe(pipe_cmpxchg);
 7970 %}
 7971 
 7972 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7973   predicate(!n->as_LoadStore()->result_not_used());
 7974   match(Set newval (GetAndAddB mem newval));
 7975   effect(KILL cr);
 7976   format %{ "xaddb_lock  $mem, $newval" %}
 7977   ins_encode %{
 7978     __ lock();
 7979     __ xaddb($mem$$Address, $newval$$Register);
 7980   %}
 7981   ins_pipe(pipe_cmpxchg);
 7982 %}
 7983 
 7984 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7985   predicate(n->as_LoadStore()->result_not_used());
 7986   match(Set dummy (GetAndAddS mem add));
 7987   effect(KILL cr);
 7988   format %{ "addw_lock   $mem, $add" %}
 7989   ins_encode %{
 7990     __ lock();
 7991     __ addw($mem$$Address, $add$$Register);
 7992   %}
 7993   ins_pipe(pipe_cmpxchg);
 7994 %}
 7995 
 7996 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7997   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 7998   match(Set dummy (GetAndAddS mem add));
 7999   effect(KILL cr);
 8000   format %{ "addw_lock   $mem, $add" %}
 8001   ins_encode %{
 8002     __ lock();
 8003     __ addw($mem$$Address, $add$$constant);
 8004   %}
 8005   ins_pipe(pipe_cmpxchg);
 8006 %}
 8007 
 8008 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8009   predicate(!n->as_LoadStore()->result_not_used());
 8010   match(Set newval (GetAndAddS mem newval));
 8011   effect(KILL cr);
 8012   format %{ "xaddw_lock  $mem, $newval" %}
 8013   ins_encode %{
 8014     __ lock();
 8015     __ xaddw($mem$$Address, $newval$$Register);
 8016   %}
 8017   ins_pipe(pipe_cmpxchg);
 8018 %}
 8019 
 8020 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8021   predicate(n->as_LoadStore()->result_not_used());
 8022   match(Set dummy (GetAndAddI mem add));
 8023   effect(KILL cr);
 8024   format %{ "addl_lock   $mem, $add" %}
 8025   ins_encode %{
 8026     __ lock();
 8027     __ addl($mem$$Address, $add$$Register);
 8028   %}
 8029   ins_pipe(pipe_cmpxchg);
 8030 %}
 8031 
 8032 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8033   predicate(n->as_LoadStore()->result_not_used());
 8034   match(Set dummy (GetAndAddI mem add));
 8035   effect(KILL cr);
 8036   format %{ "addl_lock   $mem, $add" %}
 8037   ins_encode %{
 8038     __ lock();
 8039     __ addl($mem$$Address, $add$$constant);
 8040   %}
 8041   ins_pipe(pipe_cmpxchg);
 8042 %}
 8043 
 8044 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8045   predicate(!n->as_LoadStore()->result_not_used());
 8046   match(Set newval (GetAndAddI mem newval));
 8047   effect(KILL cr);
 8048   format %{ "xaddl_lock  $mem, $newval" %}
 8049   ins_encode %{
 8050     __ lock();
 8051     __ xaddl($mem$$Address, $newval$$Register);
 8052   %}
 8053   ins_pipe(pipe_cmpxchg);
 8054 %}
 8055 
 8056 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8057   predicate(n->as_LoadStore()->result_not_used());
 8058   match(Set dummy (GetAndAddL mem add));
 8059   effect(KILL cr);
 8060   format %{ "addq_lock   $mem, $add" %}
 8061   ins_encode %{
 8062     __ lock();
 8063     __ addq($mem$$Address, $add$$Register);
 8064   %}
 8065   ins_pipe(pipe_cmpxchg);
 8066 %}
 8067 
 8068 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8069   predicate(n->as_LoadStore()->result_not_used());
 8070   match(Set dummy (GetAndAddL mem add));
 8071   effect(KILL cr);
 8072   format %{ "addq_lock   $mem, $add" %}
 8073   ins_encode %{
 8074     __ lock();
 8075     __ addq($mem$$Address, $add$$constant);
 8076   %}
 8077   ins_pipe(pipe_cmpxchg);
 8078 %}
 8079 
 8080 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8081   predicate(!n->as_LoadStore()->result_not_used());
 8082   match(Set newval (GetAndAddL mem newval));
 8083   effect(KILL cr);
 8084   format %{ "xaddq_lock  $mem, $newval" %}
 8085   ins_encode %{
 8086     __ lock();
 8087     __ xaddq($mem$$Address, $newval$$Register);
 8088   %}
 8089   ins_pipe(pipe_cmpxchg);
 8090 %}
 8091 
 8092 instruct xchgB( memory mem, rRegI newval) %{
 8093   match(Set newval (GetAndSetB mem newval));
 8094   format %{ "XCHGB  $newval,[$mem]" %}
 8095   ins_encode %{
 8096     __ xchgb($newval$$Register, $mem$$Address);
 8097   %}
 8098   ins_pipe( pipe_cmpxchg );
 8099 %}
 8100 
 8101 instruct xchgS( memory mem, rRegI newval) %{
 8102   match(Set newval (GetAndSetS mem newval));
 8103   format %{ "XCHGW  $newval,[$mem]" %}
 8104   ins_encode %{
 8105     __ xchgw($newval$$Register, $mem$$Address);
 8106   %}
 8107   ins_pipe( pipe_cmpxchg );
 8108 %}
 8109 
 8110 instruct xchgI( memory mem, rRegI newval) %{
 8111   match(Set newval (GetAndSetI mem newval));
 8112   format %{ "XCHGL  $newval,[$mem]" %}
 8113   ins_encode %{
 8114     __ xchgl($newval$$Register, $mem$$Address);
 8115   %}
 8116   ins_pipe( pipe_cmpxchg );
 8117 %}
 8118 
 8119 instruct xchgL( memory mem, rRegL newval) %{
 8120   match(Set newval (GetAndSetL mem newval));
 8121   format %{ "XCHGL  $newval,[$mem]" %}
 8122   ins_encode %{
 8123     __ xchgq($newval$$Register, $mem$$Address);
 8124   %}
 8125   ins_pipe( pipe_cmpxchg );
 8126 %}
 8127 
 8128 instruct xchgP( memory mem, rRegP newval) %{
 8129   match(Set newval (GetAndSetP mem newval));
 8130   predicate(n->as_LoadStore()->barrier_data() == 0);
 8131   format %{ "XCHGQ  $newval,[$mem]" %}
 8132   ins_encode %{
 8133     __ xchgq($newval$$Register, $mem$$Address);
 8134   %}
 8135   ins_pipe( pipe_cmpxchg );
 8136 %}
 8137 
 8138 instruct xchgN( memory mem, rRegN newval) %{
 8139   predicate(n->as_LoadStore()->barrier_data() == 0);
 8140   match(Set newval (GetAndSetN mem newval));
 8141   format %{ "XCHGL  $newval,$mem]" %}
 8142   ins_encode %{
 8143     __ xchgl($newval$$Register, $mem$$Address);
 8144   %}
 8145   ins_pipe( pipe_cmpxchg );
 8146 %}
 8147 
 8148 //----------Abs Instructions-------------------------------------------
 8149 
 8150 // Integer Absolute Instructions
 8151 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8152 %{
 8153   match(Set dst (AbsI src));
 8154   effect(TEMP dst, KILL cr);
 8155   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8156             "subl    $dst, $src\n\t"
 8157             "cmovll  $dst, $src" %}
 8158   ins_encode %{
 8159     __ xorl($dst$$Register, $dst$$Register);
 8160     __ subl($dst$$Register, $src$$Register);
 8161     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8162   %}
 8163 
 8164   ins_pipe(ialu_reg_reg);
 8165 %}
 8166 
 8167 // Long Absolute Instructions
 8168 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8169 %{
 8170   match(Set dst (AbsL src));
 8171   effect(TEMP dst, KILL cr);
 8172   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8173             "subq    $dst, $src\n\t"
 8174             "cmovlq  $dst, $src" %}
 8175   ins_encode %{
 8176     __ xorl($dst$$Register, $dst$$Register);
 8177     __ subq($dst$$Register, $src$$Register);
 8178     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8179   %}
 8180 
 8181   ins_pipe(ialu_reg_reg);
 8182 %}
 8183 
 8184 //----------Subtraction Instructions-------------------------------------------
 8185 
 8186 // Integer Subtraction Instructions
 8187 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8188 %{
 8189   predicate(!UseAPX);
 8190   match(Set dst (SubI dst src));
 8191   effect(KILL cr);
 8192   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8193 
 8194   format %{ "subl    $dst, $src\t# int" %}
 8195   ins_encode %{
 8196     __ subl($dst$$Register, $src$$Register);
 8197   %}
 8198   ins_pipe(ialu_reg_reg);
 8199 %}
 8200 
 8201 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8202 %{
 8203   predicate(UseAPX);
 8204   match(Set dst (SubI src1 src2));
 8205   effect(KILL cr);
 8206   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8207 
 8208   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8209   ins_encode %{
 8210     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8211   %}
 8212   ins_pipe(ialu_reg_reg);
 8213 %}
 8214 
 8215 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8216 %{
 8217   predicate(UseAPX);
 8218   match(Set dst (SubI src1 src2));
 8219   effect(KILL cr);
 8220   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8221 
 8222   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8223   ins_encode %{
 8224     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8225   %}
 8226   ins_pipe(ialu_reg_reg);
 8227 %}
 8228 
 8229 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8230 %{
 8231   predicate(UseAPX);
 8232   match(Set dst (SubI (LoadI src1) src2));
 8233   effect(KILL cr);
 8234   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8235 
 8236   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8237   ins_encode %{
 8238     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8239   %}
 8240   ins_pipe(ialu_reg_reg);
 8241 %}
 8242 
 8243 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8244 %{
 8245   predicate(!UseAPX);
 8246   match(Set dst (SubI dst (LoadI src)));
 8247   effect(KILL cr);
 8248   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8249 
 8250   ins_cost(150);
 8251   format %{ "subl    $dst, $src\t# int" %}
 8252   ins_encode %{
 8253     __ subl($dst$$Register, $src$$Address);
 8254   %}
 8255   ins_pipe(ialu_reg_mem);
 8256 %}
 8257 
 8258 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8259 %{
 8260   predicate(UseAPX);
 8261   match(Set dst (SubI src1 (LoadI src2)));
 8262   effect(KILL cr);
 8263   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8264 
 8265   ins_cost(150);
 8266   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8267   ins_encode %{
 8268     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8269   %}
 8270   ins_pipe(ialu_reg_mem);
 8271 %}
 8272 
 8273 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8274 %{
 8275   predicate(UseAPX);
 8276   match(Set dst (SubI (LoadI src1) src2));
 8277   effect(KILL cr);
 8278   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8279 
 8280   ins_cost(150);
 8281   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8282   ins_encode %{
 8283     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8284   %}
 8285   ins_pipe(ialu_reg_mem);
 8286 %}
 8287 
 8288 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8289 %{
 8290   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8291   effect(KILL cr);
 8292   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8293 
 8294   ins_cost(150);
 8295   format %{ "subl    $dst, $src\t# int" %}
 8296   ins_encode %{
 8297     __ subl($dst$$Address, $src$$Register);
 8298   %}
 8299   ins_pipe(ialu_mem_reg);
 8300 %}
 8301 
 8302 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8303 %{
 8304   predicate(!UseAPX);
 8305   match(Set dst (SubL dst src));
 8306   effect(KILL cr);
 8307   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8308 
 8309   format %{ "subq    $dst, $src\t# long" %}
 8310   ins_encode %{
 8311     __ subq($dst$$Register, $src$$Register);
 8312   %}
 8313   ins_pipe(ialu_reg_reg);
 8314 %}
 8315 
 8316 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8317 %{
 8318   predicate(UseAPX);
 8319   match(Set dst (SubL src1 src2));
 8320   effect(KILL cr);
 8321   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8322 
 8323   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8324   ins_encode %{
 8325     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8326   %}
 8327   ins_pipe(ialu_reg_reg);
 8328 %}
 8329 
 8330 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8331 %{
 8332   predicate(UseAPX);
 8333   match(Set dst (SubL src1 src2));
 8334   effect(KILL cr);
 8335   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8336 
 8337   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8338   ins_encode %{
 8339     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8340   %}
 8341   ins_pipe(ialu_reg_reg);
 8342 %}
 8343 
 8344 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8345 %{
 8346   predicate(UseAPX);
 8347   match(Set dst (SubL (LoadL src1) src2));
 8348   effect(KILL cr);
 8349   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8350 
 8351   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8352   ins_encode %{
 8353     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8354   %}
 8355   ins_pipe(ialu_reg_reg);
 8356 %}
 8357 
 8358 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8359 %{
 8360   predicate(!UseAPX);
 8361   match(Set dst (SubL dst (LoadL src)));
 8362   effect(KILL cr);
 8363   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8364 
 8365   ins_cost(150);
 8366   format %{ "subq    $dst, $src\t# long" %}
 8367   ins_encode %{
 8368     __ subq($dst$$Register, $src$$Address);
 8369   %}
 8370   ins_pipe(ialu_reg_mem);
 8371 %}
 8372 
 8373 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8374 %{
 8375   predicate(UseAPX);
 8376   match(Set dst (SubL src1 (LoadL src2)));
 8377   effect(KILL cr);
 8378   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8379 
 8380   ins_cost(150);
 8381   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8382   ins_encode %{
 8383     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8384   %}
 8385   ins_pipe(ialu_reg_mem);
 8386 %}
 8387 
 8388 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8389 %{
 8390   predicate(UseAPX);
 8391   match(Set dst (SubL (LoadL src1) src2));
 8392   effect(KILL cr);
 8393   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8394 
 8395   ins_cost(150);
 8396   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8397   ins_encode %{
 8398     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8399   %}
 8400   ins_pipe(ialu_reg_mem);
 8401 %}
 8402 
 8403 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8404 %{
 8405   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8406   effect(KILL cr);
 8407   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8408 
 8409   ins_cost(150);
 8410   format %{ "subq    $dst, $src\t# long" %}
 8411   ins_encode %{
 8412     __ subq($dst$$Address, $src$$Register);
 8413   %}
 8414   ins_pipe(ialu_mem_reg);
 8415 %}
 8416 
 8417 // Subtract from a pointer
 8418 // XXX hmpf???
 8419 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8420 %{
 8421   match(Set dst (AddP dst (SubI zero src)));
 8422   effect(KILL cr);
 8423 
 8424   format %{ "subq    $dst, $src\t# ptr - int" %}
 8425   ins_encode %{
 8426     __ subq($dst$$Register, $src$$Register);
 8427   %}
 8428   ins_pipe(ialu_reg_reg);
 8429 %}
 8430 
 8431 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8432 %{
 8433   predicate(!UseAPX);
 8434   match(Set dst (SubI zero dst));
 8435   effect(KILL cr);
 8436   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8437 
 8438   format %{ "negl    $dst\t# int" %}
 8439   ins_encode %{
 8440     __ negl($dst$$Register);
 8441   %}
 8442   ins_pipe(ialu_reg);
 8443 %}
 8444 
 8445 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8446 %{
 8447   predicate(UseAPX);
 8448   match(Set dst (SubI zero src));
 8449   effect(KILL cr);
 8450   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8451 
 8452   format %{ "enegl    $dst, $src\t# int ndd" %}
 8453   ins_encode %{
 8454     __ enegl($dst$$Register, $src$$Register, false);
 8455   %}
 8456   ins_pipe(ialu_reg);
 8457 %}
 8458 
 8459 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8460 %{
 8461   predicate(!UseAPX);
 8462   match(Set dst (NegI dst));
 8463   effect(KILL cr);
 8464   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8465 
 8466   format %{ "negl    $dst\t# int" %}
 8467   ins_encode %{
 8468     __ negl($dst$$Register);
 8469   %}
 8470   ins_pipe(ialu_reg);
 8471 %}
 8472 
 8473 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8474 %{
 8475   predicate(UseAPX);
 8476   match(Set dst (NegI src));
 8477   effect(KILL cr);
 8478   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8479 
 8480   format %{ "enegl    $dst, $src\t# int ndd" %}
 8481   ins_encode %{
 8482     __ enegl($dst$$Register, $src$$Register, false);
 8483   %}
 8484   ins_pipe(ialu_reg);
 8485 %}
 8486 
 8487 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8488 %{
 8489   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8490   effect(KILL cr);
 8491   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8492 
 8493   format %{ "negl    $dst\t# int" %}
 8494   ins_encode %{
 8495     __ negl($dst$$Address);
 8496   %}
 8497   ins_pipe(ialu_reg);
 8498 %}
 8499 
 8500 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8501 %{
 8502   predicate(!UseAPX);
 8503   match(Set dst (SubL zero dst));
 8504   effect(KILL cr);
 8505   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8506 
 8507   format %{ "negq    $dst\t# long" %}
 8508   ins_encode %{
 8509     __ negq($dst$$Register);
 8510   %}
 8511   ins_pipe(ialu_reg);
 8512 %}
 8513 
 8514 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8515 %{
 8516   predicate(UseAPX);
 8517   match(Set dst (SubL zero src));
 8518   effect(KILL cr);
 8519   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8520 
 8521   format %{ "enegq    $dst, $src\t# long ndd" %}
 8522   ins_encode %{
 8523     __ enegq($dst$$Register, $src$$Register, false);
 8524   %}
 8525   ins_pipe(ialu_reg);
 8526 %}
 8527 
 8528 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8529 %{
 8530   predicate(!UseAPX);
 8531   match(Set dst (NegL dst));
 8532   effect(KILL cr);
 8533   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8534 
 8535   format %{ "negq    $dst\t# int" %}
 8536   ins_encode %{
 8537     __ negq($dst$$Register);
 8538   %}
 8539   ins_pipe(ialu_reg);
 8540 %}
 8541 
 8542 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8543 %{
 8544   predicate(UseAPX);
 8545   match(Set dst (NegL src));
 8546   effect(KILL cr);
 8547   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8548 
 8549   format %{ "enegq    $dst, $src\t# long ndd" %}
 8550   ins_encode %{
 8551     __ enegq($dst$$Register, $src$$Register, false);
 8552   %}
 8553   ins_pipe(ialu_reg);
 8554 %}
 8555 
 8556 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8557 %{
 8558   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8559   effect(KILL cr);
 8560   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8561 
 8562   format %{ "negq    $dst\t# long" %}
 8563   ins_encode %{
 8564     __ negq($dst$$Address);
 8565   %}
 8566   ins_pipe(ialu_reg);
 8567 %}
 8568 
 8569 //----------Multiplication/Division Instructions-------------------------------
 8570 // Integer Multiplication Instructions
 8571 // Multiply Register
 8572 
 8573 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8574 %{
 8575   predicate(!UseAPX);
 8576   match(Set dst (MulI dst src));
 8577   effect(KILL cr);
 8578 
 8579   ins_cost(300);
 8580   format %{ "imull   $dst, $src\t# int" %}
 8581   ins_encode %{
 8582     __ imull($dst$$Register, $src$$Register);
 8583   %}
 8584   ins_pipe(ialu_reg_reg_alu0);
 8585 %}
 8586 
 8587 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8588 %{
 8589   predicate(UseAPX);
 8590   match(Set dst (MulI src1 src2));
 8591   effect(KILL cr);
 8592 
 8593   ins_cost(300);
 8594   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8595   ins_encode %{
 8596     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8597   %}
 8598   ins_pipe(ialu_reg_reg_alu0);
 8599 %}
 8600 
 8601 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8602 %{
 8603   predicate(!UseAPX);
 8604   match(Set dst (MulI src imm));
 8605   effect(KILL cr);
 8606 
 8607   ins_cost(300);
 8608   format %{ "imull   $dst, $src, $imm\t# int" %}
 8609   ins_encode %{
 8610     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8611   %}
 8612   ins_pipe(ialu_reg_reg_alu0);
 8613 %}
 8614 
 8615 instruct mulI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8616 %{
 8617   predicate(UseAPX);
 8618   match(Set dst (MulI src1 src2));
 8619   effect(KILL cr);
 8620 
 8621   ins_cost(300);
 8622   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8623   ins_encode %{
 8624     __ eimull($dst$$Register, $src1$$Register, $src2$$constant, false);
 8625   %}
 8626   ins_pipe(ialu_reg_reg_alu0);
 8627 %}
 8628 
 8629 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8630 %{
 8631   predicate(!UseAPX);
 8632   match(Set dst (MulI dst (LoadI src)));
 8633   effect(KILL cr);
 8634 
 8635   ins_cost(350);
 8636   format %{ "imull   $dst, $src\t# int" %}
 8637   ins_encode %{
 8638     __ imull($dst$$Register, $src$$Address);
 8639   %}
 8640   ins_pipe(ialu_reg_mem_alu0);
 8641 %}
 8642 
 8643 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8644 %{
 8645   predicate(UseAPX);
 8646   match(Set dst (MulI src1 (LoadI src2)));
 8647   effect(KILL cr);
 8648 
 8649   ins_cost(350);
 8650   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8651   ins_encode %{
 8652     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8653   %}
 8654   ins_pipe(ialu_reg_mem_alu0);
 8655 %}
 8656 
 8657 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8658 %{
 8659   predicate(!UseAPX);
 8660   match(Set dst (MulI (LoadI src) imm));
 8661   effect(KILL cr);
 8662 
 8663   ins_cost(300);
 8664   format %{ "imull   $dst, $src, $imm\t# int" %}
 8665   ins_encode %{
 8666     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8667   %}
 8668   ins_pipe(ialu_reg_mem_alu0);
 8669 %}
 8670 
 8671 instruct mulI_rReg_mem_imm(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8672 %{
 8673   predicate(UseAPX);
 8674   match(Set dst (MulI (LoadI src1) src2));
 8675   effect(KILL cr);
 8676 
 8677   ins_cost(300);
 8678   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8679   ins_encode %{
 8680     __ eimull($dst$$Register, $src1$$Address, $src2$$constant, false);
 8681   %}
 8682   ins_pipe(ialu_reg_mem_alu0);
 8683 %}
 8684 
 8685 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8686 %{
 8687   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8688   effect(KILL cr, KILL src2);
 8689 
 8690   expand %{ mulI_rReg(dst, src1, cr);
 8691            mulI_rReg(src2, src3, cr);
 8692            addI_rReg(dst, src2, cr); %}
 8693 %}
 8694 
 8695 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8696 %{
 8697   predicate(!UseAPX);
 8698   match(Set dst (MulL dst src));
 8699   effect(KILL cr);
 8700 
 8701   ins_cost(300);
 8702   format %{ "imulq   $dst, $src\t# long" %}
 8703   ins_encode %{
 8704     __ imulq($dst$$Register, $src$$Register);
 8705   %}
 8706   ins_pipe(ialu_reg_reg_alu0);
 8707 %}
 8708 
 8709 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8710 %{
 8711   predicate(UseAPX);
 8712   match(Set dst (MulL src1 src2));
 8713   effect(KILL cr);
 8714 
 8715   ins_cost(300);
 8716   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8717   ins_encode %{
 8718     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8719   %}
 8720   ins_pipe(ialu_reg_reg_alu0);
 8721 %}
 8722 
 8723 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8724 %{
 8725   predicate(!UseAPX);
 8726   match(Set dst (MulL src imm));
 8727   effect(KILL cr);
 8728 
 8729   ins_cost(300);
 8730   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8731   ins_encode %{
 8732     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8733   %}
 8734   ins_pipe(ialu_reg_reg_alu0);
 8735 %}
 8736 
 8737 instruct mulL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8738 %{
 8739   predicate(UseAPX);
 8740   match(Set dst (MulL src1 src2));
 8741   effect(KILL cr);
 8742 
 8743   ins_cost(300);
 8744   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8745   ins_encode %{
 8746     __ eimulq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8747   %}
 8748   ins_pipe(ialu_reg_reg_alu0);
 8749 %}
 8750 
 8751 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8752 %{
 8753   predicate(!UseAPX);
 8754   match(Set dst (MulL dst (LoadL src)));
 8755   effect(KILL cr);
 8756 
 8757   ins_cost(350);
 8758   format %{ "imulq   $dst, $src\t# long" %}
 8759   ins_encode %{
 8760     __ imulq($dst$$Register, $src$$Address);
 8761   %}
 8762   ins_pipe(ialu_reg_mem_alu0);
 8763 %}
 8764 
 8765 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8766 %{
 8767   predicate(UseAPX);
 8768   match(Set dst (MulL src1 (LoadL src2)));
 8769   effect(KILL cr);
 8770 
 8771   ins_cost(350);
 8772   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8773   ins_encode %{
 8774     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8775   %}
 8776   ins_pipe(ialu_reg_mem_alu0);
 8777 %}
 8778 
 8779 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8780 %{
 8781   predicate(!UseAPX);
 8782   match(Set dst (MulL (LoadL src) imm));
 8783   effect(KILL cr);
 8784 
 8785   ins_cost(300);
 8786   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8787   ins_encode %{
 8788     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8789   %}
 8790   ins_pipe(ialu_reg_mem_alu0);
 8791 %}
 8792 
 8793 instruct mulL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8794 %{
 8795   predicate(UseAPX);
 8796   match(Set dst (MulL (LoadL src1) src2));
 8797   effect(KILL cr);
 8798 
 8799   ins_cost(300);
 8800   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8801   ins_encode %{
 8802     __ eimulq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8803   %}
 8804   ins_pipe(ialu_reg_mem_alu0);
 8805 %}
 8806 
 8807 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8808 %{
 8809   match(Set dst (MulHiL src rax));
 8810   effect(USE_KILL rax, KILL cr);
 8811 
 8812   ins_cost(300);
 8813   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8814   ins_encode %{
 8815     __ imulq($src$$Register);
 8816   %}
 8817   ins_pipe(ialu_reg_reg_alu0);
 8818 %}
 8819 
 8820 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8821 %{
 8822   match(Set dst (UMulHiL src rax));
 8823   effect(USE_KILL rax, KILL cr);
 8824 
 8825   ins_cost(300);
 8826   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8827   ins_encode %{
 8828     __ mulq($src$$Register);
 8829   %}
 8830   ins_pipe(ialu_reg_reg_alu0);
 8831 %}
 8832 
 8833 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8834                    rFlagsReg cr)
 8835 %{
 8836   match(Set rax (DivI rax div));
 8837   effect(KILL rdx, KILL cr);
 8838 
 8839   ins_cost(30*100+10*100); // XXX
 8840   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8841             "jne,s   normal\n\t"
 8842             "xorl    rdx, rdx\n\t"
 8843             "cmpl    $div, -1\n\t"
 8844             "je,s    done\n"
 8845     "normal: cdql\n\t"
 8846             "idivl   $div\n"
 8847     "done:"        %}
 8848   ins_encode(cdql_enc(div));
 8849   ins_pipe(ialu_reg_reg_alu0);
 8850 %}
 8851 
 8852 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8853                    rFlagsReg cr)
 8854 %{
 8855   match(Set rax (DivL rax div));
 8856   effect(KILL rdx, KILL cr);
 8857 
 8858   ins_cost(30*100+10*100); // XXX
 8859   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8860             "cmpq    rax, rdx\n\t"
 8861             "jne,s   normal\n\t"
 8862             "xorl    rdx, rdx\n\t"
 8863             "cmpq    $div, -1\n\t"
 8864             "je,s    done\n"
 8865     "normal: cdqq\n\t"
 8866             "idivq   $div\n"
 8867     "done:"        %}
 8868   ins_encode(cdqq_enc(div));
 8869   ins_pipe(ialu_reg_reg_alu0);
 8870 %}
 8871 
 8872 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8873 %{
 8874   match(Set rax (UDivI rax div));
 8875   effect(KILL rdx, KILL cr);
 8876 
 8877   ins_cost(300);
 8878   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8879   ins_encode %{
 8880     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8881   %}
 8882   ins_pipe(ialu_reg_reg_alu0);
 8883 %}
 8884 
 8885 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8886 %{
 8887   match(Set rax (UDivL rax div));
 8888   effect(KILL rdx, KILL cr);
 8889 
 8890   ins_cost(300);
 8891   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8892   ins_encode %{
 8893      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8894   %}
 8895   ins_pipe(ialu_reg_reg_alu0);
 8896 %}
 8897 
 8898 // Integer DIVMOD with Register, both quotient and mod results
 8899 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8900                              rFlagsReg cr)
 8901 %{
 8902   match(DivModI rax div);
 8903   effect(KILL cr);
 8904 
 8905   ins_cost(30*100+10*100); // XXX
 8906   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8907             "jne,s   normal\n\t"
 8908             "xorl    rdx, rdx\n\t"
 8909             "cmpl    $div, -1\n\t"
 8910             "je,s    done\n"
 8911     "normal: cdql\n\t"
 8912             "idivl   $div\n"
 8913     "done:"        %}
 8914   ins_encode(cdql_enc(div));
 8915   ins_pipe(pipe_slow);
 8916 %}
 8917 
 8918 // Long DIVMOD with Register, both quotient and mod results
 8919 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8920                              rFlagsReg cr)
 8921 %{
 8922   match(DivModL rax div);
 8923   effect(KILL cr);
 8924 
 8925   ins_cost(30*100+10*100); // XXX
 8926   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8927             "cmpq    rax, rdx\n\t"
 8928             "jne,s   normal\n\t"
 8929             "xorl    rdx, rdx\n\t"
 8930             "cmpq    $div, -1\n\t"
 8931             "je,s    done\n"
 8932     "normal: cdqq\n\t"
 8933             "idivq   $div\n"
 8934     "done:"        %}
 8935   ins_encode(cdqq_enc(div));
 8936   ins_pipe(pipe_slow);
 8937 %}
 8938 
 8939 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8940 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8941                               no_rax_rdx_RegI div, rFlagsReg cr)
 8942 %{
 8943   match(UDivModI rax div);
 8944   effect(TEMP tmp, KILL cr);
 8945 
 8946   ins_cost(300);
 8947   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8948             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8949           %}
 8950   ins_encode %{
 8951     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8952   %}
 8953   ins_pipe(pipe_slow);
 8954 %}
 8955 
 8956 // Unsigned long DIVMOD with Register, both quotient and mod results
 8957 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8958                               no_rax_rdx_RegL div, rFlagsReg cr)
 8959 %{
 8960   match(UDivModL rax div);
 8961   effect(TEMP tmp, KILL cr);
 8962 
 8963   ins_cost(300);
 8964   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8965             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8966           %}
 8967   ins_encode %{
 8968     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8969   %}
 8970   ins_pipe(pipe_slow);
 8971 %}
 8972 
 8973 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8974                    rFlagsReg cr)
 8975 %{
 8976   match(Set rdx (ModI rax div));
 8977   effect(KILL rax, KILL cr);
 8978 
 8979   ins_cost(300); // XXX
 8980   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8981             "jne,s   normal\n\t"
 8982             "xorl    rdx, rdx\n\t"
 8983             "cmpl    $div, -1\n\t"
 8984             "je,s    done\n"
 8985     "normal: cdql\n\t"
 8986             "idivl   $div\n"
 8987     "done:"        %}
 8988   ins_encode(cdql_enc(div));
 8989   ins_pipe(ialu_reg_reg_alu0);
 8990 %}
 8991 
 8992 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8993                    rFlagsReg cr)
 8994 %{
 8995   match(Set rdx (ModL rax div));
 8996   effect(KILL rax, KILL cr);
 8997 
 8998   ins_cost(300); // XXX
 8999   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 9000             "cmpq    rax, rdx\n\t"
 9001             "jne,s   normal\n\t"
 9002             "xorl    rdx, rdx\n\t"
 9003             "cmpq    $div, -1\n\t"
 9004             "je,s    done\n"
 9005     "normal: cdqq\n\t"
 9006             "idivq   $div\n"
 9007     "done:"        %}
 9008   ins_encode(cdqq_enc(div));
 9009   ins_pipe(ialu_reg_reg_alu0);
 9010 %}
 9011 
 9012 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 9013 %{
 9014   match(Set rdx (UModI rax div));
 9015   effect(KILL rax, KILL cr);
 9016 
 9017   ins_cost(300);
 9018   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 9019   ins_encode %{
 9020     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 9021   %}
 9022   ins_pipe(ialu_reg_reg_alu0);
 9023 %}
 9024 
 9025 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 9026 %{
 9027   match(Set rdx (UModL rax div));
 9028   effect(KILL rax, KILL cr);
 9029 
 9030   ins_cost(300);
 9031   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9032   ins_encode %{
 9033     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9034   %}
 9035   ins_pipe(ialu_reg_reg_alu0);
 9036 %}
 9037 
 9038 // Integer Shift Instructions
 9039 // Shift Left by one, two, three
 9040 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9041 %{
 9042   predicate(!UseAPX);
 9043   match(Set dst (LShiftI dst shift));
 9044   effect(KILL cr);
 9045 
 9046   format %{ "sall    $dst, $shift" %}
 9047   ins_encode %{
 9048     __ sall($dst$$Register, $shift$$constant);
 9049   %}
 9050   ins_pipe(ialu_reg);
 9051 %}
 9052 
 9053 // Shift Left by one, two, three
 9054 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9055 %{
 9056   predicate(UseAPX);
 9057   match(Set dst (LShiftI src shift));
 9058   effect(KILL cr);
 9059 
 9060   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9061   ins_encode %{
 9062     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9063   %}
 9064   ins_pipe(ialu_reg);
 9065 %}
 9066 
 9067 // Shift Left by 8-bit immediate
 9068 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9069 %{
 9070   predicate(!UseAPX);
 9071   match(Set dst (LShiftI dst shift));
 9072   effect(KILL cr);
 9073 
 9074   format %{ "sall    $dst, $shift" %}
 9075   ins_encode %{
 9076     __ sall($dst$$Register, $shift$$constant);
 9077   %}
 9078   ins_pipe(ialu_reg);
 9079 %}
 9080 
 9081 // Shift Left by 8-bit immediate
 9082 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9083 %{
 9084   predicate(UseAPX);
 9085   match(Set dst (LShiftI src shift));
 9086   effect(KILL cr);
 9087 
 9088   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9089   ins_encode %{
 9090     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9091   %}
 9092   ins_pipe(ialu_reg);
 9093 %}
 9094 
 9095 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9096 %{
 9097   predicate(UseAPX);
 9098   match(Set dst (LShiftI (LoadI src) shift));
 9099   effect(KILL cr);
 9100 
 9101   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9102   ins_encode %{
 9103     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9104   %}
 9105   ins_pipe(ialu_reg);
 9106 %}
 9107 
 9108 // Shift Left by 8-bit immediate
 9109 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9110 %{
 9111   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9112   effect(KILL cr);
 9113 
 9114   format %{ "sall    $dst, $shift" %}
 9115   ins_encode %{
 9116     __ sall($dst$$Address, $shift$$constant);
 9117   %}
 9118   ins_pipe(ialu_mem_imm);
 9119 %}
 9120 
 9121 // Shift Left by variable
 9122 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9123 %{
 9124   predicate(!VM_Version::supports_bmi2());
 9125   match(Set dst (LShiftI dst shift));
 9126   effect(KILL cr);
 9127 
 9128   format %{ "sall    $dst, $shift" %}
 9129   ins_encode %{
 9130     __ sall($dst$$Register);
 9131   %}
 9132   ins_pipe(ialu_reg_reg);
 9133 %}
 9134 
 9135 // Shift Left by variable
 9136 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9137 %{
 9138   predicate(!VM_Version::supports_bmi2());
 9139   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9140   effect(KILL cr);
 9141 
 9142   format %{ "sall    $dst, $shift" %}
 9143   ins_encode %{
 9144     __ sall($dst$$Address);
 9145   %}
 9146   ins_pipe(ialu_mem_reg);
 9147 %}
 9148 
 9149 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9150 %{
 9151   predicate(VM_Version::supports_bmi2());
 9152   match(Set dst (LShiftI src shift));
 9153 
 9154   format %{ "shlxl   $dst, $src, $shift" %}
 9155   ins_encode %{
 9156     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9157   %}
 9158   ins_pipe(ialu_reg_reg);
 9159 %}
 9160 
 9161 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9162 %{
 9163   predicate(VM_Version::supports_bmi2());
 9164   match(Set dst (LShiftI (LoadI src) shift));
 9165   ins_cost(175);
 9166   format %{ "shlxl   $dst, $src, $shift" %}
 9167   ins_encode %{
 9168     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9169   %}
 9170   ins_pipe(ialu_reg_mem);
 9171 %}
 9172 
 9173 // Arithmetic Shift Right by 8-bit immediate
 9174 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9175 %{
 9176   predicate(!UseAPX);
 9177   match(Set dst (RShiftI dst shift));
 9178   effect(KILL cr);
 9179 
 9180   format %{ "sarl    $dst, $shift" %}
 9181   ins_encode %{
 9182     __ sarl($dst$$Register, $shift$$constant);
 9183   %}
 9184   ins_pipe(ialu_mem_imm);
 9185 %}
 9186 
 9187 // Arithmetic Shift Right by 8-bit immediate
 9188 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9189 %{
 9190   predicate(UseAPX);
 9191   match(Set dst (RShiftI src shift));
 9192   effect(KILL cr);
 9193 
 9194   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9195   ins_encode %{
 9196     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9197   %}
 9198   ins_pipe(ialu_mem_imm);
 9199 %}
 9200 
 9201 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9202 %{
 9203   predicate(UseAPX);
 9204   match(Set dst (RShiftI (LoadI src) shift));
 9205   effect(KILL cr);
 9206 
 9207   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9208   ins_encode %{
 9209     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9210   %}
 9211   ins_pipe(ialu_mem_imm);
 9212 %}
 9213 
 9214 // Arithmetic Shift Right by 8-bit immediate
 9215 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9216 %{
 9217   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9218   effect(KILL cr);
 9219 
 9220   format %{ "sarl    $dst, $shift" %}
 9221   ins_encode %{
 9222     __ sarl($dst$$Address, $shift$$constant);
 9223   %}
 9224   ins_pipe(ialu_mem_imm);
 9225 %}
 9226 
 9227 // Arithmetic Shift Right by variable
 9228 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9229 %{
 9230   predicate(!VM_Version::supports_bmi2());
 9231   match(Set dst (RShiftI dst shift));
 9232   effect(KILL cr);
 9233 
 9234   format %{ "sarl    $dst, $shift" %}
 9235   ins_encode %{
 9236     __ sarl($dst$$Register);
 9237   %}
 9238   ins_pipe(ialu_reg_reg);
 9239 %}
 9240 
 9241 // Arithmetic Shift Right by variable
 9242 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9243 %{
 9244   predicate(!VM_Version::supports_bmi2());
 9245   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9246   effect(KILL cr);
 9247 
 9248   format %{ "sarl    $dst, $shift" %}
 9249   ins_encode %{
 9250     __ sarl($dst$$Address);
 9251   %}
 9252   ins_pipe(ialu_mem_reg);
 9253 %}
 9254 
 9255 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9256 %{
 9257   predicate(VM_Version::supports_bmi2());
 9258   match(Set dst (RShiftI src shift));
 9259 
 9260   format %{ "sarxl   $dst, $src, $shift" %}
 9261   ins_encode %{
 9262     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9263   %}
 9264   ins_pipe(ialu_reg_reg);
 9265 %}
 9266 
 9267 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9268 %{
 9269   predicate(VM_Version::supports_bmi2());
 9270   match(Set dst (RShiftI (LoadI src) shift));
 9271   ins_cost(175);
 9272   format %{ "sarxl   $dst, $src, $shift" %}
 9273   ins_encode %{
 9274     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9275   %}
 9276   ins_pipe(ialu_reg_mem);
 9277 %}
 9278 
 9279 // Logical Shift Right by 8-bit immediate
 9280 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9281 %{
 9282   predicate(!UseAPX);
 9283   match(Set dst (URShiftI dst shift));
 9284   effect(KILL cr);
 9285 
 9286   format %{ "shrl    $dst, $shift" %}
 9287   ins_encode %{
 9288     __ shrl($dst$$Register, $shift$$constant);
 9289   %}
 9290   ins_pipe(ialu_reg);
 9291 %}
 9292 
 9293 // Logical Shift Right by 8-bit immediate
 9294 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9295 %{
 9296   predicate(UseAPX);
 9297   match(Set dst (URShiftI src shift));
 9298   effect(KILL cr);
 9299 
 9300   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9301   ins_encode %{
 9302     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9303   %}
 9304   ins_pipe(ialu_reg);
 9305 %}
 9306 
 9307 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9308 %{
 9309   predicate(UseAPX);
 9310   match(Set dst (URShiftI (LoadI src) shift));
 9311   effect(KILL cr);
 9312 
 9313   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9314   ins_encode %{
 9315     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9316   %}
 9317   ins_pipe(ialu_reg);
 9318 %}
 9319 
 9320 // Logical Shift Right by 8-bit immediate
 9321 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9322 %{
 9323   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9324   effect(KILL cr);
 9325 
 9326   format %{ "shrl    $dst, $shift" %}
 9327   ins_encode %{
 9328     __ shrl($dst$$Address, $shift$$constant);
 9329   %}
 9330   ins_pipe(ialu_mem_imm);
 9331 %}
 9332 
 9333 // Logical Shift Right by variable
 9334 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9335 %{
 9336   predicate(!VM_Version::supports_bmi2());
 9337   match(Set dst (URShiftI dst shift));
 9338   effect(KILL cr);
 9339 
 9340   format %{ "shrl    $dst, $shift" %}
 9341   ins_encode %{
 9342     __ shrl($dst$$Register);
 9343   %}
 9344   ins_pipe(ialu_reg_reg);
 9345 %}
 9346 
 9347 // Logical Shift Right by variable
 9348 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9349 %{
 9350   predicate(!VM_Version::supports_bmi2());
 9351   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9352   effect(KILL cr);
 9353 
 9354   format %{ "shrl    $dst, $shift" %}
 9355   ins_encode %{
 9356     __ shrl($dst$$Address);
 9357   %}
 9358   ins_pipe(ialu_mem_reg);
 9359 %}
 9360 
 9361 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9362 %{
 9363   predicate(VM_Version::supports_bmi2());
 9364   match(Set dst (URShiftI src shift));
 9365 
 9366   format %{ "shrxl   $dst, $src, $shift" %}
 9367   ins_encode %{
 9368     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9369   %}
 9370   ins_pipe(ialu_reg_reg);
 9371 %}
 9372 
 9373 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9374 %{
 9375   predicate(VM_Version::supports_bmi2());
 9376   match(Set dst (URShiftI (LoadI src) shift));
 9377   ins_cost(175);
 9378   format %{ "shrxl   $dst, $src, $shift" %}
 9379   ins_encode %{
 9380     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9381   %}
 9382   ins_pipe(ialu_reg_mem);
 9383 %}
 9384 
 9385 // Long Shift Instructions
 9386 // Shift Left by one, two, three
 9387 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9388 %{
 9389   predicate(!UseAPX);
 9390   match(Set dst (LShiftL dst shift));
 9391   effect(KILL cr);
 9392 
 9393   format %{ "salq    $dst, $shift" %}
 9394   ins_encode %{
 9395     __ salq($dst$$Register, $shift$$constant);
 9396   %}
 9397   ins_pipe(ialu_reg);
 9398 %}
 9399 
 9400 // Shift Left by one, two, three
 9401 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9402 %{
 9403   predicate(UseAPX);
 9404   match(Set dst (LShiftL src shift));
 9405   effect(KILL cr);
 9406 
 9407   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9408   ins_encode %{
 9409     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9410   %}
 9411   ins_pipe(ialu_reg);
 9412 %}
 9413 
 9414 // Shift Left by 8-bit immediate
 9415 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9416 %{
 9417   predicate(!UseAPX);
 9418   match(Set dst (LShiftL dst shift));
 9419   effect(KILL cr);
 9420 
 9421   format %{ "salq    $dst, $shift" %}
 9422   ins_encode %{
 9423     __ salq($dst$$Register, $shift$$constant);
 9424   %}
 9425   ins_pipe(ialu_reg);
 9426 %}
 9427 
 9428 // Shift Left by 8-bit immediate
 9429 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9430 %{
 9431   predicate(UseAPX);
 9432   match(Set dst (LShiftL src shift));
 9433   effect(KILL cr);
 9434 
 9435   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9436   ins_encode %{
 9437     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9438   %}
 9439   ins_pipe(ialu_reg);
 9440 %}
 9441 
 9442 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9443 %{
 9444   predicate(UseAPX);
 9445   match(Set dst (LShiftL (LoadL src) shift));
 9446   effect(KILL cr);
 9447 
 9448   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9449   ins_encode %{
 9450     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9451   %}
 9452   ins_pipe(ialu_reg);
 9453 %}
 9454 
 9455 // Shift Left by 8-bit immediate
 9456 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9457 %{
 9458   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9459   effect(KILL cr);
 9460 
 9461   format %{ "salq    $dst, $shift" %}
 9462   ins_encode %{
 9463     __ salq($dst$$Address, $shift$$constant);
 9464   %}
 9465   ins_pipe(ialu_mem_imm);
 9466 %}
 9467 
 9468 // Shift Left by variable
 9469 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9470 %{
 9471   predicate(!VM_Version::supports_bmi2());
 9472   match(Set dst (LShiftL dst shift));
 9473   effect(KILL cr);
 9474 
 9475   format %{ "salq    $dst, $shift" %}
 9476   ins_encode %{
 9477     __ salq($dst$$Register);
 9478   %}
 9479   ins_pipe(ialu_reg_reg);
 9480 %}
 9481 
 9482 // Shift Left by variable
 9483 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9484 %{
 9485   predicate(!VM_Version::supports_bmi2());
 9486   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9487   effect(KILL cr);
 9488 
 9489   format %{ "salq    $dst, $shift" %}
 9490   ins_encode %{
 9491     __ salq($dst$$Address);
 9492   %}
 9493   ins_pipe(ialu_mem_reg);
 9494 %}
 9495 
 9496 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9497 %{
 9498   predicate(VM_Version::supports_bmi2());
 9499   match(Set dst (LShiftL src shift));
 9500 
 9501   format %{ "shlxq   $dst, $src, $shift" %}
 9502   ins_encode %{
 9503     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9504   %}
 9505   ins_pipe(ialu_reg_reg);
 9506 %}
 9507 
 9508 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9509 %{
 9510   predicate(VM_Version::supports_bmi2());
 9511   match(Set dst (LShiftL (LoadL src) shift));
 9512   ins_cost(175);
 9513   format %{ "shlxq   $dst, $src, $shift" %}
 9514   ins_encode %{
 9515     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9516   %}
 9517   ins_pipe(ialu_reg_mem);
 9518 %}
 9519 
 9520 // Arithmetic Shift Right by 8-bit immediate
 9521 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9522 %{
 9523   predicate(!UseAPX);
 9524   match(Set dst (RShiftL dst shift));
 9525   effect(KILL cr);
 9526 
 9527   format %{ "sarq    $dst, $shift" %}
 9528   ins_encode %{
 9529     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9530   %}
 9531   ins_pipe(ialu_mem_imm);
 9532 %}
 9533 
 9534 // Arithmetic Shift Right by 8-bit immediate
 9535 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9536 %{
 9537   predicate(UseAPX);
 9538   match(Set dst (RShiftL src shift));
 9539   effect(KILL cr);
 9540 
 9541   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9542   ins_encode %{
 9543     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9544   %}
 9545   ins_pipe(ialu_mem_imm);
 9546 %}
 9547 
 9548 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9549 %{
 9550   predicate(UseAPX);
 9551   match(Set dst (RShiftL (LoadL src) shift));
 9552   effect(KILL cr);
 9553 
 9554   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9555   ins_encode %{
 9556     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9557   %}
 9558   ins_pipe(ialu_mem_imm);
 9559 %}
 9560 
 9561 // Arithmetic Shift Right by 8-bit immediate
 9562 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9563 %{
 9564   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9565   effect(KILL cr);
 9566 
 9567   format %{ "sarq    $dst, $shift" %}
 9568   ins_encode %{
 9569     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9570   %}
 9571   ins_pipe(ialu_mem_imm);
 9572 %}
 9573 
 9574 // Arithmetic Shift Right by variable
 9575 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9576 %{
 9577   predicate(!VM_Version::supports_bmi2());
 9578   match(Set dst (RShiftL dst shift));
 9579   effect(KILL cr);
 9580 
 9581   format %{ "sarq    $dst, $shift" %}
 9582   ins_encode %{
 9583     __ sarq($dst$$Register);
 9584   %}
 9585   ins_pipe(ialu_reg_reg);
 9586 %}
 9587 
 9588 // Arithmetic Shift Right by variable
 9589 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9590 %{
 9591   predicate(!VM_Version::supports_bmi2());
 9592   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9593   effect(KILL cr);
 9594 
 9595   format %{ "sarq    $dst, $shift" %}
 9596   ins_encode %{
 9597     __ sarq($dst$$Address);
 9598   %}
 9599   ins_pipe(ialu_mem_reg);
 9600 %}
 9601 
 9602 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9603 %{
 9604   predicate(VM_Version::supports_bmi2());
 9605   match(Set dst (RShiftL src shift));
 9606 
 9607   format %{ "sarxq   $dst, $src, $shift" %}
 9608   ins_encode %{
 9609     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9610   %}
 9611   ins_pipe(ialu_reg_reg);
 9612 %}
 9613 
 9614 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9615 %{
 9616   predicate(VM_Version::supports_bmi2());
 9617   match(Set dst (RShiftL (LoadL src) shift));
 9618   ins_cost(175);
 9619   format %{ "sarxq   $dst, $src, $shift" %}
 9620   ins_encode %{
 9621     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9622   %}
 9623   ins_pipe(ialu_reg_mem);
 9624 %}
 9625 
 9626 // Logical Shift Right by 8-bit immediate
 9627 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9628 %{
 9629   predicate(!UseAPX);
 9630   match(Set dst (URShiftL dst shift));
 9631   effect(KILL cr);
 9632 
 9633   format %{ "shrq    $dst, $shift" %}
 9634   ins_encode %{
 9635     __ shrq($dst$$Register, $shift$$constant);
 9636   %}
 9637   ins_pipe(ialu_reg);
 9638 %}
 9639 
 9640 // Logical Shift Right by 8-bit immediate
 9641 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9642 %{
 9643   predicate(UseAPX);
 9644   match(Set dst (URShiftL src shift));
 9645   effect(KILL cr);
 9646 
 9647   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9648   ins_encode %{
 9649     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9650   %}
 9651   ins_pipe(ialu_reg);
 9652 %}
 9653 
 9654 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9655 %{
 9656   predicate(UseAPX);
 9657   match(Set dst (URShiftL (LoadL src) shift));
 9658   effect(KILL cr);
 9659 
 9660   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9661   ins_encode %{
 9662     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9663   %}
 9664   ins_pipe(ialu_reg);
 9665 %}
 9666 
 9667 // Logical Shift Right by 8-bit immediate
 9668 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9669 %{
 9670   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9671   effect(KILL cr);
 9672 
 9673   format %{ "shrq    $dst, $shift" %}
 9674   ins_encode %{
 9675     __ shrq($dst$$Address, $shift$$constant);
 9676   %}
 9677   ins_pipe(ialu_mem_imm);
 9678 %}
 9679 
 9680 // Logical Shift Right by variable
 9681 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9682 %{
 9683   predicate(!VM_Version::supports_bmi2());
 9684   match(Set dst (URShiftL dst shift));
 9685   effect(KILL cr);
 9686 
 9687   format %{ "shrq    $dst, $shift" %}
 9688   ins_encode %{
 9689     __ shrq($dst$$Register);
 9690   %}
 9691   ins_pipe(ialu_reg_reg);
 9692 %}
 9693 
 9694 // Logical Shift Right by variable
 9695 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9696 %{
 9697   predicate(!VM_Version::supports_bmi2());
 9698   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9699   effect(KILL cr);
 9700 
 9701   format %{ "shrq    $dst, $shift" %}
 9702   ins_encode %{
 9703     __ shrq($dst$$Address);
 9704   %}
 9705   ins_pipe(ialu_mem_reg);
 9706 %}
 9707 
 9708 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9709 %{
 9710   predicate(VM_Version::supports_bmi2());
 9711   match(Set dst (URShiftL src shift));
 9712 
 9713   format %{ "shrxq   $dst, $src, $shift" %}
 9714   ins_encode %{
 9715     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9716   %}
 9717   ins_pipe(ialu_reg_reg);
 9718 %}
 9719 
 9720 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9721 %{
 9722   predicate(VM_Version::supports_bmi2());
 9723   match(Set dst (URShiftL (LoadL src) shift));
 9724   ins_cost(175);
 9725   format %{ "shrxq   $dst, $src, $shift" %}
 9726   ins_encode %{
 9727     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9728   %}
 9729   ins_pipe(ialu_reg_mem);
 9730 %}
 9731 
 9732 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9733 // This idiom is used by the compiler for the i2b bytecode.
 9734 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9735 %{
 9736   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9737 
 9738   format %{ "movsbl  $dst, $src\t# i2b" %}
 9739   ins_encode %{
 9740     __ movsbl($dst$$Register, $src$$Register);
 9741   %}
 9742   ins_pipe(ialu_reg_reg);
 9743 %}
 9744 
 9745 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9746 // This idiom is used by the compiler the i2s bytecode.
 9747 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9748 %{
 9749   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9750 
 9751   format %{ "movswl  $dst, $src\t# i2s" %}
 9752   ins_encode %{
 9753     __ movswl($dst$$Register, $src$$Register);
 9754   %}
 9755   ins_pipe(ialu_reg_reg);
 9756 %}
 9757 
 9758 // ROL/ROR instructions
 9759 
 9760 // Rotate left by constant.
 9761 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9762 %{
 9763   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9764   match(Set dst (RotateLeft dst shift));
 9765   effect(KILL cr);
 9766   format %{ "roll    $dst, $shift" %}
 9767   ins_encode %{
 9768     __ roll($dst$$Register, $shift$$constant);
 9769   %}
 9770   ins_pipe(ialu_reg);
 9771 %}
 9772 
 9773 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9774 %{
 9775   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9776   match(Set dst (RotateLeft src shift));
 9777   format %{ "rolxl   $dst, $src, $shift" %}
 9778   ins_encode %{
 9779     int shift = 32 - ($shift$$constant & 31);
 9780     __ rorxl($dst$$Register, $src$$Register, shift);
 9781   %}
 9782   ins_pipe(ialu_reg_reg);
 9783 %}
 9784 
 9785 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9786 %{
 9787   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9788   match(Set dst (RotateLeft (LoadI src) shift));
 9789   ins_cost(175);
 9790   format %{ "rolxl   $dst, $src, $shift" %}
 9791   ins_encode %{
 9792     int shift = 32 - ($shift$$constant & 31);
 9793     __ rorxl($dst$$Register, $src$$Address, shift);
 9794   %}
 9795   ins_pipe(ialu_reg_mem);
 9796 %}
 9797 
 9798 // Rotate Left by variable
 9799 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9800 %{
 9801   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9802   match(Set dst (RotateLeft dst shift));
 9803   effect(KILL cr);
 9804   format %{ "roll    $dst, $shift" %}
 9805   ins_encode %{
 9806     __ roll($dst$$Register);
 9807   %}
 9808   ins_pipe(ialu_reg_reg);
 9809 %}
 9810 
 9811 // Rotate Left by variable
 9812 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9813 %{
 9814   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9815   match(Set dst (RotateLeft src shift));
 9816   effect(KILL cr);
 9817 
 9818   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9819   ins_encode %{
 9820     __ eroll($dst$$Register, $src$$Register, false);
 9821   %}
 9822   ins_pipe(ialu_reg_reg);
 9823 %}
 9824 
 9825 // Rotate Right by constant.
 9826 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9827 %{
 9828   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9829   match(Set dst (RotateRight dst shift));
 9830   effect(KILL cr);
 9831   format %{ "rorl    $dst, $shift" %}
 9832   ins_encode %{
 9833     __ rorl($dst$$Register, $shift$$constant);
 9834   %}
 9835   ins_pipe(ialu_reg);
 9836 %}
 9837 
 9838 // Rotate Right by constant.
 9839 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9840 %{
 9841   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9842   match(Set dst (RotateRight src shift));
 9843   format %{ "rorxl   $dst, $src, $shift" %}
 9844   ins_encode %{
 9845     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9846   %}
 9847   ins_pipe(ialu_reg_reg);
 9848 %}
 9849 
 9850 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9851 %{
 9852   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9853   match(Set dst (RotateRight (LoadI src) shift));
 9854   ins_cost(175);
 9855   format %{ "rorxl   $dst, $src, $shift" %}
 9856   ins_encode %{
 9857     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9858   %}
 9859   ins_pipe(ialu_reg_mem);
 9860 %}
 9861 
 9862 // Rotate Right by variable
 9863 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9864 %{
 9865   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9866   match(Set dst (RotateRight dst shift));
 9867   effect(KILL cr);
 9868   format %{ "rorl    $dst, $shift" %}
 9869   ins_encode %{
 9870     __ rorl($dst$$Register);
 9871   %}
 9872   ins_pipe(ialu_reg_reg);
 9873 %}
 9874 
 9875 // Rotate Right by variable
 9876 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9877 %{
 9878   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9879   match(Set dst (RotateRight src shift));
 9880   effect(KILL cr);
 9881 
 9882   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9883   ins_encode %{
 9884     __ erorl($dst$$Register, $src$$Register, false);
 9885   %}
 9886   ins_pipe(ialu_reg_reg);
 9887 %}
 9888 
 9889 // Rotate Left by constant.
 9890 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9891 %{
 9892   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9893   match(Set dst (RotateLeft dst shift));
 9894   effect(KILL cr);
 9895   format %{ "rolq    $dst, $shift" %}
 9896   ins_encode %{
 9897     __ rolq($dst$$Register, $shift$$constant);
 9898   %}
 9899   ins_pipe(ialu_reg);
 9900 %}
 9901 
 9902 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9903 %{
 9904   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9905   match(Set dst (RotateLeft src shift));
 9906   format %{ "rolxq   $dst, $src, $shift" %}
 9907   ins_encode %{
 9908     int shift = 64 - ($shift$$constant & 63);
 9909     __ rorxq($dst$$Register, $src$$Register, shift);
 9910   %}
 9911   ins_pipe(ialu_reg_reg);
 9912 %}
 9913 
 9914 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9915 %{
 9916   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9917   match(Set dst (RotateLeft (LoadL src) shift));
 9918   ins_cost(175);
 9919   format %{ "rolxq   $dst, $src, $shift" %}
 9920   ins_encode %{
 9921     int shift = 64 - ($shift$$constant & 63);
 9922     __ rorxq($dst$$Register, $src$$Address, shift);
 9923   %}
 9924   ins_pipe(ialu_reg_mem);
 9925 %}
 9926 
 9927 // Rotate Left by variable
 9928 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9929 %{
 9930   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9931   match(Set dst (RotateLeft dst shift));
 9932   effect(KILL cr);
 9933   format %{ "rolq    $dst, $shift" %}
 9934   ins_encode %{
 9935     __ rolq($dst$$Register);
 9936   %}
 9937   ins_pipe(ialu_reg_reg);
 9938 %}
 9939 
 9940 // Rotate Left by variable
 9941 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9942 %{
 9943   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9944   match(Set dst (RotateLeft src shift));
 9945   effect(KILL cr);
 9946 
 9947   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9948   ins_encode %{
 9949     __ erolq($dst$$Register, $src$$Register, false);
 9950   %}
 9951   ins_pipe(ialu_reg_reg);
 9952 %}
 9953 
 9954 // Rotate Right by constant.
 9955 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9956 %{
 9957   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9958   match(Set dst (RotateRight dst shift));
 9959   effect(KILL cr);
 9960   format %{ "rorq    $dst, $shift" %}
 9961   ins_encode %{
 9962     __ rorq($dst$$Register, $shift$$constant);
 9963   %}
 9964   ins_pipe(ialu_reg);
 9965 %}
 9966 
 9967 // Rotate Right by constant
 9968 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9969 %{
 9970   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9971   match(Set dst (RotateRight src shift));
 9972   format %{ "rorxq   $dst, $src, $shift" %}
 9973   ins_encode %{
 9974     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9975   %}
 9976   ins_pipe(ialu_reg_reg);
 9977 %}
 9978 
 9979 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9980 %{
 9981   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9982   match(Set dst (RotateRight (LoadL src) shift));
 9983   ins_cost(175);
 9984   format %{ "rorxq   $dst, $src, $shift" %}
 9985   ins_encode %{
 9986     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9987   %}
 9988   ins_pipe(ialu_reg_mem);
 9989 %}
 9990 
 9991 // Rotate Right by variable
 9992 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9993 %{
 9994   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9995   match(Set dst (RotateRight dst shift));
 9996   effect(KILL cr);
 9997   format %{ "rorq    $dst, $shift" %}
 9998   ins_encode %{
 9999     __ rorq($dst$$Register);
10000   %}
10001   ins_pipe(ialu_reg_reg);
10002 %}
10003 
10004 // Rotate Right by variable
10005 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
10006 %{
10007   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
10008   match(Set dst (RotateRight src shift));
10009   effect(KILL cr);
10010 
10011   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
10012   ins_encode %{
10013     __ erorq($dst$$Register, $src$$Register, false);
10014   %}
10015   ins_pipe(ialu_reg_reg);
10016 %}
10017 
10018 //----------------------------- CompressBits/ExpandBits ------------------------
10019 
10020 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10021   predicate(n->bottom_type()->isa_long());
10022   match(Set dst (CompressBits src mask));
10023   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10024   ins_encode %{
10025     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
10026   %}
10027   ins_pipe( pipe_slow );
10028 %}
10029 
10030 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10031   predicate(n->bottom_type()->isa_long());
10032   match(Set dst (ExpandBits src mask));
10033   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10034   ins_encode %{
10035     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10036   %}
10037   ins_pipe( pipe_slow );
10038 %}
10039 
10040 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10041   predicate(n->bottom_type()->isa_long());
10042   match(Set dst (CompressBits src (LoadL mask)));
10043   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10044   ins_encode %{
10045     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10046   %}
10047   ins_pipe( pipe_slow );
10048 %}
10049 
10050 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10051   predicate(n->bottom_type()->isa_long());
10052   match(Set dst (ExpandBits src (LoadL mask)));
10053   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10054   ins_encode %{
10055     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10056   %}
10057   ins_pipe( pipe_slow );
10058 %}
10059 
10060 
10061 // Logical Instructions
10062 
10063 // Integer Logical Instructions
10064 
10065 // And Instructions
10066 // And Register with Register
10067 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10068 %{
10069   predicate(!UseAPX);
10070   match(Set dst (AndI dst src));
10071   effect(KILL cr);
10072   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10073 
10074   format %{ "andl    $dst, $src\t# int" %}
10075   ins_encode %{
10076     __ andl($dst$$Register, $src$$Register);
10077   %}
10078   ins_pipe(ialu_reg_reg);
10079 %}
10080 
10081 // And Register with Register using New Data Destination (NDD)
10082 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10083 %{
10084   predicate(UseAPX);
10085   match(Set dst (AndI src1 src2));
10086   effect(KILL cr);
10087   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10088 
10089   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10090   ins_encode %{
10091     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10092 
10093   %}
10094   ins_pipe(ialu_reg_reg);
10095 %}
10096 
10097 // And Register with Immediate 255
10098 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10099 %{
10100   match(Set dst (AndI src mask));
10101 
10102   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10103   ins_encode %{
10104     __ movzbl($dst$$Register, $src$$Register);
10105   %}
10106   ins_pipe(ialu_reg);
10107 %}
10108 
10109 // And Register with Immediate 255 and promote to long
10110 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10111 %{
10112   match(Set dst (ConvI2L (AndI src mask)));
10113 
10114   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10115   ins_encode %{
10116     __ movzbl($dst$$Register, $src$$Register);
10117   %}
10118   ins_pipe(ialu_reg);
10119 %}
10120 
10121 // And Register with Immediate 65535
10122 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10123 %{
10124   match(Set dst (AndI src mask));
10125 
10126   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10127   ins_encode %{
10128     __ movzwl($dst$$Register, $src$$Register);
10129   %}
10130   ins_pipe(ialu_reg);
10131 %}
10132 
10133 // And Register with Immediate 65535 and promote to long
10134 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10135 %{
10136   match(Set dst (ConvI2L (AndI src mask)));
10137 
10138   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10139   ins_encode %{
10140     __ movzwl($dst$$Register, $src$$Register);
10141   %}
10142   ins_pipe(ialu_reg);
10143 %}
10144 
10145 // Can skip int2long conversions after AND with small bitmask
10146 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10147 %{
10148   predicate(VM_Version::supports_bmi2());
10149   ins_cost(125);
10150   effect(TEMP tmp, KILL cr);
10151   match(Set dst (ConvI2L (AndI src mask)));
10152   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10153   ins_encode %{
10154     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10155     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10156   %}
10157   ins_pipe(ialu_reg_reg);
10158 %}
10159 
10160 // And Register with Immediate
10161 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10162 %{
10163   predicate(!UseAPX);
10164   match(Set dst (AndI dst src));
10165   effect(KILL cr);
10166   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10167 
10168   format %{ "andl    $dst, $src\t# int" %}
10169   ins_encode %{
10170     __ andl($dst$$Register, $src$$constant);
10171   %}
10172   ins_pipe(ialu_reg);
10173 %}
10174 
10175 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10176 %{
10177   predicate(UseAPX);
10178   match(Set dst (AndI src1 src2));
10179   effect(KILL cr);
10180   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10181 
10182   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10183   ins_encode %{
10184     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10185   %}
10186   ins_pipe(ialu_reg);
10187 %}
10188 
10189 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10190 %{
10191   predicate(UseAPX);
10192   match(Set dst (AndI (LoadI src1) src2));
10193   effect(KILL cr);
10194   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10195 
10196   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10197   ins_encode %{
10198     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10199   %}
10200   ins_pipe(ialu_reg);
10201 %}
10202 
10203 // And Register with Memory
10204 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10205 %{
10206   predicate(!UseAPX);
10207   match(Set dst (AndI dst (LoadI src)));
10208   effect(KILL cr);
10209   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10210 
10211   ins_cost(150);
10212   format %{ "andl    $dst, $src\t# int" %}
10213   ins_encode %{
10214     __ andl($dst$$Register, $src$$Address);
10215   %}
10216   ins_pipe(ialu_reg_mem);
10217 %}
10218 
10219 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10220 %{
10221   predicate(UseAPX);
10222   match(Set dst (AndI src1 (LoadI src2)));
10223   effect(KILL cr);
10224   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10225 
10226   ins_cost(150);
10227   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10228   ins_encode %{
10229     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10230   %}
10231   ins_pipe(ialu_reg_mem);
10232 %}
10233 
10234 // And Memory with Register
10235 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10236 %{
10237   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10238   effect(KILL cr);
10239   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10240 
10241   ins_cost(150);
10242   format %{ "andb    $dst, $src\t# byte" %}
10243   ins_encode %{
10244     __ andb($dst$$Address, $src$$Register);
10245   %}
10246   ins_pipe(ialu_mem_reg);
10247 %}
10248 
10249 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10250 %{
10251   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10252   effect(KILL cr);
10253   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10254 
10255   ins_cost(150);
10256   format %{ "andl    $dst, $src\t# int" %}
10257   ins_encode %{
10258     __ andl($dst$$Address, $src$$Register);
10259   %}
10260   ins_pipe(ialu_mem_reg);
10261 %}
10262 
10263 // And Memory with Immediate
10264 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10265 %{
10266   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10267   effect(KILL cr);
10268   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10269 
10270   ins_cost(125);
10271   format %{ "andl    $dst, $src\t# int" %}
10272   ins_encode %{
10273     __ andl($dst$$Address, $src$$constant);
10274   %}
10275   ins_pipe(ialu_mem_imm);
10276 %}
10277 
10278 // BMI1 instructions
10279 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10280   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10281   predicate(UseBMI1Instructions);
10282   effect(KILL cr);
10283   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10284 
10285   ins_cost(125);
10286   format %{ "andnl  $dst, $src1, $src2" %}
10287 
10288   ins_encode %{
10289     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10290   %}
10291   ins_pipe(ialu_reg_mem);
10292 %}
10293 
10294 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10295   match(Set dst (AndI (XorI src1 minus_1) src2));
10296   predicate(UseBMI1Instructions);
10297   effect(KILL cr);
10298   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10299 
10300   format %{ "andnl  $dst, $src1, $src2" %}
10301 
10302   ins_encode %{
10303     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10304   %}
10305   ins_pipe(ialu_reg);
10306 %}
10307 
10308 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10309   match(Set dst (AndI (SubI imm_zero src) src));
10310   predicate(UseBMI1Instructions);
10311   effect(KILL cr);
10312   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10313 
10314   format %{ "blsil  $dst, $src" %}
10315 
10316   ins_encode %{
10317     __ blsil($dst$$Register, $src$$Register);
10318   %}
10319   ins_pipe(ialu_reg);
10320 %}
10321 
10322 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10323   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10324   predicate(UseBMI1Instructions);
10325   effect(KILL cr);
10326   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10327 
10328   ins_cost(125);
10329   format %{ "blsil  $dst, $src" %}
10330 
10331   ins_encode %{
10332     __ blsil($dst$$Register, $src$$Address);
10333   %}
10334   ins_pipe(ialu_reg_mem);
10335 %}
10336 
10337 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10338 %{
10339   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10340   predicate(UseBMI1Instructions);
10341   effect(KILL cr);
10342   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10343 
10344   ins_cost(125);
10345   format %{ "blsmskl $dst, $src" %}
10346 
10347   ins_encode %{
10348     __ blsmskl($dst$$Register, $src$$Address);
10349   %}
10350   ins_pipe(ialu_reg_mem);
10351 %}
10352 
10353 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10354 %{
10355   match(Set dst (XorI (AddI src minus_1) src));
10356   predicate(UseBMI1Instructions);
10357   effect(KILL cr);
10358   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10359 
10360   format %{ "blsmskl $dst, $src" %}
10361 
10362   ins_encode %{
10363     __ blsmskl($dst$$Register, $src$$Register);
10364   %}
10365 
10366   ins_pipe(ialu_reg);
10367 %}
10368 
10369 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10370 %{
10371   match(Set dst (AndI (AddI src minus_1) src) );
10372   predicate(UseBMI1Instructions);
10373   effect(KILL cr);
10374   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10375 
10376   format %{ "blsrl  $dst, $src" %}
10377 
10378   ins_encode %{
10379     __ blsrl($dst$$Register, $src$$Register);
10380   %}
10381 
10382   ins_pipe(ialu_reg_mem);
10383 %}
10384 
10385 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10386 %{
10387   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10388   predicate(UseBMI1Instructions);
10389   effect(KILL cr);
10390   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10391 
10392   ins_cost(125);
10393   format %{ "blsrl  $dst, $src" %}
10394 
10395   ins_encode %{
10396     __ blsrl($dst$$Register, $src$$Address);
10397   %}
10398 
10399   ins_pipe(ialu_reg);
10400 %}
10401 
10402 // Or Instructions
10403 // Or Register with Register
10404 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10405 %{
10406   predicate(!UseAPX);
10407   match(Set dst (OrI dst src));
10408   effect(KILL cr);
10409   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10410 
10411   format %{ "orl     $dst, $src\t# int" %}
10412   ins_encode %{
10413     __ orl($dst$$Register, $src$$Register);
10414   %}
10415   ins_pipe(ialu_reg_reg);
10416 %}
10417 
10418 // Or Register with Register using New Data Destination (NDD)
10419 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10420 %{
10421   predicate(UseAPX);
10422   match(Set dst (OrI src1 src2));
10423   effect(KILL cr);
10424   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10425 
10426   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10427   ins_encode %{
10428     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10429   %}
10430   ins_pipe(ialu_reg_reg);
10431 %}
10432 
10433 // Or Register with Immediate
10434 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10435 %{
10436   predicate(!UseAPX);
10437   match(Set dst (OrI dst src));
10438   effect(KILL cr);
10439   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10440 
10441   format %{ "orl     $dst, $src\t# int" %}
10442   ins_encode %{
10443     __ orl($dst$$Register, $src$$constant);
10444   %}
10445   ins_pipe(ialu_reg);
10446 %}
10447 
10448 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10449 %{
10450   predicate(UseAPX);
10451   match(Set dst (OrI src1 src2));
10452   effect(KILL cr);
10453   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10454 
10455   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10456   ins_encode %{
10457     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10458   %}
10459   ins_pipe(ialu_reg);
10460 %}
10461 
10462 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10463 %{
10464   predicate(UseAPX);
10465   match(Set dst (OrI src1 src2));
10466   effect(KILL cr);
10467   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10468 
10469   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10470   ins_encode %{
10471     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10472   %}
10473   ins_pipe(ialu_reg);
10474 %}
10475 
10476 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10477 %{
10478   predicate(UseAPX);
10479   match(Set dst (OrI (LoadI src1) src2));
10480   effect(KILL cr);
10481   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10482 
10483   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10484   ins_encode %{
10485     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10486   %}
10487   ins_pipe(ialu_reg);
10488 %}
10489 
10490 // Or Register with Memory
10491 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10492 %{
10493   predicate(!UseAPX);
10494   match(Set dst (OrI dst (LoadI src)));
10495   effect(KILL cr);
10496   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10497 
10498   ins_cost(150);
10499   format %{ "orl     $dst, $src\t# int" %}
10500   ins_encode %{
10501     __ orl($dst$$Register, $src$$Address);
10502   %}
10503   ins_pipe(ialu_reg_mem);
10504 %}
10505 
10506 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10507 %{
10508   predicate(UseAPX);
10509   match(Set dst (OrI src1 (LoadI src2)));
10510   effect(KILL cr);
10511   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10512 
10513   ins_cost(150);
10514   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10515   ins_encode %{
10516     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10517   %}
10518   ins_pipe(ialu_reg_mem);
10519 %}
10520 
10521 // Or Memory with Register
10522 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10523 %{
10524   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10525   effect(KILL cr);
10526   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10527 
10528   ins_cost(150);
10529   format %{ "orb    $dst, $src\t# byte" %}
10530   ins_encode %{
10531     __ orb($dst$$Address, $src$$Register);
10532   %}
10533   ins_pipe(ialu_mem_reg);
10534 %}
10535 
10536 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10537 %{
10538   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10539   effect(KILL cr);
10540   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10541 
10542   ins_cost(150);
10543   format %{ "orl     $dst, $src\t# int" %}
10544   ins_encode %{
10545     __ orl($dst$$Address, $src$$Register);
10546   %}
10547   ins_pipe(ialu_mem_reg);
10548 %}
10549 
10550 // Or Memory with Immediate
10551 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10552 %{
10553   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10554   effect(KILL cr);
10555   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10556 
10557   ins_cost(125);
10558   format %{ "orl     $dst, $src\t# int" %}
10559   ins_encode %{
10560     __ orl($dst$$Address, $src$$constant);
10561   %}
10562   ins_pipe(ialu_mem_imm);
10563 %}
10564 
10565 // Xor Instructions
10566 // Xor Register with Register
10567 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10568 %{
10569   predicate(!UseAPX);
10570   match(Set dst (XorI dst src));
10571   effect(KILL cr);
10572   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10573 
10574   format %{ "xorl    $dst, $src\t# int" %}
10575   ins_encode %{
10576     __ xorl($dst$$Register, $src$$Register);
10577   %}
10578   ins_pipe(ialu_reg_reg);
10579 %}
10580 
10581 // Xor Register with Register using New Data Destination (NDD)
10582 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10583 %{
10584   predicate(UseAPX);
10585   match(Set dst (XorI src1 src2));
10586   effect(KILL cr);
10587   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10588 
10589   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10590   ins_encode %{
10591     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10592   %}
10593   ins_pipe(ialu_reg_reg);
10594 %}
10595 
10596 // Xor Register with Immediate -1
10597 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10598 %{
10599   predicate(!UseAPX);
10600   match(Set dst (XorI dst imm));
10601 
10602   format %{ "notl    $dst" %}
10603   ins_encode %{
10604      __ notl($dst$$Register);
10605   %}
10606   ins_pipe(ialu_reg);
10607 %}
10608 
10609 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10610 %{
10611   match(Set dst (XorI src imm));
10612   predicate(UseAPX);
10613 
10614   format %{ "enotl    $dst, $src" %}
10615   ins_encode %{
10616      __ enotl($dst$$Register, $src$$Register);
10617   %}
10618   ins_pipe(ialu_reg);
10619 %}
10620 
10621 // Xor Register with Immediate
10622 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10623 %{
10624   predicate(!UseAPX);
10625   match(Set dst (XorI dst src));
10626   effect(KILL cr);
10627   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10628 
10629   format %{ "xorl    $dst, $src\t# int" %}
10630   ins_encode %{
10631     __ xorl($dst$$Register, $src$$constant);
10632   %}
10633   ins_pipe(ialu_reg);
10634 %}
10635 
10636 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10637 %{
10638   predicate(UseAPX);
10639   match(Set dst (XorI src1 src2));
10640   effect(KILL cr);
10641   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10642 
10643   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10644   ins_encode %{
10645     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10646   %}
10647   ins_pipe(ialu_reg);
10648 %}
10649 
10650 // Xor Memory with Immediate
10651 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10652 %{
10653   predicate(UseAPX);
10654   match(Set dst (XorI (LoadI src1) src2));
10655   effect(KILL cr);
10656   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10657 
10658   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10659   ins_encode %{
10660     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10661   %}
10662   ins_pipe(ialu_reg);
10663 %}
10664 
10665 // Xor Register with Memory
10666 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10667 %{
10668   predicate(!UseAPX);
10669   match(Set dst (XorI dst (LoadI src)));
10670   effect(KILL cr);
10671   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10672 
10673   ins_cost(150);
10674   format %{ "xorl    $dst, $src\t# int" %}
10675   ins_encode %{
10676     __ xorl($dst$$Register, $src$$Address);
10677   %}
10678   ins_pipe(ialu_reg_mem);
10679 %}
10680 
10681 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10682 %{
10683   predicate(UseAPX);
10684   match(Set dst (XorI src1 (LoadI src2)));
10685   effect(KILL cr);
10686   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10687 
10688   ins_cost(150);
10689   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10690   ins_encode %{
10691     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10692   %}
10693   ins_pipe(ialu_reg_mem);
10694 %}
10695 
10696 instruct xorI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
10697 %{
10698   predicate(UseAPX);
10699   match(Set dst (XorI (LoadI src1) src2));
10700   effect(KILL cr);
10701   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10702 
10703   ins_cost(150);
10704   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10705   ins_encode %{
10706     __ exorl($dst$$Register, $src1$$Address, $src2$$Register, false);
10707   %}
10708   ins_pipe(ialu_reg_mem);
10709 %}
10710 
10711 // Xor Memory with Register
10712 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10713 %{
10714   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10715   effect(KILL cr);
10716   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10717 
10718   ins_cost(150);
10719   format %{ "xorb    $dst, $src\t# byte" %}
10720   ins_encode %{
10721     __ xorb($dst$$Address, $src$$Register);
10722   %}
10723   ins_pipe(ialu_mem_reg);
10724 %}
10725 
10726 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10727 %{
10728   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10729   effect(KILL cr);
10730   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10731 
10732   ins_cost(150);
10733   format %{ "xorl    $dst, $src\t# int" %}
10734   ins_encode %{
10735     __ xorl($dst$$Address, $src$$Register);
10736   %}
10737   ins_pipe(ialu_mem_reg);
10738 %}
10739 
10740 // Xor Memory with Immediate
10741 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10742 %{
10743   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10744   effect(KILL cr);
10745   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10746 
10747   ins_cost(125);
10748   format %{ "xorl    $dst, $src\t# int" %}
10749   ins_encode %{
10750     __ xorl($dst$$Address, $src$$constant);
10751   %}
10752   ins_pipe(ialu_mem_imm);
10753 %}
10754 
10755 
10756 // Long Logical Instructions
10757 
10758 // And Instructions
10759 // And Register with Register
10760 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10761 %{
10762   predicate(!UseAPX);
10763   match(Set dst (AndL dst src));
10764   effect(KILL cr);
10765   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10766 
10767   format %{ "andq    $dst, $src\t# long" %}
10768   ins_encode %{
10769     __ andq($dst$$Register, $src$$Register);
10770   %}
10771   ins_pipe(ialu_reg_reg);
10772 %}
10773 
10774 // And Register with Register using New Data Destination (NDD)
10775 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10776 %{
10777   predicate(UseAPX);
10778   match(Set dst (AndL src1 src2));
10779   effect(KILL cr);
10780   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10781 
10782   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10783   ins_encode %{
10784     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10785 
10786   %}
10787   ins_pipe(ialu_reg_reg);
10788 %}
10789 
10790 // And Register with Immediate 255
10791 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10792 %{
10793   match(Set dst (AndL src mask));
10794 
10795   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10796   ins_encode %{
10797     // movzbl zeroes out the upper 32-bit and does not need REX.W
10798     __ movzbl($dst$$Register, $src$$Register);
10799   %}
10800   ins_pipe(ialu_reg);
10801 %}
10802 
10803 // And Register with Immediate 65535
10804 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10805 %{
10806   match(Set dst (AndL src mask));
10807 
10808   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10809   ins_encode %{
10810     // movzwl zeroes out the upper 32-bit and does not need REX.W
10811     __ movzwl($dst$$Register, $src$$Register);
10812   %}
10813   ins_pipe(ialu_reg);
10814 %}
10815 
10816 // And Register with Immediate
10817 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10818 %{
10819   predicate(!UseAPX);
10820   match(Set dst (AndL dst src));
10821   effect(KILL cr);
10822   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10823 
10824   format %{ "andq    $dst, $src\t# long" %}
10825   ins_encode %{
10826     __ andq($dst$$Register, $src$$constant);
10827   %}
10828   ins_pipe(ialu_reg);
10829 %}
10830 
10831 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10832 %{
10833   predicate(UseAPX);
10834   match(Set dst (AndL src1 src2));
10835   effect(KILL cr);
10836   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10837 
10838   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10839   ins_encode %{
10840     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10841   %}
10842   ins_pipe(ialu_reg);
10843 %}
10844 
10845 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10846 %{
10847   predicate(UseAPX);
10848   match(Set dst (AndL (LoadL src1) src2));
10849   effect(KILL cr);
10850   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10851 
10852   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10853   ins_encode %{
10854     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10855   %}
10856   ins_pipe(ialu_reg);
10857 %}
10858 
10859 // And Register with Memory
10860 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10861 %{
10862   predicate(!UseAPX);
10863   match(Set dst (AndL dst (LoadL src)));
10864   effect(KILL cr);
10865   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10866 
10867   ins_cost(150);
10868   format %{ "andq    $dst, $src\t# long" %}
10869   ins_encode %{
10870     __ andq($dst$$Register, $src$$Address);
10871   %}
10872   ins_pipe(ialu_reg_mem);
10873 %}
10874 
10875 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10876 %{
10877   predicate(UseAPX);
10878   match(Set dst (AndL src1 (LoadL src2)));
10879   effect(KILL cr);
10880   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10881 
10882   ins_cost(150);
10883   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10884   ins_encode %{
10885     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10886   %}
10887   ins_pipe(ialu_reg_mem);
10888 %}
10889 
10890 instruct andL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
10891 %{
10892   predicate(UseAPX);
10893   match(Set dst (AndL (LoadL src1) src2));
10894   effect(KILL cr);
10895   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10896 
10897   ins_cost(150);
10898   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10899   ins_encode %{
10900     __ eandq($dst$$Register, $src1$$Address, $src2$$Register, false);
10901   %}
10902   ins_pipe(ialu_reg_mem);
10903 %}
10904 
10905 // And Memory with Register
10906 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10907 %{
10908   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10909   effect(KILL cr);
10910   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10911 
10912   ins_cost(150);
10913   format %{ "andq    $dst, $src\t# long" %}
10914   ins_encode %{
10915     __ andq($dst$$Address, $src$$Register);
10916   %}
10917   ins_pipe(ialu_mem_reg);
10918 %}
10919 
10920 // And Memory with Immediate
10921 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10922 %{
10923   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10924   effect(KILL cr);
10925   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10926 
10927   ins_cost(125);
10928   format %{ "andq    $dst, $src\t# long" %}
10929   ins_encode %{
10930     __ andq($dst$$Address, $src$$constant);
10931   %}
10932   ins_pipe(ialu_mem_imm);
10933 %}
10934 
10935 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10936 %{
10937   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10938   // because AND/OR works well enough for 8/32-bit values.
10939   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10940 
10941   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10942   effect(KILL cr);
10943 
10944   ins_cost(125);
10945   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10946   ins_encode %{
10947     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10948   %}
10949   ins_pipe(ialu_mem_imm);
10950 %}
10951 
10952 // BMI1 instructions
10953 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10954   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10955   predicate(UseBMI1Instructions);
10956   effect(KILL cr);
10957   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10958 
10959   ins_cost(125);
10960   format %{ "andnq  $dst, $src1, $src2" %}
10961 
10962   ins_encode %{
10963     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10964   %}
10965   ins_pipe(ialu_reg_mem);
10966 %}
10967 
10968 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10969   match(Set dst (AndL (XorL src1 minus_1) src2));
10970   predicate(UseBMI1Instructions);
10971   effect(KILL cr);
10972   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10973 
10974   format %{ "andnq  $dst, $src1, $src2" %}
10975 
10976   ins_encode %{
10977   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10978   %}
10979   ins_pipe(ialu_reg_mem);
10980 %}
10981 
10982 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10983   match(Set dst (AndL (SubL imm_zero src) src));
10984   predicate(UseBMI1Instructions);
10985   effect(KILL cr);
10986   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10987 
10988   format %{ "blsiq  $dst, $src" %}
10989 
10990   ins_encode %{
10991     __ blsiq($dst$$Register, $src$$Register);
10992   %}
10993   ins_pipe(ialu_reg);
10994 %}
10995 
10996 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10997   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10998   predicate(UseBMI1Instructions);
10999   effect(KILL cr);
11000   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11001 
11002   ins_cost(125);
11003   format %{ "blsiq  $dst, $src" %}
11004 
11005   ins_encode %{
11006     __ blsiq($dst$$Register, $src$$Address);
11007   %}
11008   ins_pipe(ialu_reg_mem);
11009 %}
11010 
11011 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11012 %{
11013   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
11014   predicate(UseBMI1Instructions);
11015   effect(KILL cr);
11016   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11017 
11018   ins_cost(125);
11019   format %{ "blsmskq $dst, $src" %}
11020 
11021   ins_encode %{
11022     __ blsmskq($dst$$Register, $src$$Address);
11023   %}
11024   ins_pipe(ialu_reg_mem);
11025 %}
11026 
11027 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11028 %{
11029   match(Set dst (XorL (AddL src minus_1) src));
11030   predicate(UseBMI1Instructions);
11031   effect(KILL cr);
11032   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
11033 
11034   format %{ "blsmskq $dst, $src" %}
11035 
11036   ins_encode %{
11037     __ blsmskq($dst$$Register, $src$$Register);
11038   %}
11039 
11040   ins_pipe(ialu_reg);
11041 %}
11042 
11043 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
11044 %{
11045   match(Set dst (AndL (AddL src minus_1) src) );
11046   predicate(UseBMI1Instructions);
11047   effect(KILL cr);
11048   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11049 
11050   format %{ "blsrq  $dst, $src" %}
11051 
11052   ins_encode %{
11053     __ blsrq($dst$$Register, $src$$Register);
11054   %}
11055 
11056   ins_pipe(ialu_reg);
11057 %}
11058 
11059 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11060 %{
11061   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
11062   predicate(UseBMI1Instructions);
11063   effect(KILL cr);
11064   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11065 
11066   ins_cost(125);
11067   format %{ "blsrq  $dst, $src" %}
11068 
11069   ins_encode %{
11070     __ blsrq($dst$$Register, $src$$Address);
11071   %}
11072 
11073   ins_pipe(ialu_reg);
11074 %}
11075 
11076 // Or Instructions
11077 // Or Register with Register
11078 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11079 %{
11080   predicate(!UseAPX);
11081   match(Set dst (OrL dst src));
11082   effect(KILL cr);
11083   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11084 
11085   format %{ "orq     $dst, $src\t# long" %}
11086   ins_encode %{
11087     __ orq($dst$$Register, $src$$Register);
11088   %}
11089   ins_pipe(ialu_reg_reg);
11090 %}
11091 
11092 // Or Register with Register using New Data Destination (NDD)
11093 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11094 %{
11095   predicate(UseAPX);
11096   match(Set dst (OrL src1 src2));
11097   effect(KILL cr);
11098   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11099 
11100   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11101   ins_encode %{
11102     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11103 
11104   %}
11105   ins_pipe(ialu_reg_reg);
11106 %}
11107 
11108 // Use any_RegP to match R15 (TLS register) without spilling.
11109 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11110   match(Set dst (OrL dst (CastP2X src)));
11111   effect(KILL cr);
11112   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11113 
11114   format %{ "orq     $dst, $src\t# long" %}
11115   ins_encode %{
11116     __ orq($dst$$Register, $src$$Register);
11117   %}
11118   ins_pipe(ialu_reg_reg);
11119 %}
11120 
11121 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11122   match(Set dst (OrL src1 (CastP2X src2)));
11123   effect(KILL cr);
11124   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11125 
11126   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11127   ins_encode %{
11128     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11129   %}
11130   ins_pipe(ialu_reg_reg);
11131 %}
11132 
11133 // Or Register with Immediate
11134 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11135 %{
11136   predicate(!UseAPX);
11137   match(Set dst (OrL dst src));
11138   effect(KILL cr);
11139   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11140 
11141   format %{ "orq     $dst, $src\t# long" %}
11142   ins_encode %{
11143     __ orq($dst$$Register, $src$$constant);
11144   %}
11145   ins_pipe(ialu_reg);
11146 %}
11147 
11148 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11149 %{
11150   predicate(UseAPX);
11151   match(Set dst (OrL src1 src2));
11152   effect(KILL cr);
11153   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11154 
11155   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11156   ins_encode %{
11157     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11158   %}
11159   ins_pipe(ialu_reg);
11160 %}
11161 
11162 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11163 %{
11164   predicate(UseAPX);
11165   match(Set dst (OrL src1 src2));
11166   effect(KILL cr);
11167   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11168 
11169   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11170   ins_encode %{
11171     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11172   %}
11173   ins_pipe(ialu_reg);
11174 %}
11175 
11176 // Or Memory with Immediate
11177 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11178 %{
11179   predicate(UseAPX);
11180   match(Set dst (OrL (LoadL src1) src2));
11181   effect(KILL cr);
11182   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11183 
11184   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11185   ins_encode %{
11186     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11187   %}
11188   ins_pipe(ialu_reg);
11189 %}
11190 
11191 // Or Register with Memory
11192 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11193 %{
11194   predicate(!UseAPX);
11195   match(Set dst (OrL dst (LoadL src)));
11196   effect(KILL cr);
11197   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11198 
11199   ins_cost(150);
11200   format %{ "orq     $dst, $src\t# long" %}
11201   ins_encode %{
11202     __ orq($dst$$Register, $src$$Address);
11203   %}
11204   ins_pipe(ialu_reg_mem);
11205 %}
11206 
11207 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11208 %{
11209   predicate(UseAPX);
11210   match(Set dst (OrL src1 (LoadL src2)));
11211   effect(KILL cr);
11212   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11213 
11214   ins_cost(150);
11215   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11216   ins_encode %{
11217     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11218   %}
11219   ins_pipe(ialu_reg_mem);
11220 %}
11221 
11222 // Or Memory with Register
11223 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11224 %{
11225   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11226   effect(KILL cr);
11227   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11228 
11229   ins_cost(150);
11230   format %{ "orq     $dst, $src\t# long" %}
11231   ins_encode %{
11232     __ orq($dst$$Address, $src$$Register);
11233   %}
11234   ins_pipe(ialu_mem_reg);
11235 %}
11236 
11237 // Or Memory with Immediate
11238 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11239 %{
11240   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11241   effect(KILL cr);
11242   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11243 
11244   ins_cost(125);
11245   format %{ "orq     $dst, $src\t# long" %}
11246   ins_encode %{
11247     __ orq($dst$$Address, $src$$constant);
11248   %}
11249   ins_pipe(ialu_mem_imm);
11250 %}
11251 
11252 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11253 %{
11254   // con should be a pure 64-bit power of 2 immediate
11255   // because AND/OR works well enough for 8/32-bit values.
11256   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11257 
11258   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11259   effect(KILL cr);
11260 
11261   ins_cost(125);
11262   format %{ "btsq    $dst, log2($con)\t# long" %}
11263   ins_encode %{
11264     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11265   %}
11266   ins_pipe(ialu_mem_imm);
11267 %}
11268 
11269 // Xor Instructions
11270 // Xor Register with Register
11271 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11272 %{
11273   predicate(!UseAPX);
11274   match(Set dst (XorL dst src));
11275   effect(KILL cr);
11276   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11277 
11278   format %{ "xorq    $dst, $src\t# long" %}
11279   ins_encode %{
11280     __ xorq($dst$$Register, $src$$Register);
11281   %}
11282   ins_pipe(ialu_reg_reg);
11283 %}
11284 
11285 // Xor Register with Register using New Data Destination (NDD)
11286 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11287 %{
11288   predicate(UseAPX);
11289   match(Set dst (XorL src1 src2));
11290   effect(KILL cr);
11291   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11292 
11293   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11294   ins_encode %{
11295     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11296   %}
11297   ins_pipe(ialu_reg_reg);
11298 %}
11299 
11300 // Xor Register with Immediate -1
11301 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11302 %{
11303   predicate(!UseAPX);
11304   match(Set dst (XorL dst imm));
11305 
11306   format %{ "notq   $dst" %}
11307   ins_encode %{
11308      __ notq($dst$$Register);
11309   %}
11310   ins_pipe(ialu_reg);
11311 %}
11312 
11313 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11314 %{
11315   predicate(UseAPX);
11316   match(Set dst (XorL src imm));
11317 
11318   format %{ "enotq   $dst, $src" %}
11319   ins_encode %{
11320     __ enotq($dst$$Register, $src$$Register);
11321   %}
11322   ins_pipe(ialu_reg);
11323 %}
11324 
11325 // Xor Register with Immediate
11326 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11327 %{
11328   predicate(!UseAPX);
11329   match(Set dst (XorL dst src));
11330   effect(KILL cr);
11331   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11332 
11333   format %{ "xorq    $dst, $src\t# long" %}
11334   ins_encode %{
11335     __ xorq($dst$$Register, $src$$constant);
11336   %}
11337   ins_pipe(ialu_reg);
11338 %}
11339 
11340 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11341 %{
11342   predicate(UseAPX);
11343   match(Set dst (XorL src1 src2));
11344   effect(KILL cr);
11345   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11346 
11347   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11348   ins_encode %{
11349     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11350   %}
11351   ins_pipe(ialu_reg);
11352 %}
11353 
11354 // Xor Memory with Immediate
11355 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11356 %{
11357   predicate(UseAPX);
11358   match(Set dst (XorL (LoadL src1) src2));
11359   effect(KILL cr);
11360   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11361 
11362   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11363   ins_encode %{
11364     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11365   %}
11366   ins_pipe(ialu_reg);
11367 %}
11368 
11369 // Xor Register with Memory
11370 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11371 %{
11372   predicate(!UseAPX);
11373   match(Set dst (XorL dst (LoadL src)));
11374   effect(KILL cr);
11375   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11376 
11377   ins_cost(150);
11378   format %{ "xorq    $dst, $src\t# long" %}
11379   ins_encode %{
11380     __ xorq($dst$$Register, $src$$Address);
11381   %}
11382   ins_pipe(ialu_reg_mem);
11383 %}
11384 
11385 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11386 %{
11387   predicate(UseAPX);
11388   match(Set dst (XorL src1 (LoadL src2)));
11389   effect(KILL cr);
11390   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11391 
11392   ins_cost(150);
11393   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11394   ins_encode %{
11395     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11396   %}
11397   ins_pipe(ialu_reg_mem);
11398 %}
11399 
11400 instruct xorL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
11401 %{
11402   predicate(UseAPX);
11403   match(Set dst (XorL (LoadL src1) src2));
11404   effect(KILL cr);
11405   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11406 
11407   ins_cost(150);
11408   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11409   ins_encode %{
11410     __ exorq($dst$$Register, $src1$$Address, $src2$$Register, false);
11411   %}
11412   ins_pipe(ialu_reg_mem);
11413 %}
11414 
11415 // Xor Memory with Register
11416 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11417 %{
11418   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11419   effect(KILL cr);
11420   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11421 
11422   ins_cost(150);
11423   format %{ "xorq    $dst, $src\t# long" %}
11424   ins_encode %{
11425     __ xorq($dst$$Address, $src$$Register);
11426   %}
11427   ins_pipe(ialu_mem_reg);
11428 %}
11429 
11430 // Xor Memory with Immediate
11431 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11432 %{
11433   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11434   effect(KILL cr);
11435   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11436 
11437   ins_cost(125);
11438   format %{ "xorq    $dst, $src\t# long" %}
11439   ins_encode %{
11440     __ xorq($dst$$Address, $src$$constant);
11441   %}
11442   ins_pipe(ialu_mem_imm);
11443 %}
11444 
11445 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11446 %{
11447   match(Set dst (CmpLTMask p q));
11448   effect(KILL cr);
11449 
11450   ins_cost(400);
11451   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11452             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11453             "negl    $dst" %}
11454   ins_encode %{
11455     __ cmpl($p$$Register, $q$$Register);
11456     __ setcc(Assembler::less, $dst$$Register);
11457     __ negl($dst$$Register);
11458   %}
11459   ins_pipe(pipe_slow);
11460 %}
11461 
11462 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11463 %{
11464   match(Set dst (CmpLTMask dst zero));
11465   effect(KILL cr);
11466 
11467   ins_cost(100);
11468   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11469   ins_encode %{
11470     __ sarl($dst$$Register, 31);
11471   %}
11472   ins_pipe(ialu_reg);
11473 %}
11474 
11475 /* Better to save a register than avoid a branch */
11476 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11477 %{
11478   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11479   effect(KILL cr);
11480   ins_cost(300);
11481   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11482             "jge     done\n\t"
11483             "addl    $p,$y\n"
11484             "done:   " %}
11485   ins_encode %{
11486     Register Rp = $p$$Register;
11487     Register Rq = $q$$Register;
11488     Register Ry = $y$$Register;
11489     Label done;
11490     __ subl(Rp, Rq);
11491     __ jccb(Assembler::greaterEqual, done);
11492     __ addl(Rp, Ry);
11493     __ bind(done);
11494   %}
11495   ins_pipe(pipe_cmplt);
11496 %}
11497 
11498 /* Better to save a register than avoid a branch */
11499 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11500 %{
11501   match(Set y (AndI (CmpLTMask p q) y));
11502   effect(KILL cr);
11503 
11504   ins_cost(300);
11505 
11506   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11507             "jlt     done\n\t"
11508             "xorl    $y, $y\n"
11509             "done:   " %}
11510   ins_encode %{
11511     Register Rp = $p$$Register;
11512     Register Rq = $q$$Register;
11513     Register Ry = $y$$Register;
11514     Label done;
11515     __ cmpl(Rp, Rq);
11516     __ jccb(Assembler::less, done);
11517     __ xorl(Ry, Ry);
11518     __ bind(done);
11519   %}
11520   ins_pipe(pipe_cmplt);
11521 %}
11522 
11523 
11524 //---------- FP Instructions------------------------------------------------
11525 
11526 // Really expensive, avoid
11527 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11528 %{
11529   match(Set cr (CmpF src1 src2));
11530 
11531   ins_cost(500);
11532   format %{ "ucomiss $src1, $src2\n\t"
11533             "jnp,s   exit\n\t"
11534             "pushfq\t# saw NaN, set CF\n\t"
11535             "andq    [rsp], #0xffffff2b\n\t"
11536             "popfq\n"
11537     "exit:" %}
11538   ins_encode %{
11539     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11540     emit_cmpfp_fixup(masm);
11541   %}
11542   ins_pipe(pipe_slow);
11543 %}
11544 
11545 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11546   match(Set cr (CmpF src1 src2));
11547 
11548   ins_cost(100);
11549   format %{ "ucomiss $src1, $src2" %}
11550   ins_encode %{
11551     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11552   %}
11553   ins_pipe(pipe_slow);
11554 %}
11555 
11556 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11557   match(Set cr (CmpF src1 (LoadF src2)));
11558 
11559   ins_cost(100);
11560   format %{ "ucomiss $src1, $src2" %}
11561   ins_encode %{
11562     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11563   %}
11564   ins_pipe(pipe_slow);
11565 %}
11566 
11567 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11568   match(Set cr (CmpF src con));
11569   ins_cost(100);
11570   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11571   ins_encode %{
11572     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11573   %}
11574   ins_pipe(pipe_slow);
11575 %}
11576 
11577 // Really expensive, avoid
11578 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11579 %{
11580   match(Set cr (CmpD src1 src2));
11581 
11582   ins_cost(500);
11583   format %{ "ucomisd $src1, $src2\n\t"
11584             "jnp,s   exit\n\t"
11585             "pushfq\t# saw NaN, set CF\n\t"
11586             "andq    [rsp], #0xffffff2b\n\t"
11587             "popfq\n"
11588     "exit:" %}
11589   ins_encode %{
11590     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11591     emit_cmpfp_fixup(masm);
11592   %}
11593   ins_pipe(pipe_slow);
11594 %}
11595 
11596 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11597   match(Set cr (CmpD src1 src2));
11598 
11599   ins_cost(100);
11600   format %{ "ucomisd $src1, $src2 test" %}
11601   ins_encode %{
11602     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11603   %}
11604   ins_pipe(pipe_slow);
11605 %}
11606 
11607 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11608   match(Set cr (CmpD src1 (LoadD src2)));
11609 
11610   ins_cost(100);
11611   format %{ "ucomisd $src1, $src2" %}
11612   ins_encode %{
11613     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11614   %}
11615   ins_pipe(pipe_slow);
11616 %}
11617 
11618 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11619   match(Set cr (CmpD src con));
11620   ins_cost(100);
11621   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11622   ins_encode %{
11623     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11624   %}
11625   ins_pipe(pipe_slow);
11626 %}
11627 
11628 // Compare into -1,0,1
11629 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11630 %{
11631   match(Set dst (CmpF3 src1 src2));
11632   effect(KILL cr);
11633 
11634   ins_cost(275);
11635   format %{ "ucomiss $src1, $src2\n\t"
11636             "movl    $dst, #-1\n\t"
11637             "jp,s    done\n\t"
11638             "jb,s    done\n\t"
11639             "setne   $dst\n\t"
11640             "movzbl  $dst, $dst\n"
11641     "done:" %}
11642   ins_encode %{
11643     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11644     emit_cmpfp3(masm, $dst$$Register);
11645   %}
11646   ins_pipe(pipe_slow);
11647 %}
11648 
11649 // Compare into -1,0,1
11650 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11651 %{
11652   match(Set dst (CmpF3 src1 (LoadF src2)));
11653   effect(KILL cr);
11654 
11655   ins_cost(275);
11656   format %{ "ucomiss $src1, $src2\n\t"
11657             "movl    $dst, #-1\n\t"
11658             "jp,s    done\n\t"
11659             "jb,s    done\n\t"
11660             "setne   $dst\n\t"
11661             "movzbl  $dst, $dst\n"
11662     "done:" %}
11663   ins_encode %{
11664     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11665     emit_cmpfp3(masm, $dst$$Register);
11666   %}
11667   ins_pipe(pipe_slow);
11668 %}
11669 
11670 // Compare into -1,0,1
11671 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11672   match(Set dst (CmpF3 src con));
11673   effect(KILL cr);
11674 
11675   ins_cost(275);
11676   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11677             "movl    $dst, #-1\n\t"
11678             "jp,s    done\n\t"
11679             "jb,s    done\n\t"
11680             "setne   $dst\n\t"
11681             "movzbl  $dst, $dst\n"
11682     "done:" %}
11683   ins_encode %{
11684     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11685     emit_cmpfp3(masm, $dst$$Register);
11686   %}
11687   ins_pipe(pipe_slow);
11688 %}
11689 
11690 // Compare into -1,0,1
11691 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11692 %{
11693   match(Set dst (CmpD3 src1 src2));
11694   effect(KILL cr);
11695 
11696   ins_cost(275);
11697   format %{ "ucomisd $src1, $src2\n\t"
11698             "movl    $dst, #-1\n\t"
11699             "jp,s    done\n\t"
11700             "jb,s    done\n\t"
11701             "setne   $dst\n\t"
11702             "movzbl  $dst, $dst\n"
11703     "done:" %}
11704   ins_encode %{
11705     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11706     emit_cmpfp3(masm, $dst$$Register);
11707   %}
11708   ins_pipe(pipe_slow);
11709 %}
11710 
11711 // Compare into -1,0,1
11712 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11713 %{
11714   match(Set dst (CmpD3 src1 (LoadD src2)));
11715   effect(KILL cr);
11716 
11717   ins_cost(275);
11718   format %{ "ucomisd $src1, $src2\n\t"
11719             "movl    $dst, #-1\n\t"
11720             "jp,s    done\n\t"
11721             "jb,s    done\n\t"
11722             "setne   $dst\n\t"
11723             "movzbl  $dst, $dst\n"
11724     "done:" %}
11725   ins_encode %{
11726     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11727     emit_cmpfp3(masm, $dst$$Register);
11728   %}
11729   ins_pipe(pipe_slow);
11730 %}
11731 
11732 // Compare into -1,0,1
11733 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11734   match(Set dst (CmpD3 src con));
11735   effect(KILL cr);
11736 
11737   ins_cost(275);
11738   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11739             "movl    $dst, #-1\n\t"
11740             "jp,s    done\n\t"
11741             "jb,s    done\n\t"
11742             "setne   $dst\n\t"
11743             "movzbl  $dst, $dst\n"
11744     "done:" %}
11745   ins_encode %{
11746     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11747     emit_cmpfp3(masm, $dst$$Register);
11748   %}
11749   ins_pipe(pipe_slow);
11750 %}
11751 
11752 //----------Arithmetic Conversion Instructions---------------------------------
11753 
11754 instruct convF2D_reg_reg(regD dst, regF src)
11755 %{
11756   match(Set dst (ConvF2D src));
11757 
11758   format %{ "cvtss2sd $dst, $src" %}
11759   ins_encode %{
11760     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11761   %}
11762   ins_pipe(pipe_slow); // XXX
11763 %}
11764 
11765 instruct convF2D_reg_mem(regD dst, memory src)
11766 %{
11767   predicate(UseAVX == 0);
11768   match(Set dst (ConvF2D (LoadF src)));
11769 
11770   format %{ "cvtss2sd $dst, $src" %}
11771   ins_encode %{
11772     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11773   %}
11774   ins_pipe(pipe_slow); // XXX
11775 %}
11776 
11777 instruct convD2F_reg_reg(regF dst, regD src)
11778 %{
11779   match(Set dst (ConvD2F src));
11780 
11781   format %{ "cvtsd2ss $dst, $src" %}
11782   ins_encode %{
11783     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11784   %}
11785   ins_pipe(pipe_slow); // XXX
11786 %}
11787 
11788 instruct convD2F_reg_mem(regF dst, memory src)
11789 %{
11790   predicate(UseAVX == 0);
11791   match(Set dst (ConvD2F (LoadD src)));
11792 
11793   format %{ "cvtsd2ss $dst, $src" %}
11794   ins_encode %{
11795     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11796   %}
11797   ins_pipe(pipe_slow); // XXX
11798 %}
11799 
11800 // XXX do mem variants
11801 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11802 %{
11803   match(Set dst (ConvF2I src));
11804   effect(KILL cr);
11805   format %{ "convert_f2i $dst, $src" %}
11806   ins_encode %{
11807     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11808   %}
11809   ins_pipe(pipe_slow);
11810 %}
11811 
11812 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11813 %{
11814   match(Set dst (ConvF2L src));
11815   effect(KILL cr);
11816   format %{ "convert_f2l $dst, $src"%}
11817   ins_encode %{
11818     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11819   %}
11820   ins_pipe(pipe_slow);
11821 %}
11822 
11823 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11824 %{
11825   match(Set dst (ConvD2I src));
11826   effect(KILL cr);
11827   format %{ "convert_d2i $dst, $src"%}
11828   ins_encode %{
11829     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11830   %}
11831   ins_pipe(pipe_slow);
11832 %}
11833 
11834 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11835 %{
11836   match(Set dst (ConvD2L src));
11837   effect(KILL cr);
11838   format %{ "convert_d2l $dst, $src"%}
11839   ins_encode %{
11840     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11841   %}
11842   ins_pipe(pipe_slow);
11843 %}
11844 
11845 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11846 %{
11847   match(Set dst (RoundD src));
11848   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11849   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11850   ins_encode %{
11851     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11852   %}
11853   ins_pipe(pipe_slow);
11854 %}
11855 
11856 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11857 %{
11858   match(Set dst (RoundF src));
11859   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11860   format %{ "round_float $dst,$src" %}
11861   ins_encode %{
11862     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11863   %}
11864   ins_pipe(pipe_slow);
11865 %}
11866 
11867 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11868 %{
11869   predicate(!UseXmmI2F);
11870   match(Set dst (ConvI2F src));
11871 
11872   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11873   ins_encode %{
11874     if (UseAVX > 0) {
11875       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11876     }
11877     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11878   %}
11879   ins_pipe(pipe_slow); // XXX
11880 %}
11881 
11882 instruct convI2F_reg_mem(regF dst, memory src)
11883 %{
11884   predicate(UseAVX == 0);
11885   match(Set dst (ConvI2F (LoadI src)));
11886 
11887   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11888   ins_encode %{
11889     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11890   %}
11891   ins_pipe(pipe_slow); // XXX
11892 %}
11893 
11894 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11895 %{
11896   predicate(!UseXmmI2D);
11897   match(Set dst (ConvI2D src));
11898 
11899   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11900   ins_encode %{
11901     if (UseAVX > 0) {
11902       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11903     }
11904     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11905   %}
11906   ins_pipe(pipe_slow); // XXX
11907 %}
11908 
11909 instruct convI2D_reg_mem(regD dst, memory src)
11910 %{
11911   predicate(UseAVX == 0);
11912   match(Set dst (ConvI2D (LoadI src)));
11913 
11914   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11915   ins_encode %{
11916     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11917   %}
11918   ins_pipe(pipe_slow); // XXX
11919 %}
11920 
11921 instruct convXI2F_reg(regF dst, rRegI src)
11922 %{
11923   predicate(UseXmmI2F);
11924   match(Set dst (ConvI2F src));
11925 
11926   format %{ "movdl $dst, $src\n\t"
11927             "cvtdq2psl $dst, $dst\t# i2f" %}
11928   ins_encode %{
11929     __ movdl($dst$$XMMRegister, $src$$Register);
11930     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11931   %}
11932   ins_pipe(pipe_slow); // XXX
11933 %}
11934 
11935 instruct convXI2D_reg(regD dst, rRegI src)
11936 %{
11937   predicate(UseXmmI2D);
11938   match(Set dst (ConvI2D src));
11939 
11940   format %{ "movdl $dst, $src\n\t"
11941             "cvtdq2pdl $dst, $dst\t# i2d" %}
11942   ins_encode %{
11943     __ movdl($dst$$XMMRegister, $src$$Register);
11944     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11945   %}
11946   ins_pipe(pipe_slow); // XXX
11947 %}
11948 
11949 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11950 %{
11951   match(Set dst (ConvL2F src));
11952 
11953   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11954   ins_encode %{
11955     if (UseAVX > 0) {
11956       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11957     }
11958     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11959   %}
11960   ins_pipe(pipe_slow); // XXX
11961 %}
11962 
11963 instruct convL2F_reg_mem(regF dst, memory src)
11964 %{
11965   predicate(UseAVX == 0);
11966   match(Set dst (ConvL2F (LoadL src)));
11967 
11968   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11969   ins_encode %{
11970     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11971   %}
11972   ins_pipe(pipe_slow); // XXX
11973 %}
11974 
11975 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11976 %{
11977   match(Set dst (ConvL2D src));
11978 
11979   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11980   ins_encode %{
11981     if (UseAVX > 0) {
11982       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11983     }
11984     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11985   %}
11986   ins_pipe(pipe_slow); // XXX
11987 %}
11988 
11989 instruct convL2D_reg_mem(regD dst, memory src)
11990 %{
11991   predicate(UseAVX == 0);
11992   match(Set dst (ConvL2D (LoadL src)));
11993 
11994   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11995   ins_encode %{
11996     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11997   %}
11998   ins_pipe(pipe_slow); // XXX
11999 %}
12000 
12001 instruct convI2L_reg_reg(rRegL dst, rRegI src)
12002 %{
12003   match(Set dst (ConvI2L src));
12004 
12005   ins_cost(125);
12006   format %{ "movslq  $dst, $src\t# i2l" %}
12007   ins_encode %{
12008     __ movslq($dst$$Register, $src$$Register);
12009   %}
12010   ins_pipe(ialu_reg_reg);
12011 %}
12012 
12013 // Zero-extend convert int to long
12014 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
12015 %{
12016   match(Set dst (AndL (ConvI2L src) mask));
12017 
12018   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12019   ins_encode %{
12020     if ($dst$$reg != $src$$reg) {
12021       __ movl($dst$$Register, $src$$Register);
12022     }
12023   %}
12024   ins_pipe(ialu_reg_reg);
12025 %}
12026 
12027 // Zero-extend convert int to long
12028 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
12029 %{
12030   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
12031 
12032   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12033   ins_encode %{
12034     __ movl($dst$$Register, $src$$Address);
12035   %}
12036   ins_pipe(ialu_reg_mem);
12037 %}
12038 
12039 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
12040 %{
12041   match(Set dst (AndL src mask));
12042 
12043   format %{ "movl    $dst, $src\t# zero-extend long" %}
12044   ins_encode %{
12045     __ movl($dst$$Register, $src$$Register);
12046   %}
12047   ins_pipe(ialu_reg_reg);
12048 %}
12049 
12050 instruct convL2I_reg_reg(rRegI dst, rRegL src)
12051 %{
12052   match(Set dst (ConvL2I src));
12053 
12054   format %{ "movl    $dst, $src\t# l2i" %}
12055   ins_encode %{
12056     __ movl($dst$$Register, $src$$Register);
12057   %}
12058   ins_pipe(ialu_reg_reg);
12059 %}
12060 
12061 
12062 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
12063   match(Set dst (MoveF2I src));
12064   effect(DEF dst, USE src);
12065 
12066   ins_cost(125);
12067   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
12068   ins_encode %{
12069     __ movl($dst$$Register, Address(rsp, $src$$disp));
12070   %}
12071   ins_pipe(ialu_reg_mem);
12072 %}
12073 
12074 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12075   match(Set dst (MoveI2F src));
12076   effect(DEF dst, USE src);
12077 
12078   ins_cost(125);
12079   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
12080   ins_encode %{
12081     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
12082   %}
12083   ins_pipe(pipe_slow);
12084 %}
12085 
12086 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12087   match(Set dst (MoveD2L src));
12088   effect(DEF dst, USE src);
12089 
12090   ins_cost(125);
12091   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12092   ins_encode %{
12093     __ movq($dst$$Register, Address(rsp, $src$$disp));
12094   %}
12095   ins_pipe(ialu_reg_mem);
12096 %}
12097 
12098 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12099   predicate(!UseXmmLoadAndClearUpper);
12100   match(Set dst (MoveL2D src));
12101   effect(DEF dst, USE src);
12102 
12103   ins_cost(125);
12104   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12105   ins_encode %{
12106     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12107   %}
12108   ins_pipe(pipe_slow);
12109 %}
12110 
12111 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12112   predicate(UseXmmLoadAndClearUpper);
12113   match(Set dst (MoveL2D src));
12114   effect(DEF dst, USE src);
12115 
12116   ins_cost(125);
12117   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12118   ins_encode %{
12119     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12120   %}
12121   ins_pipe(pipe_slow);
12122 %}
12123 
12124 
12125 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12126   match(Set dst (MoveF2I src));
12127   effect(DEF dst, USE src);
12128 
12129   ins_cost(95); // XXX
12130   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12131   ins_encode %{
12132     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12133   %}
12134   ins_pipe(pipe_slow);
12135 %}
12136 
12137 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12138   match(Set dst (MoveI2F src));
12139   effect(DEF dst, USE src);
12140 
12141   ins_cost(100);
12142   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12143   ins_encode %{
12144     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12145   %}
12146   ins_pipe( ialu_mem_reg );
12147 %}
12148 
12149 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12150   match(Set dst (MoveD2L src));
12151   effect(DEF dst, USE src);
12152 
12153   ins_cost(95); // XXX
12154   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12155   ins_encode %{
12156     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12157   %}
12158   ins_pipe(pipe_slow);
12159 %}
12160 
12161 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12162   match(Set dst (MoveL2D src));
12163   effect(DEF dst, USE src);
12164 
12165   ins_cost(100);
12166   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12167   ins_encode %{
12168     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12169   %}
12170   ins_pipe(ialu_mem_reg);
12171 %}
12172 
12173 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12174   match(Set dst (MoveF2I src));
12175   effect(DEF dst, USE src);
12176   ins_cost(85);
12177   format %{ "movd    $dst,$src\t# MoveF2I" %}
12178   ins_encode %{
12179     __ movdl($dst$$Register, $src$$XMMRegister);
12180   %}
12181   ins_pipe( pipe_slow );
12182 %}
12183 
12184 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12185   match(Set dst (MoveD2L src));
12186   effect(DEF dst, USE src);
12187   ins_cost(85);
12188   format %{ "movd    $dst,$src\t# MoveD2L" %}
12189   ins_encode %{
12190     __ movdq($dst$$Register, $src$$XMMRegister);
12191   %}
12192   ins_pipe( pipe_slow );
12193 %}
12194 
12195 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12196   match(Set dst (MoveI2F src));
12197   effect(DEF dst, USE src);
12198   ins_cost(100);
12199   format %{ "movd    $dst,$src\t# MoveI2F" %}
12200   ins_encode %{
12201     __ movdl($dst$$XMMRegister, $src$$Register);
12202   %}
12203   ins_pipe( pipe_slow );
12204 %}
12205 
12206 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12207   match(Set dst (MoveL2D src));
12208   effect(DEF dst, USE src);
12209   ins_cost(100);
12210   format %{ "movd    $dst,$src\t# MoveL2D" %}
12211   ins_encode %{
12212      __ movdq($dst$$XMMRegister, $src$$Register);
12213   %}
12214   ins_pipe( pipe_slow );
12215 %}
12216 
12217 // Fast clearing of an array
12218 // Small non-constant lenght ClearArray for non-AVX512 targets.
12219 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12220                   Universe dummy, rFlagsReg cr)
12221 %{
12222   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12223   match(Set dummy (ClearArray cnt base));
12224   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12225 
12226   format %{ $$template
12227     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12228     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12229     $$emit$$"jg      LARGE\n\t"
12230     $$emit$$"dec     rcx\n\t"
12231     $$emit$$"js      DONE\t# Zero length\n\t"
12232     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12233     $$emit$$"dec     rcx\n\t"
12234     $$emit$$"jge     LOOP\n\t"
12235     $$emit$$"jmp     DONE\n\t"
12236     $$emit$$"# LARGE:\n\t"
12237     if (UseFastStosb) {
12238        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12239        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12240     } else if (UseXMMForObjInit) {
12241        $$emit$$"mov     rdi,rax\n\t"
12242        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12243        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12244        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12245        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12246        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12247        $$emit$$"add     0x40,rax\n\t"
12248        $$emit$$"# L_zero_64_bytes:\n\t"
12249        $$emit$$"sub     0x8,rcx\n\t"
12250        $$emit$$"jge     L_loop\n\t"
12251        $$emit$$"add     0x4,rcx\n\t"
12252        $$emit$$"jl      L_tail\n\t"
12253        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12254        $$emit$$"add     0x20,rax\n\t"
12255        $$emit$$"sub     0x4,rcx\n\t"
12256        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12257        $$emit$$"add     0x4,rcx\n\t"
12258        $$emit$$"jle     L_end\n\t"
12259        $$emit$$"dec     rcx\n\t"
12260        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12261        $$emit$$"vmovq   xmm0,(rax)\n\t"
12262        $$emit$$"add     0x8,rax\n\t"
12263        $$emit$$"dec     rcx\n\t"
12264        $$emit$$"jge     L_sloop\n\t"
12265        $$emit$$"# L_end:\n\t"
12266     } else {
12267        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12268     }
12269     $$emit$$"# DONE"
12270   %}
12271   ins_encode %{
12272     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12273                  $tmp$$XMMRegister, false, knoreg);
12274   %}
12275   ins_pipe(pipe_slow);
12276 %}
12277 
12278 // Small non-constant length ClearArray for AVX512 targets.
12279 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12280                        Universe dummy, rFlagsReg cr)
12281 %{
12282   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12283   match(Set dummy (ClearArray cnt base));
12284   ins_cost(125);
12285   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12286 
12287   format %{ $$template
12288     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12289     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12290     $$emit$$"jg      LARGE\n\t"
12291     $$emit$$"dec     rcx\n\t"
12292     $$emit$$"js      DONE\t# Zero length\n\t"
12293     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12294     $$emit$$"dec     rcx\n\t"
12295     $$emit$$"jge     LOOP\n\t"
12296     $$emit$$"jmp     DONE\n\t"
12297     $$emit$$"# LARGE:\n\t"
12298     if (UseFastStosb) {
12299        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12300        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12301     } else if (UseXMMForObjInit) {
12302        $$emit$$"mov     rdi,rax\n\t"
12303        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12304        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12305        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12306        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12307        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12308        $$emit$$"add     0x40,rax\n\t"
12309        $$emit$$"# L_zero_64_bytes:\n\t"
12310        $$emit$$"sub     0x8,rcx\n\t"
12311        $$emit$$"jge     L_loop\n\t"
12312        $$emit$$"add     0x4,rcx\n\t"
12313        $$emit$$"jl      L_tail\n\t"
12314        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12315        $$emit$$"add     0x20,rax\n\t"
12316        $$emit$$"sub     0x4,rcx\n\t"
12317        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12318        $$emit$$"add     0x4,rcx\n\t"
12319        $$emit$$"jle     L_end\n\t"
12320        $$emit$$"dec     rcx\n\t"
12321        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12322        $$emit$$"vmovq   xmm0,(rax)\n\t"
12323        $$emit$$"add     0x8,rax\n\t"
12324        $$emit$$"dec     rcx\n\t"
12325        $$emit$$"jge     L_sloop\n\t"
12326        $$emit$$"# L_end:\n\t"
12327     } else {
12328        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12329     }
12330     $$emit$$"# DONE"
12331   %}
12332   ins_encode %{
12333     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12334                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12335   %}
12336   ins_pipe(pipe_slow);
12337 %}
12338 
12339 // Large non-constant length ClearArray for non-AVX512 targets.
12340 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12341                         Universe dummy, rFlagsReg cr)
12342 %{
12343   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12344   match(Set dummy (ClearArray cnt base));
12345   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12346 
12347   format %{ $$template
12348     if (UseFastStosb) {
12349        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12350        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12351        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12352     } else if (UseXMMForObjInit) {
12353        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12354        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12355        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12356        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12357        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12358        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12359        $$emit$$"add     0x40,rax\n\t"
12360        $$emit$$"# L_zero_64_bytes:\n\t"
12361        $$emit$$"sub     0x8,rcx\n\t"
12362        $$emit$$"jge     L_loop\n\t"
12363        $$emit$$"add     0x4,rcx\n\t"
12364        $$emit$$"jl      L_tail\n\t"
12365        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12366        $$emit$$"add     0x20,rax\n\t"
12367        $$emit$$"sub     0x4,rcx\n\t"
12368        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12369        $$emit$$"add     0x4,rcx\n\t"
12370        $$emit$$"jle     L_end\n\t"
12371        $$emit$$"dec     rcx\n\t"
12372        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12373        $$emit$$"vmovq   xmm0,(rax)\n\t"
12374        $$emit$$"add     0x8,rax\n\t"
12375        $$emit$$"dec     rcx\n\t"
12376        $$emit$$"jge     L_sloop\n\t"
12377        $$emit$$"# L_end:\n\t"
12378     } else {
12379        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12380        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12381     }
12382   %}
12383   ins_encode %{
12384     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12385                  $tmp$$XMMRegister, true, knoreg);
12386   %}
12387   ins_pipe(pipe_slow);
12388 %}
12389 
12390 // Large non-constant length ClearArray for AVX512 targets.
12391 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12392                              Universe dummy, rFlagsReg cr)
12393 %{
12394   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12395   match(Set dummy (ClearArray cnt base));
12396   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12397 
12398   format %{ $$template
12399     if (UseFastStosb) {
12400        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12401        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12402        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12403     } else if (UseXMMForObjInit) {
12404        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12405        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12406        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12407        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12408        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12409        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12410        $$emit$$"add     0x40,rax\n\t"
12411        $$emit$$"# L_zero_64_bytes:\n\t"
12412        $$emit$$"sub     0x8,rcx\n\t"
12413        $$emit$$"jge     L_loop\n\t"
12414        $$emit$$"add     0x4,rcx\n\t"
12415        $$emit$$"jl      L_tail\n\t"
12416        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12417        $$emit$$"add     0x20,rax\n\t"
12418        $$emit$$"sub     0x4,rcx\n\t"
12419        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12420        $$emit$$"add     0x4,rcx\n\t"
12421        $$emit$$"jle     L_end\n\t"
12422        $$emit$$"dec     rcx\n\t"
12423        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12424        $$emit$$"vmovq   xmm0,(rax)\n\t"
12425        $$emit$$"add     0x8,rax\n\t"
12426        $$emit$$"dec     rcx\n\t"
12427        $$emit$$"jge     L_sloop\n\t"
12428        $$emit$$"# L_end:\n\t"
12429     } else {
12430        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12431        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12432     }
12433   %}
12434   ins_encode %{
12435     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12436                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12437   %}
12438   ins_pipe(pipe_slow);
12439 %}
12440 
12441 // Small constant length ClearArray for AVX512 targets.
12442 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12443 %{
12444   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12445   match(Set dummy (ClearArray cnt base));
12446   ins_cost(100);
12447   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12448   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12449   ins_encode %{
12450    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12451   %}
12452   ins_pipe(pipe_slow);
12453 %}
12454 
12455 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12456                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12457 %{
12458   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12459   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12460   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12461 
12462   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12463   ins_encode %{
12464     __ string_compare($str1$$Register, $str2$$Register,
12465                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12466                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12467   %}
12468   ins_pipe( pipe_slow );
12469 %}
12470 
12471 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12472                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12473 %{
12474   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12475   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12476   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12477 
12478   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12479   ins_encode %{
12480     __ string_compare($str1$$Register, $str2$$Register,
12481                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12482                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12483   %}
12484   ins_pipe( pipe_slow );
12485 %}
12486 
12487 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12488                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12489 %{
12490   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12491   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12492   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12493 
12494   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12495   ins_encode %{
12496     __ string_compare($str1$$Register, $str2$$Register,
12497                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12498                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12499   %}
12500   ins_pipe( pipe_slow );
12501 %}
12502 
12503 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12504                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12505 %{
12506   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12507   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12508   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12509 
12510   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12511   ins_encode %{
12512     __ string_compare($str1$$Register, $str2$$Register,
12513                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12514                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12515   %}
12516   ins_pipe( pipe_slow );
12517 %}
12518 
12519 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12520                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12521 %{
12522   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12523   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12524   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12525 
12526   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12527   ins_encode %{
12528     __ string_compare($str1$$Register, $str2$$Register,
12529                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12530                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12531   %}
12532   ins_pipe( pipe_slow );
12533 %}
12534 
12535 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12536                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12537 %{
12538   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12539   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12540   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12541 
12542   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12543   ins_encode %{
12544     __ string_compare($str1$$Register, $str2$$Register,
12545                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12546                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12547   %}
12548   ins_pipe( pipe_slow );
12549 %}
12550 
12551 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12552                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12553 %{
12554   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12555   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12556   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12557 
12558   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12559   ins_encode %{
12560     __ string_compare($str2$$Register, $str1$$Register,
12561                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12562                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12563   %}
12564   ins_pipe( pipe_slow );
12565 %}
12566 
12567 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12568                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12569 %{
12570   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12571   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12572   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12573 
12574   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12575   ins_encode %{
12576     __ string_compare($str2$$Register, $str1$$Register,
12577                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12578                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12579   %}
12580   ins_pipe( pipe_slow );
12581 %}
12582 
12583 // fast search of substring with known size.
12584 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12585                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12586 %{
12587   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12588   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12589   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12590 
12591   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12592   ins_encode %{
12593     int icnt2 = (int)$int_cnt2$$constant;
12594     if (icnt2 >= 16) {
12595       // IndexOf for constant substrings with size >= 16 elements
12596       // which don't need to be loaded through stack.
12597       __ string_indexofC8($str1$$Register, $str2$$Register,
12598                           $cnt1$$Register, $cnt2$$Register,
12599                           icnt2, $result$$Register,
12600                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12601     } else {
12602       // Small strings are loaded through stack if they cross page boundary.
12603       __ string_indexof($str1$$Register, $str2$$Register,
12604                         $cnt1$$Register, $cnt2$$Register,
12605                         icnt2, $result$$Register,
12606                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12607     }
12608   %}
12609   ins_pipe( pipe_slow );
12610 %}
12611 
12612 // fast search of substring with known size.
12613 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12614                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12615 %{
12616   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12617   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12618   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12619 
12620   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12621   ins_encode %{
12622     int icnt2 = (int)$int_cnt2$$constant;
12623     if (icnt2 >= 8) {
12624       // IndexOf for constant substrings with size >= 8 elements
12625       // which don't need to be loaded through stack.
12626       __ string_indexofC8($str1$$Register, $str2$$Register,
12627                           $cnt1$$Register, $cnt2$$Register,
12628                           icnt2, $result$$Register,
12629                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12630     } else {
12631       // Small strings are loaded through stack if they cross page boundary.
12632       __ string_indexof($str1$$Register, $str2$$Register,
12633                         $cnt1$$Register, $cnt2$$Register,
12634                         icnt2, $result$$Register,
12635                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12636     }
12637   %}
12638   ins_pipe( pipe_slow );
12639 %}
12640 
12641 // fast search of substring with known size.
12642 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12643                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12644 %{
12645   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12646   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12647   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12648 
12649   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12650   ins_encode %{
12651     int icnt2 = (int)$int_cnt2$$constant;
12652     if (icnt2 >= 8) {
12653       // IndexOf for constant substrings with size >= 8 elements
12654       // which don't need to be loaded through stack.
12655       __ string_indexofC8($str1$$Register, $str2$$Register,
12656                           $cnt1$$Register, $cnt2$$Register,
12657                           icnt2, $result$$Register,
12658                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12659     } else {
12660       // Small strings are loaded through stack if they cross page boundary.
12661       __ string_indexof($str1$$Register, $str2$$Register,
12662                         $cnt1$$Register, $cnt2$$Register,
12663                         icnt2, $result$$Register,
12664                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12665     }
12666   %}
12667   ins_pipe( pipe_slow );
12668 %}
12669 
12670 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12671                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12672 %{
12673   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12674   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12675   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12676 
12677   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12678   ins_encode %{
12679     __ string_indexof($str1$$Register, $str2$$Register,
12680                       $cnt1$$Register, $cnt2$$Register,
12681                       (-1), $result$$Register,
12682                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12683   %}
12684   ins_pipe( pipe_slow );
12685 %}
12686 
12687 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12688                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12689 %{
12690   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12691   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12692   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12693 
12694   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12695   ins_encode %{
12696     __ string_indexof($str1$$Register, $str2$$Register,
12697                       $cnt1$$Register, $cnt2$$Register,
12698                       (-1), $result$$Register,
12699                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12700   %}
12701   ins_pipe( pipe_slow );
12702 %}
12703 
12704 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12705                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12706 %{
12707   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12708   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12709   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12710 
12711   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12712   ins_encode %{
12713     __ string_indexof($str1$$Register, $str2$$Register,
12714                       $cnt1$$Register, $cnt2$$Register,
12715                       (-1), $result$$Register,
12716                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12717   %}
12718   ins_pipe( pipe_slow );
12719 %}
12720 
12721 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12722                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12723 %{
12724   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12725   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12726   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12727   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12728   ins_encode %{
12729     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12730                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12731   %}
12732   ins_pipe( pipe_slow );
12733 %}
12734 
12735 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12736                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12737 %{
12738   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12739   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12740   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12741   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12742   ins_encode %{
12743     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12744                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12745   %}
12746   ins_pipe( pipe_slow );
12747 %}
12748 
12749 // fast string equals
12750 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12751                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12752 %{
12753   predicate(!VM_Version::supports_avx512vlbw());
12754   match(Set result (StrEquals (Binary str1 str2) cnt));
12755   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12756 
12757   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12758   ins_encode %{
12759     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12760                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12761                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12762   %}
12763   ins_pipe( pipe_slow );
12764 %}
12765 
12766 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12767                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12768 %{
12769   predicate(VM_Version::supports_avx512vlbw());
12770   match(Set result (StrEquals (Binary str1 str2) cnt));
12771   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12772 
12773   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12774   ins_encode %{
12775     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12776                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12777                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12778   %}
12779   ins_pipe( pipe_slow );
12780 %}
12781 
12782 // fast array equals
12783 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12784                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12785 %{
12786   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12787   match(Set result (AryEq ary1 ary2));
12788   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12789 
12790   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12791   ins_encode %{
12792     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12793                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12794                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12795   %}
12796   ins_pipe( pipe_slow );
12797 %}
12798 
12799 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12800                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12801 %{
12802   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12803   match(Set result (AryEq ary1 ary2));
12804   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12805 
12806   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12807   ins_encode %{
12808     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12809                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12810                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12811   %}
12812   ins_pipe( pipe_slow );
12813 %}
12814 
12815 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12816                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12817 %{
12818   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12819   match(Set result (AryEq ary1 ary2));
12820   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12821 
12822   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12823   ins_encode %{
12824     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12825                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12826                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12827   %}
12828   ins_pipe( pipe_slow );
12829 %}
12830 
12831 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12832                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12833 %{
12834   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12835   match(Set result (AryEq ary1 ary2));
12836   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12837 
12838   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12839   ins_encode %{
12840     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12841                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12842                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12843   %}
12844   ins_pipe( pipe_slow );
12845 %}
12846 
12847 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12848                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12849                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12850                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12851                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12852 %{
12853   predicate(UseAVX >= 2);
12854   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12855   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12856          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12857          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12858          USE basic_type, KILL cr);
12859 
12860   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12861   ins_encode %{
12862     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12863                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12864                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12865                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12866                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12867                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12868                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12869   %}
12870   ins_pipe( pipe_slow );
12871 %}
12872 
12873 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12874                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12875 %{
12876   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12877   match(Set result (CountPositives ary1 len));
12878   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12879 
12880   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12881   ins_encode %{
12882     __ count_positives($ary1$$Register, $len$$Register,
12883                        $result$$Register, $tmp3$$Register,
12884                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12885   %}
12886   ins_pipe( pipe_slow );
12887 %}
12888 
12889 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12890                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12891 %{
12892   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12893   match(Set result (CountPositives ary1 len));
12894   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12895 
12896   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12897   ins_encode %{
12898     __ count_positives($ary1$$Register, $len$$Register,
12899                        $result$$Register, $tmp3$$Register,
12900                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12901   %}
12902   ins_pipe( pipe_slow );
12903 %}
12904 
12905 // fast char[] to byte[] compression
12906 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12907                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12908   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12909   match(Set result (StrCompressedCopy src (Binary dst len)));
12910   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12911          USE_KILL len, KILL tmp5, KILL cr);
12912 
12913   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12914   ins_encode %{
12915     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12916                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12917                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12918                            knoreg, knoreg);
12919   %}
12920   ins_pipe( pipe_slow );
12921 %}
12922 
12923 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12924                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12925   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12926   match(Set result (StrCompressedCopy src (Binary dst len)));
12927   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12928          USE_KILL len, KILL tmp5, KILL cr);
12929 
12930   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12931   ins_encode %{
12932     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12933                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12934                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12935                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12936   %}
12937   ins_pipe( pipe_slow );
12938 %}
12939 // fast byte[] to char[] inflation
12940 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12941                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12942   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12943   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12944   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12945 
12946   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12947   ins_encode %{
12948     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12949                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12950   %}
12951   ins_pipe( pipe_slow );
12952 %}
12953 
12954 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12955                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12956   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12957   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12958   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12959 
12960   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12961   ins_encode %{
12962     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12963                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12964   %}
12965   ins_pipe( pipe_slow );
12966 %}
12967 
12968 // encode char[] to byte[] in ISO_8859_1
12969 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12970                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12971                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12972   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12973   match(Set result (EncodeISOArray src (Binary dst len)));
12974   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12975 
12976   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12977   ins_encode %{
12978     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12979                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12980                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12981   %}
12982   ins_pipe( pipe_slow );
12983 %}
12984 
12985 // encode char[] to byte[] in ASCII
12986 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12987                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12988                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12989   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12990   match(Set result (EncodeISOArray src (Binary dst len)));
12991   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12992 
12993   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12994   ins_encode %{
12995     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12996                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12997                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12998   %}
12999   ins_pipe( pipe_slow );
13000 %}
13001 
13002 //----------Overflow Math Instructions-----------------------------------------
13003 
13004 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13005 %{
13006   match(Set cr (OverflowAddI op1 op2));
13007   effect(DEF cr, USE_KILL op1, USE op2);
13008 
13009   format %{ "addl    $op1, $op2\t# overflow check int" %}
13010 
13011   ins_encode %{
13012     __ addl($op1$$Register, $op2$$Register);
13013   %}
13014   ins_pipe(ialu_reg_reg);
13015 %}
13016 
13017 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
13018 %{
13019   match(Set cr (OverflowAddI op1 op2));
13020   effect(DEF cr, USE_KILL op1, USE op2);
13021 
13022   format %{ "addl    $op1, $op2\t# overflow check int" %}
13023 
13024   ins_encode %{
13025     __ addl($op1$$Register, $op2$$constant);
13026   %}
13027   ins_pipe(ialu_reg_reg);
13028 %}
13029 
13030 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13031 %{
13032   match(Set cr (OverflowAddL op1 op2));
13033   effect(DEF cr, USE_KILL op1, USE op2);
13034 
13035   format %{ "addq    $op1, $op2\t# overflow check long" %}
13036   ins_encode %{
13037     __ addq($op1$$Register, $op2$$Register);
13038   %}
13039   ins_pipe(ialu_reg_reg);
13040 %}
13041 
13042 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
13043 %{
13044   match(Set cr (OverflowAddL op1 op2));
13045   effect(DEF cr, USE_KILL op1, USE op2);
13046 
13047   format %{ "addq    $op1, $op2\t# overflow check long" %}
13048   ins_encode %{
13049     __ addq($op1$$Register, $op2$$constant);
13050   %}
13051   ins_pipe(ialu_reg_reg);
13052 %}
13053 
13054 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13055 %{
13056   match(Set cr (OverflowSubI op1 op2));
13057 
13058   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13059   ins_encode %{
13060     __ cmpl($op1$$Register, $op2$$Register);
13061   %}
13062   ins_pipe(ialu_reg_reg);
13063 %}
13064 
13065 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13066 %{
13067   match(Set cr (OverflowSubI op1 op2));
13068 
13069   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13070   ins_encode %{
13071     __ cmpl($op1$$Register, $op2$$constant);
13072   %}
13073   ins_pipe(ialu_reg_reg);
13074 %}
13075 
13076 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13077 %{
13078   match(Set cr (OverflowSubL op1 op2));
13079 
13080   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13081   ins_encode %{
13082     __ cmpq($op1$$Register, $op2$$Register);
13083   %}
13084   ins_pipe(ialu_reg_reg);
13085 %}
13086 
13087 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13088 %{
13089   match(Set cr (OverflowSubL op1 op2));
13090 
13091   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13092   ins_encode %{
13093     __ cmpq($op1$$Register, $op2$$constant);
13094   %}
13095   ins_pipe(ialu_reg_reg);
13096 %}
13097 
13098 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13099 %{
13100   match(Set cr (OverflowSubI zero op2));
13101   effect(DEF cr, USE_KILL op2);
13102 
13103   format %{ "negl    $op2\t# overflow check int" %}
13104   ins_encode %{
13105     __ negl($op2$$Register);
13106   %}
13107   ins_pipe(ialu_reg_reg);
13108 %}
13109 
13110 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13111 %{
13112   match(Set cr (OverflowSubL zero op2));
13113   effect(DEF cr, USE_KILL op2);
13114 
13115   format %{ "negq    $op2\t# overflow check long" %}
13116   ins_encode %{
13117     __ negq($op2$$Register);
13118   %}
13119   ins_pipe(ialu_reg_reg);
13120 %}
13121 
13122 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13123 %{
13124   match(Set cr (OverflowMulI op1 op2));
13125   effect(DEF cr, USE_KILL op1, USE op2);
13126 
13127   format %{ "imull    $op1, $op2\t# overflow check int" %}
13128   ins_encode %{
13129     __ imull($op1$$Register, $op2$$Register);
13130   %}
13131   ins_pipe(ialu_reg_reg_alu0);
13132 %}
13133 
13134 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13135 %{
13136   match(Set cr (OverflowMulI op1 op2));
13137   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13138 
13139   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13140   ins_encode %{
13141     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13142   %}
13143   ins_pipe(ialu_reg_reg_alu0);
13144 %}
13145 
13146 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13147 %{
13148   match(Set cr (OverflowMulL op1 op2));
13149   effect(DEF cr, USE_KILL op1, USE op2);
13150 
13151   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13152   ins_encode %{
13153     __ imulq($op1$$Register, $op2$$Register);
13154   %}
13155   ins_pipe(ialu_reg_reg_alu0);
13156 %}
13157 
13158 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13159 %{
13160   match(Set cr (OverflowMulL op1 op2));
13161   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13162 
13163   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13164   ins_encode %{
13165     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13166   %}
13167   ins_pipe(ialu_reg_reg_alu0);
13168 %}
13169 
13170 
13171 //----------Control Flow Instructions------------------------------------------
13172 // Signed compare Instructions
13173 
13174 // XXX more variants!!
13175 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13176 %{
13177   match(Set cr (CmpI op1 op2));
13178   effect(DEF cr, USE op1, USE op2);
13179 
13180   format %{ "cmpl    $op1, $op2" %}
13181   ins_encode %{
13182     __ cmpl($op1$$Register, $op2$$Register);
13183   %}
13184   ins_pipe(ialu_cr_reg_reg);
13185 %}
13186 
13187 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13188 %{
13189   match(Set cr (CmpI op1 op2));
13190 
13191   format %{ "cmpl    $op1, $op2" %}
13192   ins_encode %{
13193     __ cmpl($op1$$Register, $op2$$constant);
13194   %}
13195   ins_pipe(ialu_cr_reg_imm);
13196 %}
13197 
13198 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13199 %{
13200   match(Set cr (CmpI op1 (LoadI op2)));
13201 
13202   ins_cost(500); // XXX
13203   format %{ "cmpl    $op1, $op2" %}
13204   ins_encode %{
13205     __ cmpl($op1$$Register, $op2$$Address);
13206   %}
13207   ins_pipe(ialu_cr_reg_mem);
13208 %}
13209 
13210 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13211 %{
13212   match(Set cr (CmpI src zero));
13213 
13214   format %{ "testl   $src, $src" %}
13215   ins_encode %{
13216     __ testl($src$$Register, $src$$Register);
13217   %}
13218   ins_pipe(ialu_cr_reg_imm);
13219 %}
13220 
13221 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13222 %{
13223   match(Set cr (CmpI (AndI src con) zero));
13224 
13225   format %{ "testl   $src, $con" %}
13226   ins_encode %{
13227     __ testl($src$$Register, $con$$constant);
13228   %}
13229   ins_pipe(ialu_cr_reg_imm);
13230 %}
13231 
13232 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13233 %{
13234   match(Set cr (CmpI (AndI src1 src2) zero));
13235 
13236   format %{ "testl   $src1, $src2" %}
13237   ins_encode %{
13238     __ testl($src1$$Register, $src2$$Register);
13239   %}
13240   ins_pipe(ialu_cr_reg_imm);
13241 %}
13242 
13243 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13244 %{
13245   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13246 
13247   format %{ "testl   $src, $mem" %}
13248   ins_encode %{
13249     __ testl($src$$Register, $mem$$Address);
13250   %}
13251   ins_pipe(ialu_cr_reg_mem);
13252 %}
13253 
13254 // Unsigned compare Instructions; really, same as signed except they
13255 // produce an rFlagsRegU instead of rFlagsReg.
13256 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13257 %{
13258   match(Set cr (CmpU op1 op2));
13259 
13260   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13261   ins_encode %{
13262     __ cmpl($op1$$Register, $op2$$Register);
13263   %}
13264   ins_pipe(ialu_cr_reg_reg);
13265 %}
13266 
13267 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13268 %{
13269   match(Set cr (CmpU op1 op2));
13270 
13271   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13272   ins_encode %{
13273     __ cmpl($op1$$Register, $op2$$constant);
13274   %}
13275   ins_pipe(ialu_cr_reg_imm);
13276 %}
13277 
13278 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13279 %{
13280   match(Set cr (CmpU op1 (LoadI op2)));
13281 
13282   ins_cost(500); // XXX
13283   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13284   ins_encode %{
13285     __ cmpl($op1$$Register, $op2$$Address);
13286   %}
13287   ins_pipe(ialu_cr_reg_mem);
13288 %}
13289 
13290 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13291 %{
13292   match(Set cr (CmpU src zero));
13293 
13294   format %{ "testl   $src, $src\t# unsigned" %}
13295   ins_encode %{
13296     __ testl($src$$Register, $src$$Register);
13297   %}
13298   ins_pipe(ialu_cr_reg_imm);
13299 %}
13300 
13301 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13302 %{
13303   match(Set cr (CmpP op1 op2));
13304 
13305   format %{ "cmpq    $op1, $op2\t# ptr" %}
13306   ins_encode %{
13307     __ cmpq($op1$$Register, $op2$$Register);
13308   %}
13309   ins_pipe(ialu_cr_reg_reg);
13310 %}
13311 
13312 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13313 %{
13314   match(Set cr (CmpP op1 (LoadP op2)));
13315   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13316 
13317   ins_cost(500); // XXX
13318   format %{ "cmpq    $op1, $op2\t# ptr" %}
13319   ins_encode %{
13320     __ cmpq($op1$$Register, $op2$$Address);
13321   %}
13322   ins_pipe(ialu_cr_reg_mem);
13323 %}
13324 
13325 // XXX this is generalized by compP_rReg_mem???
13326 // Compare raw pointer (used in out-of-heap check).
13327 // Only works because non-oop pointers must be raw pointers
13328 // and raw pointers have no anti-dependencies.
13329 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13330 %{
13331   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13332             n->in(2)->as_Load()->barrier_data() == 0);
13333   match(Set cr (CmpP op1 (LoadP op2)));
13334 
13335   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13336   ins_encode %{
13337     __ cmpq($op1$$Register, $op2$$Address);
13338   %}
13339   ins_pipe(ialu_cr_reg_mem);
13340 %}
13341 
13342 // This will generate a signed flags result. This should be OK since
13343 // any compare to a zero should be eq/neq.
13344 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13345 %{
13346   match(Set cr (CmpP src zero));
13347 
13348   format %{ "testq   $src, $src\t# ptr" %}
13349   ins_encode %{
13350     __ testq($src$$Register, $src$$Register);
13351   %}
13352   ins_pipe(ialu_cr_reg_imm);
13353 %}
13354 
13355 // This will generate a signed flags result. This should be OK since
13356 // any compare to a zero should be eq/neq.
13357 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13358 %{
13359   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13360             n->in(1)->as_Load()->barrier_data() == 0);
13361   match(Set cr (CmpP (LoadP op) zero));
13362 
13363   ins_cost(500); // XXX
13364   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13365   ins_encode %{
13366     __ testq($op$$Address, 0xFFFFFFFF);
13367   %}
13368   ins_pipe(ialu_cr_reg_imm);
13369 %}
13370 
13371 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13372 %{
13373   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13374             n->in(1)->as_Load()->barrier_data() == 0);
13375   match(Set cr (CmpP (LoadP mem) zero));
13376 
13377   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13378   ins_encode %{
13379     __ cmpq(r12, $mem$$Address);
13380   %}
13381   ins_pipe(ialu_cr_reg_mem);
13382 %}
13383 
13384 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13385 %{
13386   match(Set cr (CmpN op1 op2));
13387 
13388   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13389   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13390   ins_pipe(ialu_cr_reg_reg);
13391 %}
13392 
13393 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13394 %{
13395   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13396   match(Set cr (CmpN src (LoadN mem)));
13397 
13398   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13399   ins_encode %{
13400     __ cmpl($src$$Register, $mem$$Address);
13401   %}
13402   ins_pipe(ialu_cr_reg_mem);
13403 %}
13404 
13405 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13406   match(Set cr (CmpN op1 op2));
13407 
13408   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13409   ins_encode %{
13410     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13411   %}
13412   ins_pipe(ialu_cr_reg_imm);
13413 %}
13414 
13415 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13416 %{
13417   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13418   match(Set cr (CmpN src (LoadN mem)));
13419 
13420   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13421   ins_encode %{
13422     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13423   %}
13424   ins_pipe(ialu_cr_reg_mem);
13425 %}
13426 
13427 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13428   match(Set cr (CmpN op1 op2));
13429 
13430   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13431   ins_encode %{
13432     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13433   %}
13434   ins_pipe(ialu_cr_reg_imm);
13435 %}
13436 
13437 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13438 %{
13439   predicate(!UseCompactObjectHeaders);
13440   match(Set cr (CmpN src (LoadNKlass mem)));
13441 
13442   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13443   ins_encode %{
13444     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13445   %}
13446   ins_pipe(ialu_cr_reg_mem);
13447 %}
13448 
13449 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13450   match(Set cr (CmpN src zero));
13451 
13452   format %{ "testl   $src, $src\t# compressed ptr" %}
13453   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13454   ins_pipe(ialu_cr_reg_imm);
13455 %}
13456 
13457 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13458 %{
13459   predicate(CompressedOops::base() != nullptr &&
13460             n->in(1)->as_Load()->barrier_data() == 0);
13461   match(Set cr (CmpN (LoadN mem) zero));
13462 
13463   ins_cost(500); // XXX
13464   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13465   ins_encode %{
13466     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13467   %}
13468   ins_pipe(ialu_cr_reg_mem);
13469 %}
13470 
13471 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13472 %{
13473   predicate(CompressedOops::base() == nullptr &&
13474             n->in(1)->as_Load()->barrier_data() == 0);
13475   match(Set cr (CmpN (LoadN mem) zero));
13476 
13477   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13478   ins_encode %{
13479     __ cmpl(r12, $mem$$Address);
13480   %}
13481   ins_pipe(ialu_cr_reg_mem);
13482 %}
13483 
13484 // Yanked all unsigned pointer compare operations.
13485 // Pointer compares are done with CmpP which is already unsigned.
13486 
13487 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13488 %{
13489   match(Set cr (CmpL op1 op2));
13490 
13491   format %{ "cmpq    $op1, $op2" %}
13492   ins_encode %{
13493     __ cmpq($op1$$Register, $op2$$Register);
13494   %}
13495   ins_pipe(ialu_cr_reg_reg);
13496 %}
13497 
13498 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13499 %{
13500   match(Set cr (CmpL op1 op2));
13501 
13502   format %{ "cmpq    $op1, $op2" %}
13503   ins_encode %{
13504     __ cmpq($op1$$Register, $op2$$constant);
13505   %}
13506   ins_pipe(ialu_cr_reg_imm);
13507 %}
13508 
13509 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13510 %{
13511   match(Set cr (CmpL op1 (LoadL op2)));
13512 
13513   format %{ "cmpq    $op1, $op2" %}
13514   ins_encode %{
13515     __ cmpq($op1$$Register, $op2$$Address);
13516   %}
13517   ins_pipe(ialu_cr_reg_mem);
13518 %}
13519 
13520 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13521 %{
13522   match(Set cr (CmpL src zero));
13523 
13524   format %{ "testq   $src, $src" %}
13525   ins_encode %{
13526     __ testq($src$$Register, $src$$Register);
13527   %}
13528   ins_pipe(ialu_cr_reg_imm);
13529 %}
13530 
13531 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13532 %{
13533   match(Set cr (CmpL (AndL src con) zero));
13534 
13535   format %{ "testq   $src, $con\t# long" %}
13536   ins_encode %{
13537     __ testq($src$$Register, $con$$constant);
13538   %}
13539   ins_pipe(ialu_cr_reg_imm);
13540 %}
13541 
13542 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13543 %{
13544   match(Set cr (CmpL (AndL src1 src2) zero));
13545 
13546   format %{ "testq   $src1, $src2\t# long" %}
13547   ins_encode %{
13548     __ testq($src1$$Register, $src2$$Register);
13549   %}
13550   ins_pipe(ialu_cr_reg_imm);
13551 %}
13552 
13553 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13554 %{
13555   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13556 
13557   format %{ "testq   $src, $mem" %}
13558   ins_encode %{
13559     __ testq($src$$Register, $mem$$Address);
13560   %}
13561   ins_pipe(ialu_cr_reg_mem);
13562 %}
13563 
13564 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13565 %{
13566   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13567 
13568   format %{ "testq   $src, $mem" %}
13569   ins_encode %{
13570     __ testq($src$$Register, $mem$$Address);
13571   %}
13572   ins_pipe(ialu_cr_reg_mem);
13573 %}
13574 
13575 // Manifest a CmpU result in an integer register.  Very painful.
13576 // This is the test to avoid.
13577 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13578 %{
13579   match(Set dst (CmpU3 src1 src2));
13580   effect(KILL flags);
13581 
13582   ins_cost(275); // XXX
13583   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13584             "movl    $dst, -1\n\t"
13585             "jb,u    done\n\t"
13586             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13587     "done:" %}
13588   ins_encode %{
13589     Label done;
13590     __ cmpl($src1$$Register, $src2$$Register);
13591     __ movl($dst$$Register, -1);
13592     __ jccb(Assembler::below, done);
13593     __ setcc(Assembler::notZero, $dst$$Register);
13594     __ bind(done);
13595   %}
13596   ins_pipe(pipe_slow);
13597 %}
13598 
13599 // Manifest a CmpL result in an integer register.  Very painful.
13600 // This is the test to avoid.
13601 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13602 %{
13603   match(Set dst (CmpL3 src1 src2));
13604   effect(KILL flags);
13605 
13606   ins_cost(275); // XXX
13607   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13608             "movl    $dst, -1\n\t"
13609             "jl,s    done\n\t"
13610             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13611     "done:" %}
13612   ins_encode %{
13613     Label done;
13614     __ cmpq($src1$$Register, $src2$$Register);
13615     __ movl($dst$$Register, -1);
13616     __ jccb(Assembler::less, done);
13617     __ setcc(Assembler::notZero, $dst$$Register);
13618     __ bind(done);
13619   %}
13620   ins_pipe(pipe_slow);
13621 %}
13622 
13623 // Manifest a CmpUL result in an integer register.  Very painful.
13624 // This is the test to avoid.
13625 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13626 %{
13627   match(Set dst (CmpUL3 src1 src2));
13628   effect(KILL flags);
13629 
13630   ins_cost(275); // XXX
13631   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13632             "movl    $dst, -1\n\t"
13633             "jb,u    done\n\t"
13634             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13635     "done:" %}
13636   ins_encode %{
13637     Label done;
13638     __ cmpq($src1$$Register, $src2$$Register);
13639     __ movl($dst$$Register, -1);
13640     __ jccb(Assembler::below, done);
13641     __ setcc(Assembler::notZero, $dst$$Register);
13642     __ bind(done);
13643   %}
13644   ins_pipe(pipe_slow);
13645 %}
13646 
13647 // Unsigned long compare Instructions; really, same as signed long except they
13648 // produce an rFlagsRegU instead of rFlagsReg.
13649 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13650 %{
13651   match(Set cr (CmpUL op1 op2));
13652 
13653   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13654   ins_encode %{
13655     __ cmpq($op1$$Register, $op2$$Register);
13656   %}
13657   ins_pipe(ialu_cr_reg_reg);
13658 %}
13659 
13660 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13661 %{
13662   match(Set cr (CmpUL op1 op2));
13663 
13664   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13665   ins_encode %{
13666     __ cmpq($op1$$Register, $op2$$constant);
13667   %}
13668   ins_pipe(ialu_cr_reg_imm);
13669 %}
13670 
13671 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13672 %{
13673   match(Set cr (CmpUL op1 (LoadL op2)));
13674 
13675   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13676   ins_encode %{
13677     __ cmpq($op1$$Register, $op2$$Address);
13678   %}
13679   ins_pipe(ialu_cr_reg_mem);
13680 %}
13681 
13682 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13683 %{
13684   match(Set cr (CmpUL src zero));
13685 
13686   format %{ "testq   $src, $src\t# unsigned" %}
13687   ins_encode %{
13688     __ testq($src$$Register, $src$$Register);
13689   %}
13690   ins_pipe(ialu_cr_reg_imm);
13691 %}
13692 
13693 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13694 %{
13695   match(Set cr (CmpI (LoadB mem) imm));
13696 
13697   ins_cost(125);
13698   format %{ "cmpb    $mem, $imm" %}
13699   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13700   ins_pipe(ialu_cr_reg_mem);
13701 %}
13702 
13703 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13704 %{
13705   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13706 
13707   ins_cost(125);
13708   format %{ "testb   $mem, $imm\t# ubyte" %}
13709   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13710   ins_pipe(ialu_cr_reg_mem);
13711 %}
13712 
13713 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13714 %{
13715   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13716 
13717   ins_cost(125);
13718   format %{ "testb   $mem, $imm\t# byte" %}
13719   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13720   ins_pipe(ialu_cr_reg_mem);
13721 %}
13722 
13723 //----------Max and Min--------------------------------------------------------
13724 // Min Instructions
13725 
13726 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13727 %{
13728   predicate(!UseAPX);
13729   effect(USE_DEF dst, USE src, USE cr);
13730 
13731   format %{ "cmovlgt $dst, $src\t# min" %}
13732   ins_encode %{
13733     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13734   %}
13735   ins_pipe(pipe_cmov_reg);
13736 %}
13737 
13738 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13739 %{
13740   predicate(UseAPX);
13741   effect(DEF dst, USE src1, USE src2, USE cr);
13742 
13743   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13744   ins_encode %{
13745     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13746   %}
13747   ins_pipe(pipe_cmov_reg);
13748 %}
13749 
13750 instruct minI_rReg(rRegI dst, rRegI src)
13751 %{
13752   predicate(!UseAPX);
13753   match(Set dst (MinI dst src));
13754 
13755   ins_cost(200);
13756   expand %{
13757     rFlagsReg cr;
13758     compI_rReg(cr, dst, src);
13759     cmovI_reg_g(dst, src, cr);
13760   %}
13761 %}
13762 
13763 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13764 %{
13765   predicate(UseAPX);
13766   match(Set dst (MinI src1 src2));
13767   effect(DEF dst, USE src1, USE src2);
13768 
13769   ins_cost(200);
13770   expand %{
13771     rFlagsReg cr;
13772     compI_rReg(cr, src1, src2);
13773     cmovI_reg_g_ndd(dst, src1, src2, cr);
13774   %}
13775 %}
13776 
13777 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13778 %{
13779   predicate(!UseAPX);
13780   effect(USE_DEF dst, USE src, USE cr);
13781 
13782   format %{ "cmovllt $dst, $src\t# max" %}
13783   ins_encode %{
13784     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13785   %}
13786   ins_pipe(pipe_cmov_reg);
13787 %}
13788 
13789 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13790 %{
13791   predicate(UseAPX);
13792   effect(DEF dst, USE src1, USE src2, USE cr);
13793 
13794   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13795   ins_encode %{
13796     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13797   %}
13798   ins_pipe(pipe_cmov_reg);
13799 %}
13800 
13801 instruct maxI_rReg(rRegI dst, rRegI src)
13802 %{
13803   predicate(!UseAPX);
13804   match(Set dst (MaxI dst src));
13805 
13806   ins_cost(200);
13807   expand %{
13808     rFlagsReg cr;
13809     compI_rReg(cr, dst, src);
13810     cmovI_reg_l(dst, src, cr);
13811   %}
13812 %}
13813 
13814 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13815 %{
13816   predicate(UseAPX);
13817   match(Set dst (MaxI src1 src2));
13818   effect(DEF dst, USE src1, USE src2);
13819 
13820   ins_cost(200);
13821   expand %{
13822     rFlagsReg cr;
13823     compI_rReg(cr, src1, src2);
13824     cmovI_reg_l_ndd(dst, src1, src2, cr);
13825   %}
13826 %}
13827 
13828 // ============================================================================
13829 // Branch Instructions
13830 
13831 // Jump Direct - Label defines a relative address from JMP+1
13832 instruct jmpDir(label labl)
13833 %{
13834   match(Goto);
13835   effect(USE labl);
13836 
13837   ins_cost(300);
13838   format %{ "jmp     $labl" %}
13839   size(5);
13840   ins_encode %{
13841     Label* L = $labl$$label;
13842     __ jmp(*L, false); // Always long jump
13843   %}
13844   ins_pipe(pipe_jmp);
13845 %}
13846 
13847 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13848 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13849 %{
13850   match(If cop cr);
13851   effect(USE labl);
13852 
13853   ins_cost(300);
13854   format %{ "j$cop     $labl" %}
13855   size(6);
13856   ins_encode %{
13857     Label* L = $labl$$label;
13858     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13859   %}
13860   ins_pipe(pipe_jcc);
13861 %}
13862 
13863 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13864 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13865 %{
13866   match(CountedLoopEnd cop cr);
13867   effect(USE labl);
13868 
13869   ins_cost(300);
13870   format %{ "j$cop     $labl\t# loop end" %}
13871   size(6);
13872   ins_encode %{
13873     Label* L = $labl$$label;
13874     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13875   %}
13876   ins_pipe(pipe_jcc);
13877 %}
13878 
13879 // Jump Direct Conditional - using unsigned comparison
13880 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13881   match(If cop cmp);
13882   effect(USE labl);
13883 
13884   ins_cost(300);
13885   format %{ "j$cop,u   $labl" %}
13886   size(6);
13887   ins_encode %{
13888     Label* L = $labl$$label;
13889     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13890   %}
13891   ins_pipe(pipe_jcc);
13892 %}
13893 
13894 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13895   match(If cop cmp);
13896   effect(USE labl);
13897 
13898   ins_cost(200);
13899   format %{ "j$cop,u   $labl" %}
13900   size(6);
13901   ins_encode %{
13902     Label* L = $labl$$label;
13903     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13904   %}
13905   ins_pipe(pipe_jcc);
13906 %}
13907 
13908 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13909   match(If cop cmp);
13910   effect(USE labl);
13911 
13912   ins_cost(200);
13913   format %{ $$template
13914     if ($cop$$cmpcode == Assembler::notEqual) {
13915       $$emit$$"jp,u    $labl\n\t"
13916       $$emit$$"j$cop,u   $labl"
13917     } else {
13918       $$emit$$"jp,u    done\n\t"
13919       $$emit$$"j$cop,u   $labl\n\t"
13920       $$emit$$"done:"
13921     }
13922   %}
13923   ins_encode %{
13924     Label* l = $labl$$label;
13925     if ($cop$$cmpcode == Assembler::notEqual) {
13926       __ jcc(Assembler::parity, *l, false);
13927       __ jcc(Assembler::notEqual, *l, false);
13928     } else if ($cop$$cmpcode == Assembler::equal) {
13929       Label done;
13930       __ jccb(Assembler::parity, done);
13931       __ jcc(Assembler::equal, *l, false);
13932       __ bind(done);
13933     } else {
13934        ShouldNotReachHere();
13935     }
13936   %}
13937   ins_pipe(pipe_jcc);
13938 %}
13939 
13940 // ============================================================================
13941 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13942 // superklass array for an instance of the superklass.  Set a hidden
13943 // internal cache on a hit (cache is checked with exposed code in
13944 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13945 // encoding ALSO sets flags.
13946 
13947 instruct partialSubtypeCheck(rdi_RegP result,
13948                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13949                              rFlagsReg cr)
13950 %{
13951   match(Set result (PartialSubtypeCheck sub super));
13952   predicate(!UseSecondarySupersTable);
13953   effect(KILL rcx, KILL cr);
13954 
13955   ins_cost(1100);  // slightly larger than the next version
13956   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13957             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13958             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13959             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13960             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13961             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13962             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13963     "miss:\t" %}
13964 
13965   ins_encode %{
13966     Label miss;
13967     // NB: Callers may assume that, when $result is a valid register,
13968     // check_klass_subtype_slow_path_linear sets it to a nonzero
13969     // value.
13970     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13971                                             $rcx$$Register, $result$$Register,
13972                                             nullptr, &miss,
13973                                             /*set_cond_codes:*/ true);
13974     __ xorptr($result$$Register, $result$$Register);
13975     __ bind(miss);
13976   %}
13977 
13978   ins_pipe(pipe_slow);
13979 %}
13980 
13981 // ============================================================================
13982 // Two versions of hashtable-based partialSubtypeCheck, both used when
13983 // we need to search for a super class in the secondary supers array.
13984 // The first is used when we don't know _a priori_ the class being
13985 // searched for. The second, far more common, is used when we do know:
13986 // this is used for instanceof, checkcast, and any case where C2 can
13987 // determine it by constant propagation.
13988 
13989 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
13990                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13991                                        rFlagsReg cr)
13992 %{
13993   match(Set result (PartialSubtypeCheck sub super));
13994   predicate(UseSecondarySupersTable);
13995   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13996 
13997   ins_cost(1000);
13998   format %{ "partialSubtypeCheck $result, $sub, $super" %}
13999 
14000   ins_encode %{
14001     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
14002 					 $temp3$$Register, $temp4$$Register, $result$$Register);
14003   %}
14004 
14005   ins_pipe(pipe_slow);
14006 %}
14007 
14008 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
14009                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14010                                        rFlagsReg cr)
14011 %{
14012   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
14013   predicate(UseSecondarySupersTable);
14014   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14015 
14016   ins_cost(700);  // smaller than the next version
14017   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
14018 
14019   ins_encode %{
14020     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
14021     if (InlineSecondarySupersTest) {
14022       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
14023                                        $temp3$$Register, $temp4$$Register, $result$$Register,
14024                                        super_klass_slot);
14025     } else {
14026       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
14027     }
14028   %}
14029 
14030   ins_pipe(pipe_slow);
14031 %}
14032 
14033 // ============================================================================
14034 // Branch Instructions -- short offset versions
14035 //
14036 // These instructions are used to replace jumps of a long offset (the default
14037 // match) with jumps of a shorter offset.  These instructions are all tagged
14038 // with the ins_short_branch attribute, which causes the ADLC to suppress the
14039 // match rules in general matching.  Instead, the ADLC generates a conversion
14040 // method in the MachNode which can be used to do in-place replacement of the
14041 // long variant with the shorter variant.  The compiler will determine if a
14042 // branch can be taken by the is_short_branch_offset() predicate in the machine
14043 // specific code section of the file.
14044 
14045 // Jump Direct - Label defines a relative address from JMP+1
14046 instruct jmpDir_short(label labl) %{
14047   match(Goto);
14048   effect(USE labl);
14049 
14050   ins_cost(300);
14051   format %{ "jmp,s   $labl" %}
14052   size(2);
14053   ins_encode %{
14054     Label* L = $labl$$label;
14055     __ jmpb(*L);
14056   %}
14057   ins_pipe(pipe_jmp);
14058   ins_short_branch(1);
14059 %}
14060 
14061 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14062 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
14063   match(If cop cr);
14064   effect(USE labl);
14065 
14066   ins_cost(300);
14067   format %{ "j$cop,s   $labl" %}
14068   size(2);
14069   ins_encode %{
14070     Label* L = $labl$$label;
14071     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14072   %}
14073   ins_pipe(pipe_jcc);
14074   ins_short_branch(1);
14075 %}
14076 
14077 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14078 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
14079   match(CountedLoopEnd cop cr);
14080   effect(USE labl);
14081 
14082   ins_cost(300);
14083   format %{ "j$cop,s   $labl\t# loop end" %}
14084   size(2);
14085   ins_encode %{
14086     Label* L = $labl$$label;
14087     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14088   %}
14089   ins_pipe(pipe_jcc);
14090   ins_short_branch(1);
14091 %}
14092 
14093 // Jump Direct Conditional - using unsigned comparison
14094 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14095   match(If cop cmp);
14096   effect(USE labl);
14097 
14098   ins_cost(300);
14099   format %{ "j$cop,us  $labl" %}
14100   size(2);
14101   ins_encode %{
14102     Label* L = $labl$$label;
14103     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14104   %}
14105   ins_pipe(pipe_jcc);
14106   ins_short_branch(1);
14107 %}
14108 
14109 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14110   match(If cop cmp);
14111   effect(USE labl);
14112 
14113   ins_cost(300);
14114   format %{ "j$cop,us  $labl" %}
14115   size(2);
14116   ins_encode %{
14117     Label* L = $labl$$label;
14118     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14119   %}
14120   ins_pipe(pipe_jcc);
14121   ins_short_branch(1);
14122 %}
14123 
14124 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14125   match(If cop cmp);
14126   effect(USE labl);
14127 
14128   ins_cost(300);
14129   format %{ $$template
14130     if ($cop$$cmpcode == Assembler::notEqual) {
14131       $$emit$$"jp,u,s  $labl\n\t"
14132       $$emit$$"j$cop,u,s  $labl"
14133     } else {
14134       $$emit$$"jp,u,s  done\n\t"
14135       $$emit$$"j$cop,u,s  $labl\n\t"
14136       $$emit$$"done:"
14137     }
14138   %}
14139   size(4);
14140   ins_encode %{
14141     Label* l = $labl$$label;
14142     if ($cop$$cmpcode == Assembler::notEqual) {
14143       __ jccb(Assembler::parity, *l);
14144       __ jccb(Assembler::notEqual, *l);
14145     } else if ($cop$$cmpcode == Assembler::equal) {
14146       Label done;
14147       __ jccb(Assembler::parity, done);
14148       __ jccb(Assembler::equal, *l);
14149       __ bind(done);
14150     } else {
14151        ShouldNotReachHere();
14152     }
14153   %}
14154   ins_pipe(pipe_jcc);
14155   ins_short_branch(1);
14156 %}
14157 
14158 // ============================================================================
14159 // inlined locking and unlocking
14160 
14161 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
14162   predicate(LockingMode != LM_LIGHTWEIGHT);
14163   match(Set cr (FastLock object box));
14164   effect(TEMP tmp, TEMP scr, USE_KILL box);
14165   ins_cost(300);
14166   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
14167   ins_encode %{
14168     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
14169                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
14170   %}
14171   ins_pipe(pipe_slow);
14172 %}
14173 
14174 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
14175   predicate(LockingMode != LM_LIGHTWEIGHT);
14176   match(Set cr (FastUnlock object box));
14177   effect(TEMP tmp, USE_KILL box);
14178   ins_cost(300);
14179   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
14180   ins_encode %{
14181     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
14182   %}
14183   ins_pipe(pipe_slow);
14184 %}
14185 
14186 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14187   predicate(LockingMode == LM_LIGHTWEIGHT);
14188   match(Set cr (FastLock object box));
14189   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14190   ins_cost(300);
14191   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14192   ins_encode %{
14193     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14194   %}
14195   ins_pipe(pipe_slow);
14196 %}
14197 
14198 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14199   predicate(LockingMode == LM_LIGHTWEIGHT);
14200   match(Set cr (FastUnlock object rax_reg));
14201   effect(TEMP tmp, USE_KILL rax_reg);
14202   ins_cost(300);
14203   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14204   ins_encode %{
14205     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14206   %}
14207   ins_pipe(pipe_slow);
14208 %}
14209 
14210 
14211 // ============================================================================
14212 // Safepoint Instructions
14213 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14214 %{
14215   match(SafePoint poll);
14216   effect(KILL cr, USE poll);
14217 
14218   format %{ "testl   rax, [$poll]\t"
14219             "# Safepoint: poll for GC" %}
14220   ins_cost(125);
14221   ins_encode %{
14222     __ relocate(relocInfo::poll_type);
14223     address pre_pc = __ pc();
14224     __ testl(rax, Address($poll$$Register, 0));
14225     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14226   %}
14227   ins_pipe(ialu_reg_mem);
14228 %}
14229 
14230 instruct mask_all_evexL(kReg dst, rRegL src) %{
14231   match(Set dst (MaskAll src));
14232   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14233   ins_encode %{
14234     int mask_len = Matcher::vector_length(this);
14235     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14236   %}
14237   ins_pipe( pipe_slow );
14238 %}
14239 
14240 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14241   predicate(Matcher::vector_length(n) > 32);
14242   match(Set dst (MaskAll src));
14243   effect(TEMP tmp);
14244   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14245   ins_encode %{
14246     int mask_len = Matcher::vector_length(this);
14247     __ movslq($tmp$$Register, $src$$Register);
14248     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14249   %}
14250   ins_pipe( pipe_slow );
14251 %}
14252 
14253 // ============================================================================
14254 // Procedure Call/Return Instructions
14255 // Call Java Static Instruction
14256 // Note: If this code changes, the corresponding ret_addr_offset() and
14257 //       compute_padding() functions will have to be adjusted.
14258 instruct CallStaticJavaDirect(method meth) %{
14259   match(CallStaticJava);
14260   effect(USE meth);
14261 
14262   ins_cost(300);
14263   format %{ "call,static " %}
14264   opcode(0xE8); /* E8 cd */
14265   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14266   ins_pipe(pipe_slow);
14267   ins_alignment(4);
14268 %}
14269 
14270 // Call Java Dynamic Instruction
14271 // Note: If this code changes, the corresponding ret_addr_offset() and
14272 //       compute_padding() functions will have to be adjusted.
14273 instruct CallDynamicJavaDirect(method meth)
14274 %{
14275   match(CallDynamicJava);
14276   effect(USE meth);
14277 
14278   ins_cost(300);
14279   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14280             "call,dynamic " %}
14281   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14282   ins_pipe(pipe_slow);
14283   ins_alignment(4);
14284 %}
14285 
14286 // Call Runtime Instruction
14287 instruct CallRuntimeDirect(method meth)
14288 %{
14289   match(CallRuntime);
14290   effect(USE meth);
14291 
14292   ins_cost(300);
14293   format %{ "call,runtime " %}
14294   ins_encode(clear_avx, Java_To_Runtime(meth));
14295   ins_pipe(pipe_slow);
14296 %}
14297 
14298 // Call runtime without safepoint
14299 instruct CallLeafDirect(method meth)
14300 %{
14301   match(CallLeaf);
14302   effect(USE meth);
14303 
14304   ins_cost(300);
14305   format %{ "call_leaf,runtime " %}
14306   ins_encode(clear_avx, Java_To_Runtime(meth));
14307   ins_pipe(pipe_slow);
14308 %}
14309 
14310 // Call runtime without safepoint and with vector arguments
14311 instruct CallLeafDirectVector(method meth)
14312 %{
14313   match(CallLeafVector);
14314   effect(USE meth);
14315 
14316   ins_cost(300);
14317   format %{ "call_leaf,vector " %}
14318   ins_encode(Java_To_Runtime(meth));
14319   ins_pipe(pipe_slow);
14320 %}
14321 
14322 // Call runtime without safepoint
14323 instruct CallLeafNoFPDirect(method meth)
14324 %{
14325   match(CallLeafNoFP);
14326   effect(USE meth);
14327 
14328   ins_cost(300);
14329   format %{ "call_leaf_nofp,runtime " %}
14330   ins_encode(clear_avx, Java_To_Runtime(meth));
14331   ins_pipe(pipe_slow);
14332 %}
14333 
14334 // Return Instruction
14335 // Remove the return address & jump to it.
14336 // Notice: We always emit a nop after a ret to make sure there is room
14337 // for safepoint patching
14338 instruct Ret()
14339 %{
14340   match(Return);
14341 
14342   format %{ "ret" %}
14343   ins_encode %{
14344     __ ret(0);
14345   %}
14346   ins_pipe(pipe_jmp);
14347 %}
14348 
14349 // Tail Call; Jump from runtime stub to Java code.
14350 // Also known as an 'interprocedural jump'.
14351 // Target of jump will eventually return to caller.
14352 // TailJump below removes the return address.
14353 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14354 // emitted just above the TailCall which has reset rbp to the caller state.
14355 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14356 %{
14357   match(TailCall jump_target method_ptr);
14358 
14359   ins_cost(300);
14360   format %{ "jmp     $jump_target\t# rbx holds method" %}
14361   ins_encode %{
14362     __ jmp($jump_target$$Register);
14363   %}
14364   ins_pipe(pipe_jmp);
14365 %}
14366 
14367 // Tail Jump; remove the return address; jump to target.
14368 // TailCall above leaves the return address around.
14369 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14370 %{
14371   match(TailJump jump_target ex_oop);
14372 
14373   ins_cost(300);
14374   format %{ "popq    rdx\t# pop return address\n\t"
14375             "jmp     $jump_target" %}
14376   ins_encode %{
14377     __ popq(as_Register(RDX_enc));
14378     __ jmp($jump_target$$Register);
14379   %}
14380   ins_pipe(pipe_jmp);
14381 %}
14382 
14383 // Forward exception.
14384 instruct ForwardExceptionjmp()
14385 %{
14386   match(ForwardException);
14387 
14388   format %{ "jmp     forward_exception_stub" %}
14389   ins_encode %{
14390     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14391   %}
14392   ins_pipe(pipe_jmp);
14393 %}
14394 
14395 // Create exception oop: created by stack-crawling runtime code.
14396 // Created exception is now available to this handler, and is setup
14397 // just prior to jumping to this handler.  No code emitted.
14398 instruct CreateException(rax_RegP ex_oop)
14399 %{
14400   match(Set ex_oop (CreateEx));
14401 
14402   size(0);
14403   // use the following format syntax
14404   format %{ "# exception oop is in rax; no code emitted" %}
14405   ins_encode();
14406   ins_pipe(empty);
14407 %}
14408 
14409 // Rethrow exception:
14410 // The exception oop will come in the first argument position.
14411 // Then JUMP (not call) to the rethrow stub code.
14412 instruct RethrowException()
14413 %{
14414   match(Rethrow);
14415 
14416   // use the following format syntax
14417   format %{ "jmp     rethrow_stub" %}
14418   ins_encode %{
14419     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14420   %}
14421   ins_pipe(pipe_jmp);
14422 %}
14423 
14424 // ============================================================================
14425 // This name is KNOWN by the ADLC and cannot be changed.
14426 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14427 // for this guy.
14428 instruct tlsLoadP(r15_RegP dst) %{
14429   match(Set dst (ThreadLocal));
14430   effect(DEF dst);
14431 
14432   size(0);
14433   format %{ "# TLS is in R15" %}
14434   ins_encode( /*empty encoding*/ );
14435   ins_pipe(ialu_reg_reg);
14436 %}
14437 
14438 
14439 //----------PEEPHOLE RULES-----------------------------------------------------
14440 // These must follow all instruction definitions as they use the names
14441 // defined in the instructions definitions.
14442 //
14443 // peeppredicate ( rule_predicate );
14444 // // the predicate unless which the peephole rule will be ignored
14445 //
14446 // peepmatch ( root_instr_name [preceding_instruction]* );
14447 //
14448 // peepprocedure ( procedure_name );
14449 // // provide a procedure name to perform the optimization, the procedure should
14450 // // reside in the architecture dependent peephole file, the method has the
14451 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14452 // // with the arguments being the basic block, the current node index inside the
14453 // // block, the register allocator, the functions upon invoked return a new node
14454 // // defined in peepreplace, and the rules of the nodes appearing in the
14455 // // corresponding peepmatch, the function return true if successful, else
14456 // // return false
14457 //
14458 // peepconstraint %{
14459 // (instruction_number.operand_name relational_op instruction_number.operand_name
14460 //  [, ...] );
14461 // // instruction numbers are zero-based using left to right order in peepmatch
14462 //
14463 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14464 // // provide an instruction_number.operand_name for each operand that appears
14465 // // in the replacement instruction's match rule
14466 //
14467 // ---------VM FLAGS---------------------------------------------------------
14468 //
14469 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14470 //
14471 // Each peephole rule is given an identifying number starting with zero and
14472 // increasing by one in the order seen by the parser.  An individual peephole
14473 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14474 // on the command-line.
14475 //
14476 // ---------CURRENT LIMITATIONS----------------------------------------------
14477 //
14478 // Only transformations inside a basic block (do we need more for peephole)
14479 //
14480 // ---------EXAMPLE----------------------------------------------------------
14481 //
14482 // // pertinent parts of existing instructions in architecture description
14483 // instruct movI(rRegI dst, rRegI src)
14484 // %{
14485 //   match(Set dst (CopyI src));
14486 // %}
14487 //
14488 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14489 // %{
14490 //   match(Set dst (AddI dst src));
14491 //   effect(KILL cr);
14492 // %}
14493 //
14494 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14495 // %{
14496 //   match(Set dst (AddI dst src));
14497 // %}
14498 //
14499 // 1. Simple replacement
14500 // - Only match adjacent instructions in same basic block
14501 // - Only equality constraints
14502 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14503 // - Only one replacement instruction
14504 //
14505 // // Change (inc mov) to lea
14506 // peephole %{
14507 //   // lea should only be emitted when beneficial
14508 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14509 //   // increment preceded by register-register move
14510 //   peepmatch ( incI_rReg movI );
14511 //   // require that the destination register of the increment
14512 //   // match the destination register of the move
14513 //   peepconstraint ( 0.dst == 1.dst );
14514 //   // construct a replacement instruction that sets
14515 //   // the destination to ( move's source register + one )
14516 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14517 // %}
14518 //
14519 // 2. Procedural replacement
14520 // - More flexible finding relevent nodes
14521 // - More flexible constraints
14522 // - More flexible transformations
14523 // - May utilise architecture-dependent API more effectively
14524 // - Currently only one replacement instruction due to adlc parsing capabilities
14525 //
14526 // // Change (inc mov) to lea
14527 // peephole %{
14528 //   // lea should only be emitted when beneficial
14529 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14530 //   // the rule numbers of these nodes inside are passed into the function below
14531 //   peepmatch ( incI_rReg movI );
14532 //   // the method that takes the responsibility of transformation
14533 //   peepprocedure ( inc_mov_to_lea );
14534 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14535 //   // node is passed into the function above
14536 //   peepreplace ( leaI_rReg_immI() );
14537 // %}
14538 
14539 // These instructions is not matched by the matcher but used by the peephole
14540 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14541 %{
14542   predicate(false);
14543   match(Set dst (AddI src1 src2));
14544   format %{ "leal    $dst, [$src1 + $src2]" %}
14545   ins_encode %{
14546     Register dst = $dst$$Register;
14547     Register src1 = $src1$$Register;
14548     Register src2 = $src2$$Register;
14549     if (src1 != rbp && src1 != r13) {
14550       __ leal(dst, Address(src1, src2, Address::times_1));
14551     } else {
14552       assert(src2 != rbp && src2 != r13, "");
14553       __ leal(dst, Address(src2, src1, Address::times_1));
14554     }
14555   %}
14556   ins_pipe(ialu_reg_reg);
14557 %}
14558 
14559 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14560 %{
14561   predicate(false);
14562   match(Set dst (AddI src1 src2));
14563   format %{ "leal    $dst, [$src1 + $src2]" %}
14564   ins_encode %{
14565     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14566   %}
14567   ins_pipe(ialu_reg_reg);
14568 %}
14569 
14570 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14571 %{
14572   predicate(false);
14573   match(Set dst (LShiftI src shift));
14574   format %{ "leal    $dst, [$src << $shift]" %}
14575   ins_encode %{
14576     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14577     Register src = $src$$Register;
14578     if (scale == Address::times_2 && src != rbp && src != r13) {
14579       __ leal($dst$$Register, Address(src, src, Address::times_1));
14580     } else {
14581       __ leal($dst$$Register, Address(noreg, src, scale));
14582     }
14583   %}
14584   ins_pipe(ialu_reg_reg);
14585 %}
14586 
14587 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14588 %{
14589   predicate(false);
14590   match(Set dst (AddL src1 src2));
14591   format %{ "leaq    $dst, [$src1 + $src2]" %}
14592   ins_encode %{
14593     Register dst = $dst$$Register;
14594     Register src1 = $src1$$Register;
14595     Register src2 = $src2$$Register;
14596     if (src1 != rbp && src1 != r13) {
14597       __ leaq(dst, Address(src1, src2, Address::times_1));
14598     } else {
14599       assert(src2 != rbp && src2 != r13, "");
14600       __ leaq(dst, Address(src2, src1, Address::times_1));
14601     }
14602   %}
14603   ins_pipe(ialu_reg_reg);
14604 %}
14605 
14606 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14607 %{
14608   predicate(false);
14609   match(Set dst (AddL src1 src2));
14610   format %{ "leaq    $dst, [$src1 + $src2]" %}
14611   ins_encode %{
14612     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14613   %}
14614   ins_pipe(ialu_reg_reg);
14615 %}
14616 
14617 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14618 %{
14619   predicate(false);
14620   match(Set dst (LShiftL src shift));
14621   format %{ "leaq    $dst, [$src << $shift]" %}
14622   ins_encode %{
14623     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14624     Register src = $src$$Register;
14625     if (scale == Address::times_2 && src != rbp && src != r13) {
14626       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14627     } else {
14628       __ leaq($dst$$Register, Address(noreg, src, scale));
14629     }
14630   %}
14631   ins_pipe(ialu_reg_reg);
14632 %}
14633 
14634 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14635 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14636 // processors with at least partial ALU support for lea
14637 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14638 // beneficial for processors with full ALU support
14639 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14640 
14641 peephole
14642 %{
14643   peeppredicate(VM_Version::supports_fast_2op_lea());
14644   peepmatch (addI_rReg);
14645   peepprocedure (lea_coalesce_reg);
14646   peepreplace (leaI_rReg_rReg_peep());
14647 %}
14648 
14649 peephole
14650 %{
14651   peeppredicate(VM_Version::supports_fast_2op_lea());
14652   peepmatch (addI_rReg_imm);
14653   peepprocedure (lea_coalesce_imm);
14654   peepreplace (leaI_rReg_immI_peep());
14655 %}
14656 
14657 peephole
14658 %{
14659   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14660                 VM_Version::is_intel_cascade_lake());
14661   peepmatch (incI_rReg);
14662   peepprocedure (lea_coalesce_imm);
14663   peepreplace (leaI_rReg_immI_peep());
14664 %}
14665 
14666 peephole
14667 %{
14668   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14669                 VM_Version::is_intel_cascade_lake());
14670   peepmatch (decI_rReg);
14671   peepprocedure (lea_coalesce_imm);
14672   peepreplace (leaI_rReg_immI_peep());
14673 %}
14674 
14675 peephole
14676 %{
14677   peeppredicate(VM_Version::supports_fast_2op_lea());
14678   peepmatch (salI_rReg_immI2);
14679   peepprocedure (lea_coalesce_imm);
14680   peepreplace (leaI_rReg_immI2_peep());
14681 %}
14682 
14683 peephole
14684 %{
14685   peeppredicate(VM_Version::supports_fast_2op_lea());
14686   peepmatch (addL_rReg);
14687   peepprocedure (lea_coalesce_reg);
14688   peepreplace (leaL_rReg_rReg_peep());
14689 %}
14690 
14691 peephole
14692 %{
14693   peeppredicate(VM_Version::supports_fast_2op_lea());
14694   peepmatch (addL_rReg_imm);
14695   peepprocedure (lea_coalesce_imm);
14696   peepreplace (leaL_rReg_immL32_peep());
14697 %}
14698 
14699 peephole
14700 %{
14701   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14702                 VM_Version::is_intel_cascade_lake());
14703   peepmatch (incL_rReg);
14704   peepprocedure (lea_coalesce_imm);
14705   peepreplace (leaL_rReg_immL32_peep());
14706 %}
14707 
14708 peephole
14709 %{
14710   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14711                 VM_Version::is_intel_cascade_lake());
14712   peepmatch (decL_rReg);
14713   peepprocedure (lea_coalesce_imm);
14714   peepreplace (leaL_rReg_immL32_peep());
14715 %}
14716 
14717 peephole
14718 %{
14719   peeppredicate(VM_Version::supports_fast_2op_lea());
14720   peepmatch (salL_rReg_immI2);
14721   peepprocedure (lea_coalesce_imm);
14722   peepreplace (leaL_rReg_immI2_peep());
14723 %}
14724 
14725 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14726 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14727 
14728 //int variant
14729 peephole
14730 %{
14731   peepmatch (testI_reg);
14732   peepprocedure (test_may_remove);
14733 %}
14734 
14735 //long variant
14736 peephole
14737 %{
14738   peepmatch (testL_reg);
14739   peepprocedure (test_may_remove);
14740 %}
14741 
14742 
14743 //----------SMARTSPILL RULES---------------------------------------------------
14744 // These must follow all instruction definitions as they use the names
14745 // defined in the instructions definitions.