1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 
  442 extern RegMask _ANY_REG_mask;
  443 extern RegMask _PTR_REG_mask;
  444 extern RegMask _PTR_REG_NO_RBP_mask;
  445 extern RegMask _PTR_NO_RAX_REG_mask;
  446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  447 extern RegMask _LONG_REG_mask;
  448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  449 extern RegMask _LONG_NO_RCX_REG_mask;
  450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  451 extern RegMask _INT_REG_mask;
  452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  453 extern RegMask _INT_NO_RCX_REG_mask;
  454 extern RegMask _INT_NO_RBP_R13_REG_mask;
  455 extern RegMask _FLOAT_REG_mask;
  456 
  457 extern RegMask _STACK_OR_PTR_REG_mask;
  458 extern RegMask _STACK_OR_LONG_REG_mask;
  459 extern RegMask _STACK_OR_INT_REG_mask;
  460 
  461 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  462 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  463 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  464 
  465 %}
  466 
  467 source %{
  468 #define   RELOC_IMM64    Assembler::imm_operand
  469 #define   RELOC_DISP32   Assembler::disp32_operand
  470 
  471 #define __ masm->
  472 
  473 RegMask _ANY_REG_mask;
  474 RegMask _PTR_REG_mask;
  475 RegMask _PTR_REG_NO_RBP_mask;
  476 RegMask _PTR_NO_RAX_REG_mask;
  477 RegMask _PTR_NO_RAX_RBX_REG_mask;
  478 RegMask _LONG_REG_mask;
  479 RegMask _LONG_NO_RAX_RDX_REG_mask;
  480 RegMask _LONG_NO_RCX_REG_mask;
  481 RegMask _LONG_NO_RBP_R13_REG_mask;
  482 RegMask _INT_REG_mask;
  483 RegMask _INT_NO_RAX_RDX_REG_mask;
  484 RegMask _INT_NO_RCX_REG_mask;
  485 RegMask _INT_NO_RBP_R13_REG_mask;
  486 RegMask _FLOAT_REG_mask;
  487 RegMask _STACK_OR_PTR_REG_mask;
  488 RegMask _STACK_OR_LONG_REG_mask;
  489 RegMask _STACK_OR_INT_REG_mask;
  490 
  491 static bool need_r12_heapbase() {
  492   return UseCompressedOops;
  493 }
  494 
  495 void reg_mask_init() {
  496   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  497 
  498   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  499   // We derive a number of subsets from it.
  500   _ANY_REG_mask = _ALL_REG_mask;
  501 
  502   if (PreserveFramePointer) {
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  504     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  505   }
  506   if (need_r12_heapbase()) {
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  508     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  509   }
  510 
  511   _PTR_REG_mask = _ANY_REG_mask;
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  515   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  516   if (!UseAPX) {
  517     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  519       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  520     }
  521   }
  522 
  523   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  524   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  525 
  526   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  528   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  529 
  530   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  532   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  533 
  534   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  536   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  537 
  538 
  539   _LONG_REG_mask = _PTR_REG_mask;
  540   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  541   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  542 
  543   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  547   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  548 
  549   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  551   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  552 
  553   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  557   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  558 
  559   _INT_REG_mask = _ALL_INT_REG_mask;
  560   if (!UseAPX) {
  561     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  562       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  563     }
  564   }
  565 
  566   if (PreserveFramePointer) {
  567     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  568   }
  569   if (need_r12_heapbase()) {
  570     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  571   }
  572 
  573   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  574   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  575 
  576   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  578   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  579 
  580   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  581   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  582 
  583   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  585   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  586 
  587   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  588   // from the float_reg_legacy/float_reg_evex register class.
  589   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  590 }
  591 
  592 static bool generate_vzeroupper(Compile* C) {
  593   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  594 }
  595 
  596 static int clear_avx_size() {
  597   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  598 }
  599 
  600 // !!!!! Special hack to get all types of calls to specify the byte offset
  601 //       from the start of the call to the point where the return address
  602 //       will point.
  603 int MachCallStaticJavaNode::ret_addr_offset()
  604 {
  605   int offset = 5; // 5 bytes from start of call to where return address points
  606   offset += clear_avx_size();
  607   return offset;
  608 }
  609 
  610 int MachCallDynamicJavaNode::ret_addr_offset()
  611 {
  612   int offset = 15; // 15 bytes from start of call to where return address points
  613   offset += clear_avx_size();
  614   return offset;
  615 }
  616 
  617 int MachCallRuntimeNode::ret_addr_offset() {
  618   int offset = 13; // movq r10,#addr; callq (r10)
  619   if (this->ideal_Opcode() != Op_CallLeafVector) {
  620     offset += clear_avx_size();
  621   }
  622   return offset;
  623 }
  624 //
  625 // Compute padding required for nodes which need alignment
  626 //
  627 
  628 // The address of the call instruction needs to be 4-byte aligned to
  629 // ensure that it does not span a cache line so that it can be patched.
  630 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  631 {
  632   current_offset += clear_avx_size(); // skip vzeroupper
  633   current_offset += 1; // skip call opcode byte
  634   return align_up(current_offset, alignment_required()) - current_offset;
  635 }
  636 
  637 // The address of the call instruction needs to be 4-byte aligned to
  638 // ensure that it does not span a cache line so that it can be patched.
  639 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  640 {
  641   current_offset += clear_avx_size(); // skip vzeroupper
  642   current_offset += 11; // skip movq instruction + call opcode byte
  643   return align_up(current_offset, alignment_required()) - current_offset;
  644 }
  645 
  646 // This could be in MacroAssembler but it's fairly C2 specific
  647 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  648   Label exit;
  649   __ jccb(Assembler::noParity, exit);
  650   __ pushf();
  651   //
  652   // comiss/ucomiss instructions set ZF,PF,CF flags and
  653   // zero OF,AF,SF for NaN values.
  654   // Fixup flags by zeroing ZF,PF so that compare of NaN
  655   // values returns 'less than' result (CF is set).
  656   // Leave the rest of flags unchanged.
  657   //
  658   //    7 6 5 4 3 2 1 0
  659   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  660   //    0 0 1 0 1 0 1 1   (0x2B)
  661   //
  662   __ andq(Address(rsp, 0), 0xffffff2b);
  663   __ popf();
  664   __ bind(exit);
  665 }
  666 
  667 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  668   Label done;
  669   __ movl(dst, -1);
  670   __ jcc(Assembler::parity, done);
  671   __ jcc(Assembler::below, done);
  672   __ setcc(Assembler::notEqual, dst);
  673   __ bind(done);
  674 }
  675 
  676 // Math.min()    # Math.max()
  677 // --------------------------
  678 // ucomis[s/d]   #
  679 // ja   -> b     # a
  680 // jp   -> NaN   # NaN
  681 // jb   -> a     # b
  682 // je            #
  683 // |-jz -> a | b # a & b
  684 // |    -> a     #
  685 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  686                             XMMRegister a, XMMRegister b,
  687                             XMMRegister xmmt, Register rt,
  688                             bool min, bool single) {
  689 
  690   Label nan, zero, below, above, done;
  691 
  692   if (single)
  693     __ ucomiss(a, b);
  694   else
  695     __ ucomisd(a, b);
  696 
  697   if (dst->encoding() != (min ? b : a)->encoding())
  698     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  699   else
  700     __ jccb(Assembler::above, done);
  701 
  702   __ jccb(Assembler::parity, nan);  // PF=1
  703   __ jccb(Assembler::below, below); // CF=1
  704 
  705   // equal
  706   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  707   if (single) {
  708     __ ucomiss(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movflt(dst, a);
  712     __ jmp(done);
  713   }
  714   else {
  715     __ ucomisd(a, xmmt);
  716     __ jccb(Assembler::equal, zero);
  717 
  718     __ movdbl(dst, a);
  719     __ jmp(done);
  720   }
  721 
  722   __ bind(zero);
  723   if (min)
  724     __ vpor(dst, a, b, Assembler::AVX_128bit);
  725   else
  726     __ vpand(dst, a, b, Assembler::AVX_128bit);
  727 
  728   __ jmp(done);
  729 
  730   __ bind(above);
  731   if (single)
  732     __ movflt(dst, min ? b : a);
  733   else
  734     __ movdbl(dst, min ? b : a);
  735 
  736   __ jmp(done);
  737 
  738   __ bind(nan);
  739   if (single) {
  740     __ movl(rt, 0x7fc00000); // Float.NaN
  741     __ movdl(dst, rt);
  742   }
  743   else {
  744     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  745     __ movdq(dst, rt);
  746   }
  747   __ jmp(done);
  748 
  749   __ bind(below);
  750   if (single)
  751     __ movflt(dst, min ? a : b);
  752   else
  753     __ movdbl(dst, min ? a : b);
  754 
  755   __ bind(done);
  756 }
  757 
  758 //=============================================================================
  759 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  760 
  761 int ConstantTable::calculate_table_base_offset() const {
  762   return 0;  // absolute addressing, no offset
  763 }
  764 
  765 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  766 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  767   ShouldNotReachHere();
  768 }
  769 
  770 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  771   // Empty encoding
  772 }
  773 
  774 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  775   return 0;
  776 }
  777 
  778 #ifndef PRODUCT
  779 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  780   st->print("# MachConstantBaseNode (empty encoding)");
  781 }
  782 #endif
  783 
  784 
  785 //=============================================================================
  786 #ifndef PRODUCT
  787 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  788   Compile* C = ra_->C;
  789 
  790   int framesize = C->output()->frame_size_in_bytes();
  791   int bangsize = C->output()->bang_size_in_bytes();
  792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  793   // Remove wordSize for return addr which is already pushed.
  794   framesize -= wordSize;
  795 
  796   if (C->output()->need_stack_bang(bangsize)) {
  797     framesize -= wordSize;
  798     st->print("# stack bang (%d bytes)", bangsize);
  799     st->print("\n\t");
  800     st->print("pushq   rbp\t# Save rbp");
  801     if (PreserveFramePointer) {
  802         st->print("\n\t");
  803         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  804     }
  805     if (framesize) {
  806       st->print("\n\t");
  807       st->print("subq    rsp, #%d\t# Create frame",framesize);
  808     }
  809   } else {
  810     st->print("subq    rsp, #%d\t# Create frame",framesize);
  811     st->print("\n\t");
  812     framesize -= wordSize;
  813     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  814     if (PreserveFramePointer) {
  815       st->print("\n\t");
  816       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  817       if (framesize > 0) {
  818         st->print("\n\t");
  819         st->print("addq    rbp, #%d", framesize);
  820       }
  821     }
  822   }
  823 
  824   if (VerifyStackAtCalls) {
  825     st->print("\n\t");
  826     framesize -= wordSize;
  827     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  828 #ifdef ASSERT
  829     st->print("\n\t");
  830     st->print("# stack alignment check");
  831 #endif
  832   }
  833   if (C->stub_function() != nullptr) {
  834     st->print("\n\t");
  835     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  836     st->print("\n\t");
  837     st->print("je      fast_entry\t");
  838     st->print("\n\t");
  839     st->print("call    #nmethod_entry_barrier_stub\t");
  840     st->print("\n\tfast_entry:");
  841   }
  842   st->cr();
  843 }
  844 #endif
  845 
  846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  847   Compile* C = ra_->C;
  848 
  849   int framesize = C->output()->frame_size_in_bytes();
  850   int bangsize = C->output()->bang_size_in_bytes();
  851 
  852   if (C->clinit_barrier_on_entry()) {
  853     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  854     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  855 
  856     Label L_skip_barrier;
  857     Register klass = rscratch1;
  858 
  859     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  860     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  861 
  862     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  863 
  864     __ bind(L_skip_barrier);
  865   }
  866 
  867   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  868 
  869   C->output()->set_frame_complete(__ offset());
  870 
  871   if (C->has_mach_constant_base_node()) {
  872     // NOTE: We set the table base offset here because users might be
  873     // emitted before MachConstantBaseNode.
  874     ConstantTable& constant_table = C->output()->constant_table();
  875     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  876   }
  877 }
  878 
  879 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  880 {
  881   return MachNode::size(ra_); // too many variables; just compute it
  882                               // the hard way
  883 }
  884 
  885 int MachPrologNode::reloc() const
  886 {
  887   return 0; // a large enough number
  888 }
  889 
  890 //=============================================================================
  891 #ifndef PRODUCT
  892 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  893 {
  894   Compile* C = ra_->C;
  895   if (generate_vzeroupper(C)) {
  896     st->print("vzeroupper");
  897     st->cr(); st->print("\t");
  898   }
  899 
  900   int framesize = C->output()->frame_size_in_bytes();
  901   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  902   // Remove word for return adr already pushed
  903   // and RBP
  904   framesize -= 2*wordSize;
  905 
  906   if (framesize) {
  907     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  908     st->print("\t");
  909   }
  910 
  911   st->print_cr("popq    rbp");
  912   if (do_polling() && C->is_method_compilation()) {
  913     st->print("\t");
  914     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  915                  "ja      #safepoint_stub\t"
  916                  "# Safepoint: poll for GC");
  917   }
  918 }
  919 #endif
  920 
  921 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  922 {
  923   Compile* C = ra_->C;
  924 
  925   if (generate_vzeroupper(C)) {
  926     // Clear upper bits of YMM registers when current compiled code uses
  927     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  928     __ vzeroupper();
  929   }
  930 
  931   int framesize = C->output()->frame_size_in_bytes();
  932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  933   // Remove word for return adr already pushed
  934   // and RBP
  935   framesize -= 2*wordSize;
  936 
  937   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  938 
  939   if (framesize) {
  940     __ addq(rsp, framesize);
  941   }
  942 
  943   __ popq(rbp);
  944 
  945   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  946     __ reserved_stack_check();
  947   }
  948 
  949   if (do_polling() && C->is_method_compilation()) {
  950     Label dummy_label;
  951     Label* code_stub = &dummy_label;
  952     if (!C->output()->in_scratch_emit_size()) {
  953       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  954       C->output()->add_stub(stub);
  955       code_stub = &stub->entry();
  956     }
  957     __ relocate(relocInfo::poll_return_type);
  958     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  959   }
  960 }
  961 
  962 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  963 {
  964   return MachNode::size(ra_); // too many variables; just compute it
  965                               // the hard way
  966 }
  967 
  968 int MachEpilogNode::reloc() const
  969 {
  970   return 2; // a large enough number
  971 }
  972 
  973 const Pipeline* MachEpilogNode::pipeline() const
  974 {
  975   return MachNode::pipeline_class();
  976 }
  977 
  978 //=============================================================================
  979 
  980 enum RC {
  981   rc_bad,
  982   rc_int,
  983   rc_kreg,
  984   rc_float,
  985   rc_stack
  986 };
  987 
  988 static enum RC rc_class(OptoReg::Name reg)
  989 {
  990   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  991 
  992   if (OptoReg::is_stack(reg)) return rc_stack;
  993 
  994   VMReg r = OptoReg::as_VMReg(reg);
  995 
  996   if (r->is_Register()) return rc_int;
  997 
  998   if (r->is_KRegister()) return rc_kreg;
  999 
 1000   assert(r->is_XMMRegister(), "must be");
 1001   return rc_float;
 1002 }
 1003 
 1004 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1005 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1006                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1007 
 1008 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1009                      int stack_offset, int reg, uint ireg, outputStream* st);
 1010 
 1011 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1012                                       int dst_offset, uint ireg, outputStream* st) {
 1013   if (masm) {
 1014     switch (ireg) {
 1015     case Op_VecS:
 1016       __ movq(Address(rsp, -8), rax);
 1017       __ movl(rax, Address(rsp, src_offset));
 1018       __ movl(Address(rsp, dst_offset), rax);
 1019       __ movq(rax, Address(rsp, -8));
 1020       break;
 1021     case Op_VecD:
 1022       __ pushq(Address(rsp, src_offset));
 1023       __ popq (Address(rsp, dst_offset));
 1024       break;
 1025     case Op_VecX:
 1026       __ pushq(Address(rsp, src_offset));
 1027       __ popq (Address(rsp, dst_offset));
 1028       __ pushq(Address(rsp, src_offset+8));
 1029       __ popq (Address(rsp, dst_offset+8));
 1030       break;
 1031     case Op_VecY:
 1032       __ vmovdqu(Address(rsp, -32), xmm0);
 1033       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1034       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1035       __ vmovdqu(xmm0, Address(rsp, -32));
 1036       break;
 1037     case Op_VecZ:
 1038       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1039       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1040       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1041       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #ifndef PRODUCT
 1047   } else {
 1048     switch (ireg) {
 1049     case Op_VecS:
 1050       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                 "movl    rax, [rsp + #%d]\n\t"
 1052                 "movl    [rsp + #%d], rax\n\t"
 1053                 "movq    rax, [rsp - #8]",
 1054                 src_offset, dst_offset);
 1055       break;
 1056     case Op_VecD:
 1057       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1058                 "popq    [rsp + #%d]",
 1059                 src_offset, dst_offset);
 1060       break;
 1061      case Op_VecX:
 1062       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1063                 "popq    [rsp + #%d]\n\t"
 1064                 "pushq   [rsp + #%d]\n\t"
 1065                 "popq    [rsp + #%d]",
 1066                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1067       break;
 1068     case Op_VecY:
 1069       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1070                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1071                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1072                 "vmovdqu xmm0, [rsp - #32]",
 1073                 src_offset, dst_offset);
 1074       break;
 1075     case Op_VecZ:
 1076       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1077                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1078                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1079                 "vmovdqu xmm0, [rsp - #64]",
 1080                 src_offset, dst_offset);
 1081       break;
 1082     default:
 1083       ShouldNotReachHere();
 1084     }
 1085 #endif
 1086   }
 1087 }
 1088 
 1089 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1090                                        PhaseRegAlloc* ra_,
 1091                                        bool do_size,
 1092                                        outputStream* st) const {
 1093   assert(masm != nullptr || st  != nullptr, "sanity");
 1094   // Get registers to move
 1095   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1096   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1097   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1098   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1099 
 1100   enum RC src_second_rc = rc_class(src_second);
 1101   enum RC src_first_rc = rc_class(src_first);
 1102   enum RC dst_second_rc = rc_class(dst_second);
 1103   enum RC dst_first_rc = rc_class(dst_first);
 1104 
 1105   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1106          "must move at least 1 register" );
 1107 
 1108   if (src_first == dst_first && src_second == dst_second) {
 1109     // Self copy, no move
 1110     return 0;
 1111   }
 1112   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1113     uint ireg = ideal_reg();
 1114     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1115     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1116     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1117       // mem -> mem
 1118       int src_offset = ra_->reg2offset(src_first);
 1119       int dst_offset = ra_->reg2offset(dst_first);
 1120       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1121     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1122       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1123     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1124       int stack_offset = ra_->reg2offset(dst_first);
 1125       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1126     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1127       int stack_offset = ra_->reg2offset(src_first);
 1128       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1129     } else {
 1130       ShouldNotReachHere();
 1131     }
 1132     return 0;
 1133   }
 1134   if (src_first_rc == rc_stack) {
 1135     // mem ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // mem -> mem
 1138       assert(src_second != dst_first, "overlap");
 1139       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1140           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1141         // 64-bit
 1142         int src_offset = ra_->reg2offset(src_first);
 1143         int dst_offset = ra_->reg2offset(dst_first);
 1144         if (masm) {
 1145           __ pushq(Address(rsp, src_offset));
 1146           __ popq (Address(rsp, dst_offset));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1150                     "popq    [rsp + #%d]",
 1151                      src_offset, dst_offset);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         // No pushl/popl, so:
 1159         int src_offset = ra_->reg2offset(src_first);
 1160         int dst_offset = ra_->reg2offset(dst_first);
 1161         if (masm) {
 1162           __ movq(Address(rsp, -8), rax);
 1163           __ movl(rax, Address(rsp, src_offset));
 1164           __ movl(Address(rsp, dst_offset), rax);
 1165           __ movq(rax, Address(rsp, -8));
 1166 #ifndef PRODUCT
 1167         } else {
 1168           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1169                     "movl    rax, [rsp + #%d]\n\t"
 1170                     "movl    [rsp + #%d], rax\n\t"
 1171                     "movq    rax, [rsp - #8]",
 1172                      src_offset, dst_offset);
 1173 #endif
 1174         }
 1175       }
 1176       return 0;
 1177     } else if (dst_first_rc == rc_int) {
 1178       // mem -> gpr
 1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1181         // 64-bit
 1182         int offset = ra_->reg2offset(src_first);
 1183         if (masm) {
 1184           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1185 #ifndef PRODUCT
 1186         } else {
 1187           st->print("movq    %s, [rsp + #%d]\t# spill",
 1188                      Matcher::regName[dst_first],
 1189                      offset);
 1190 #endif
 1191         }
 1192       } else {
 1193         // 32-bit
 1194         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1195         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1196         int offset = ra_->reg2offset(src_first);
 1197         if (masm) {
 1198           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1199 #ifndef PRODUCT
 1200         } else {
 1201           st->print("movl    %s, [rsp + #%d]\t# spill",
 1202                      Matcher::regName[dst_first],
 1203                      offset);
 1204 #endif
 1205         }
 1206       }
 1207       return 0;
 1208     } else if (dst_first_rc == rc_float) {
 1209       // mem-> xmm
 1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1212         // 64-bit
 1213         int offset = ra_->reg2offset(src_first);
 1214         if (masm) {
 1215           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1216 #ifndef PRODUCT
 1217         } else {
 1218           st->print("%s  %s, [rsp + #%d]\t# spill",
 1219                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1220                      Matcher::regName[dst_first],
 1221                      offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         int offset = ra_->reg2offset(src_first);
 1229         if (masm) {
 1230           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("movss   %s, [rsp + #%d]\t# spill",
 1234                      Matcher::regName[dst_first],
 1235                      offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_kreg) {
 1241       // mem -> kreg
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (masm) {
 1247           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1248 #ifndef PRODUCT
 1249         } else {
 1250           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1251                      Matcher::regName[dst_first],
 1252                      offset);
 1253 #endif
 1254         }
 1255       }
 1256       return 0;
 1257     }
 1258   } else if (src_first_rc == rc_int) {
 1259     // gpr ->
 1260     if (dst_first_rc == rc_stack) {
 1261       // gpr -> mem
 1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1264         // 64-bit
 1265         int offset = ra_->reg2offset(dst_first);
 1266         if (masm) {
 1267           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1268 #ifndef PRODUCT
 1269         } else {
 1270           st->print("movq    [rsp + #%d], %s\t# spill",
 1271                      offset,
 1272                      Matcher::regName[src_first]);
 1273 #endif
 1274         }
 1275       } else {
 1276         // 32-bit
 1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1279         int offset = ra_->reg2offset(dst_first);
 1280         if (masm) {
 1281           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("movl    [rsp + #%d], %s\t# spill",
 1285                      offset,
 1286                      Matcher::regName[src_first]);
 1287 #endif
 1288         }
 1289       }
 1290       return 0;
 1291     } else if (dst_first_rc == rc_int) {
 1292       // gpr -> gpr
 1293       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1294           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1295         // 64-bit
 1296         if (masm) {
 1297           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1298                   as_Register(Matcher::_regEncode[src_first]));
 1299 #ifndef PRODUCT
 1300         } else {
 1301           st->print("movq    %s, %s\t# spill",
 1302                      Matcher::regName[dst_first],
 1303                      Matcher::regName[src_first]);
 1304 #endif
 1305         }
 1306         return 0;
 1307       } else {
 1308         // 32-bit
 1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1311         if (masm) {
 1312           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1313                   as_Register(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("movl    %s, %s\t# spill",
 1317                      Matcher::regName[dst_first],
 1318                      Matcher::regName[src_first]);
 1319 #endif
 1320         }
 1321         return 0;
 1322       }
 1323     } else if (dst_first_rc == rc_float) {
 1324       // gpr -> xmm
 1325       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1326           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1327         // 64-bit
 1328         if (masm) {
 1329           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1330 #ifndef PRODUCT
 1331         } else {
 1332           st->print("movdq   %s, %s\t# spill",
 1333                      Matcher::regName[dst_first],
 1334                      Matcher::regName[src_first]);
 1335 #endif
 1336         }
 1337       } else {
 1338         // 32-bit
 1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1341         if (masm) {
 1342           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1343 #ifndef PRODUCT
 1344         } else {
 1345           st->print("movdl   %s, %s\t# spill",
 1346                      Matcher::regName[dst_first],
 1347                      Matcher::regName[src_first]);
 1348 #endif
 1349         }
 1350       }
 1351       return 0;
 1352     } else if (dst_first_rc == rc_kreg) {
 1353       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1354           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1355         // 64-bit
 1356         if (masm) {
 1357           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1358   #ifndef PRODUCT
 1359         } else {
 1360            st->print("kmovq   %s, %s\t# spill",
 1361                        Matcher::regName[dst_first],
 1362                        Matcher::regName[src_first]);
 1363   #endif
 1364         }
 1365       }
 1366       Unimplemented();
 1367       return 0;
 1368     }
 1369   } else if (src_first_rc == rc_float) {
 1370     // xmm ->
 1371     if (dst_first_rc == rc_stack) {
 1372       // xmm -> mem
 1373       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1374           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1375         // 64-bit
 1376         int offset = ra_->reg2offset(dst_first);
 1377         if (masm) {
 1378           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1379 #ifndef PRODUCT
 1380         } else {
 1381           st->print("movsd   [rsp + #%d], %s\t# spill",
 1382                      offset,
 1383                      Matcher::regName[src_first]);
 1384 #endif
 1385         }
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         int offset = ra_->reg2offset(dst_first);
 1391         if (masm) {
 1392           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1393 #ifndef PRODUCT
 1394         } else {
 1395           st->print("movss   [rsp + #%d], %s\t# spill",
 1396                      offset,
 1397                      Matcher::regName[src_first]);
 1398 #endif
 1399         }
 1400       }
 1401       return 0;
 1402     } else if (dst_first_rc == rc_int) {
 1403       // xmm -> gpr
 1404       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1405           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1406         // 64-bit
 1407         if (masm) {
 1408           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       } else {
 1417         // 32-bit
 1418         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1419         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1420         if (masm) {
 1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1422 #ifndef PRODUCT
 1423         } else {
 1424           st->print("movdl   %s, %s\t# spill",
 1425                      Matcher::regName[dst_first],
 1426                      Matcher::regName[src_first]);
 1427 #endif
 1428         }
 1429       }
 1430       return 0;
 1431     } else if (dst_first_rc == rc_float) {
 1432       // xmm -> xmm
 1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1435         // 64-bit
 1436         if (masm) {
 1437           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1438 #ifndef PRODUCT
 1439         } else {
 1440           st->print("%s  %s, %s\t# spill",
 1441                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       } else {
 1447         // 32-bit
 1448         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1449         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1450         if (masm) {
 1451           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1452 #ifndef PRODUCT
 1453         } else {
 1454           st->print("%s  %s, %s\t# spill",
 1455                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1456                      Matcher::regName[dst_first],
 1457                      Matcher::regName[src_first]);
 1458 #endif
 1459         }
 1460       }
 1461       return 0;
 1462     } else if (dst_first_rc == rc_kreg) {
 1463       assert(false, "Illegal spilling");
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_kreg) {
 1467     if (dst_first_rc == rc_stack) {
 1468       // mem -> kreg
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         int offset = ra_->reg2offset(dst_first);
 1473         if (masm) {
 1474           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1478                      offset,
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       }
 1483       return 0;
 1484     } else if (dst_first_rc == rc_int) {
 1485       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1486           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1487         // 64-bit
 1488         if (masm) {
 1489           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1490 #ifndef PRODUCT
 1491         } else {
 1492          st->print("kmovq   %s, %s\t# spill",
 1493                      Matcher::regName[dst_first],
 1494                      Matcher::regName[src_first]);
 1495 #endif
 1496         }
 1497       }
 1498       Unimplemented();
 1499       return 0;
 1500     } else if (dst_first_rc == rc_kreg) {
 1501       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1502           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1503         // 64-bit
 1504         if (masm) {
 1505           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508          st->print("kmovq   %s, %s\t# spill",
 1509                      Matcher::regName[dst_first],
 1510                      Matcher::regName[src_first]);
 1511 #endif
 1512         }
 1513       }
 1514       return 0;
 1515     } else if (dst_first_rc == rc_float) {
 1516       assert(false, "Illegal spill");
 1517       return 0;
 1518     }
 1519   }
 1520 
 1521   assert(0," foo ");
 1522   Unimplemented();
 1523   return 0;
 1524 }
 1525 
 1526 #ifndef PRODUCT
 1527 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1528   implementation(nullptr, ra_, false, st);
 1529 }
 1530 #endif
 1531 
 1532 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1533   implementation(masm, ra_, false, nullptr);
 1534 }
 1535 
 1536 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1537   return MachNode::size(ra_);
 1538 }
 1539 
 1540 //=============================================================================
 1541 #ifndef PRODUCT
 1542 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1543 {
 1544   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1545   int reg = ra_->get_reg_first(this);
 1546   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1547             Matcher::regName[reg], offset);
 1548 }
 1549 #endif
 1550 
 1551 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1552 {
 1553   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1554   int reg = ra_->get_encode(this);
 1555 
 1556   __ lea(as_Register(reg), Address(rsp, offset));
 1557 }
 1558 
 1559 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1560 {
 1561   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1562   if (ra_->get_encode(this) > 15) {
 1563     return (offset < 0x80) ? 6 : 9; // REX2
 1564   } else {
 1565     return (offset < 0x80) ? 5 : 8; // REX
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1590 {
 1591   return MachNode::size(ra_); // too many variables; just compute it
 1592                               // the hard way
 1593 }
 1594 
 1595 
 1596 //=============================================================================
 1597 
 1598 bool Matcher::supports_vector_calling_convention(void) {
 1599   return EnableVectorSupport;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     DEBUG_ONLY(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     DEBUG_ONLY(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     __ lea(r10, RuntimeAddress((address)$meth$$method));
 1863     __ call(r10);
 1864     __ post_call_nop();
 1865   %}
 1866 
 1867   enc_class Java_Static_Call(method meth)
 1868   %{
 1869     // JAVA STATIC CALL
 1870     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1871     // determine who we intended to call.
 1872     if (!_method) {
 1873       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1874     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1875       // The NOP here is purely to ensure that eliding a call to
 1876       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1877       __ addr_nop_5();
 1878       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1879     } else {
 1880       int method_index = resolved_method_index(masm);
 1881       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1882                                                   : static_call_Relocation::spec(method_index);
 1883       address mark = __ pc();
 1884       int call_offset = __ offset();
 1885       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1886       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1887         // Calls of the same statically bound method can share
 1888         // a stub to the interpreter.
 1889         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1890       } else {
 1891         // Emit stubs for static call.
 1892         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1893         __ clear_inst_mark();
 1894         if (stub == nullptr) {
 1895           ciEnv::current()->record_failure("CodeCache is full");
 1896           return;
 1897         }
 1898       }
 1899     }
 1900     __ post_call_nop();
 1901   %}
 1902 
 1903   enc_class Java_Dynamic_Call(method meth) %{
 1904     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1905     __ post_call_nop();
 1906   %}
 1907 
 1908 %}
 1909 
 1910 
 1911 
 1912 //----------FRAME--------------------------------------------------------------
 1913 // Definition of frame structure and management information.
 1914 //
 1915 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1916 //                             |   (to get allocators register number
 1917 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1918 //  r   CALLER     |        |
 1919 //  o     |        +--------+      pad to even-align allocators stack-slot
 1920 //  w     V        |  pad0  |        numbers; owned by CALLER
 1921 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1922 //  h     ^        |   in   |  5
 1923 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1924 //  |     |        |        |  3
 1925 //  |     |        +--------+
 1926 //  V     |        | old out|      Empty on Intel, window on Sparc
 1927 //        |    old |preserve|      Must be even aligned.
 1928 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1929 //        |        |   in   |  3   area for Intel ret address
 1930 //     Owned by    |preserve|      Empty on Sparc.
 1931 //       SELF      +--------+
 1932 //        |        |  pad2  |  2   pad to align old SP
 1933 //        |        +--------+  1
 1934 //        |        | locks  |  0
 1935 //        |        +--------+----> OptoReg::stack0(), even aligned
 1936 //        |        |  pad1  | 11   pad to align new SP
 1937 //        |        +--------+
 1938 //        |        |        | 10
 1939 //        |        | spills |  9   spills
 1940 //        V        |        |  8   (pad0 slot for callee)
 1941 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1942 //        ^        |  out   |  7
 1943 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1944 //     Owned by    +--------+
 1945 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1946 //        |    new |preserve|      Must be even-aligned.
 1947 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1948 //        |        |        |
 1949 //
 1950 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1951 //         known from SELF's arguments and the Java calling convention.
 1952 //         Region 6-7 is determined per call site.
 1953 // Note 2: If the calling convention leaves holes in the incoming argument
 1954 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1955 //         are owned by the CALLEE.  Holes should not be necessary in the
 1956 //         incoming area, as the Java calling convention is completely under
 1957 //         the control of the AD file.  Doubles can be sorted and packed to
 1958 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1959 //         varargs C calling conventions.
 1960 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1961 //         even aligned with pad0 as needed.
 1962 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1963 //         region 6-11 is even aligned; it may be padded out more so that
 1964 //         the region from SP to FP meets the minimum stack alignment.
 1965 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1966 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1967 //         SP meets the minimum alignment.
 1968 
 1969 frame
 1970 %{
 1971   // These three registers define part of the calling convention
 1972   // between compiled code and the interpreter.
 1973   inline_cache_reg(RAX);                // Inline Cache Register
 1974 
 1975   // Optional: name the operand used by cisc-spilling to access
 1976   // [stack_pointer + offset]
 1977   cisc_spilling_operand_name(indOffset32);
 1978 
 1979   // Number of stack slots consumed by locking an object
 1980   sync_stack_slots(2);
 1981 
 1982   // Compiled code's Frame Pointer
 1983   frame_pointer(RSP);
 1984 
 1985   // Interpreter stores its frame pointer in a register which is
 1986   // stored to the stack by I2CAdaptors.
 1987   // I2CAdaptors convert from interpreted java to compiled java.
 1988   interpreter_frame_pointer(RBP);
 1989 
 1990   // Stack alignment requirement
 1991   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1992 
 1993   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1994   // for calls to C.  Supports the var-args backing area for register parms.
 1995   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1996 
 1997   // The after-PROLOG location of the return address.  Location of
 1998   // return address specifies a type (REG or STACK) and a number
 1999   // representing the register number (i.e. - use a register name) or
 2000   // stack slot.
 2001   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2002   // Otherwise, it is above the locks and verification slot and alignment word
 2003   return_addr(STACK - 2 +
 2004               align_up((Compile::current()->in_preserve_stack_slots() +
 2005                         Compile::current()->fixed_slots()),
 2006                        stack_alignment_in_slots()));
 2007 
 2008   // Location of compiled Java return values.  Same as C for now.
 2009   return_value
 2010   %{
 2011     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2012            "only return normal values");
 2013 
 2014     static const int lo[Op_RegL + 1] = {
 2015       0,
 2016       0,
 2017       RAX_num,  // Op_RegN
 2018       RAX_num,  // Op_RegI
 2019       RAX_num,  // Op_RegP
 2020       XMM0_num, // Op_RegF
 2021       XMM0_num, // Op_RegD
 2022       RAX_num   // Op_RegL
 2023     };
 2024     static const int hi[Op_RegL + 1] = {
 2025       0,
 2026       0,
 2027       OptoReg::Bad, // Op_RegN
 2028       OptoReg::Bad, // Op_RegI
 2029       RAX_H_num,    // Op_RegP
 2030       OptoReg::Bad, // Op_RegF
 2031       XMM0b_num,    // Op_RegD
 2032       RAX_H_num     // Op_RegL
 2033     };
 2034     // Excluded flags and vector registers.
 2035     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2036     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2037   %}
 2038 %}
 2039 
 2040 //----------ATTRIBUTES---------------------------------------------------------
 2041 //----------Operand Attributes-------------------------------------------------
 2042 op_attrib op_cost(0);        // Required cost attribute
 2043 
 2044 //----------Instruction Attributes---------------------------------------------
 2045 ins_attrib ins_cost(100);       // Required cost attribute
 2046 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2047 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2048                                 // a non-matching short branch variant
 2049                                 // of some long branch?
 2050 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2051                                 // be a power of 2) specifies the
 2052                                 // alignment that some part of the
 2053                                 // instruction (not necessarily the
 2054                                 // start) requires.  If > 1, a
 2055                                 // compute_padding() function must be
 2056                                 // provided for the instruction
 2057 
 2058 // Whether this node is expanded during code emission into a sequence of
 2059 // instructions and the first instruction can perform an implicit null check.
 2060 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 2061 
 2062 //----------OPERANDS-----------------------------------------------------------
 2063 // Operand definitions must precede instruction definitions for correct parsing
 2064 // in the ADLC because operands constitute user defined types which are used in
 2065 // instruction definitions.
 2066 
 2067 //----------Simple Operands----------------------------------------------------
 2068 // Immediate Operands
 2069 // Integer Immediate
 2070 operand immI()
 2071 %{
 2072   match(ConI);
 2073 
 2074   op_cost(10);
 2075   format %{ %}
 2076   interface(CONST_INTER);
 2077 %}
 2078 
 2079 // Constant for test vs zero
 2080 operand immI_0()
 2081 %{
 2082   predicate(n->get_int() == 0);
 2083   match(ConI);
 2084 
 2085   op_cost(0);
 2086   format %{ %}
 2087   interface(CONST_INTER);
 2088 %}
 2089 
 2090 // Constant for increment
 2091 operand immI_1()
 2092 %{
 2093   predicate(n->get_int() == 1);
 2094   match(ConI);
 2095 
 2096   op_cost(0);
 2097   format %{ %}
 2098   interface(CONST_INTER);
 2099 %}
 2100 
 2101 // Constant for decrement
 2102 operand immI_M1()
 2103 %{
 2104   predicate(n->get_int() == -1);
 2105   match(ConI);
 2106 
 2107   op_cost(0);
 2108   format %{ %}
 2109   interface(CONST_INTER);
 2110 %}
 2111 
 2112 operand immI_2()
 2113 %{
 2114   predicate(n->get_int() == 2);
 2115   match(ConI);
 2116 
 2117   op_cost(0);
 2118   format %{ %}
 2119   interface(CONST_INTER);
 2120 %}
 2121 
 2122 operand immI_4()
 2123 %{
 2124   predicate(n->get_int() == 4);
 2125   match(ConI);
 2126 
 2127   op_cost(0);
 2128   format %{ %}
 2129   interface(CONST_INTER);
 2130 %}
 2131 
 2132 operand immI_8()
 2133 %{
 2134   predicate(n->get_int() == 8);
 2135   match(ConI);
 2136 
 2137   op_cost(0);
 2138   format %{ %}
 2139   interface(CONST_INTER);
 2140 %}
 2141 
 2142 // Valid scale values for addressing modes
 2143 operand immI2()
 2144 %{
 2145   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2146   match(ConI);
 2147 
 2148   format %{ %}
 2149   interface(CONST_INTER);
 2150 %}
 2151 
 2152 operand immU7()
 2153 %{
 2154   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2155   match(ConI);
 2156 
 2157   op_cost(5);
 2158   format %{ %}
 2159   interface(CONST_INTER);
 2160 %}
 2161 
 2162 operand immI8()
 2163 %{
 2164   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2165   match(ConI);
 2166 
 2167   op_cost(5);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 operand immU8()
 2173 %{
 2174   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2175   match(ConI);
 2176 
 2177   op_cost(5);
 2178   format %{ %}
 2179   interface(CONST_INTER);
 2180 %}
 2181 
 2182 operand immI16()
 2183 %{
 2184   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2185   match(ConI);
 2186 
 2187   op_cost(10);
 2188   format %{ %}
 2189   interface(CONST_INTER);
 2190 %}
 2191 
 2192 // Int Immediate non-negative
 2193 operand immU31()
 2194 %{
 2195   predicate(n->get_int() >= 0);
 2196   match(ConI);
 2197 
 2198   op_cost(0);
 2199   format %{ %}
 2200   interface(CONST_INTER);
 2201 %}
 2202 
 2203 // Pointer Immediate
 2204 operand immP()
 2205 %{
 2206   match(ConP);
 2207 
 2208   op_cost(10);
 2209   format %{ %}
 2210   interface(CONST_INTER);
 2211 %}
 2212 
 2213 // Null Pointer Immediate
 2214 operand immP0()
 2215 %{
 2216   predicate(n->get_ptr() == 0);
 2217   match(ConP);
 2218 
 2219   op_cost(5);
 2220   format %{ %}
 2221   interface(CONST_INTER);
 2222 %}
 2223 
 2224 // Pointer Immediate
 2225 operand immN() %{
 2226   match(ConN);
 2227 
 2228   op_cost(10);
 2229   format %{ %}
 2230   interface(CONST_INTER);
 2231 %}
 2232 
 2233 operand immNKlass() %{
 2234   match(ConNKlass);
 2235 
 2236   op_cost(10);
 2237   format %{ %}
 2238   interface(CONST_INTER);
 2239 %}
 2240 
 2241 // Null Pointer Immediate
 2242 operand immN0() %{
 2243   predicate(n->get_narrowcon() == 0);
 2244   match(ConN);
 2245 
 2246   op_cost(5);
 2247   format %{ %}
 2248   interface(CONST_INTER);
 2249 %}
 2250 
 2251 operand immP31()
 2252 %{
 2253   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2254             && (n->get_ptr() >> 31) == 0);
 2255   match(ConP);
 2256 
 2257   op_cost(5);
 2258   format %{ %}
 2259   interface(CONST_INTER);
 2260 %}
 2261 
 2262 
 2263 // Long Immediate
 2264 operand immL()
 2265 %{
 2266   match(ConL);
 2267 
 2268   op_cost(20);
 2269   format %{ %}
 2270   interface(CONST_INTER);
 2271 %}
 2272 
 2273 // Long Immediate 8-bit
 2274 operand immL8()
 2275 %{
 2276   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2277   match(ConL);
 2278 
 2279   op_cost(5);
 2280   format %{ %}
 2281   interface(CONST_INTER);
 2282 %}
 2283 
 2284 // Long Immediate 32-bit unsigned
 2285 operand immUL32()
 2286 %{
 2287   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2288   match(ConL);
 2289 
 2290   op_cost(10);
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Long Immediate 32-bit signed
 2296 operand immL32()
 2297 %{
 2298   predicate(n->get_long() == (int) (n->get_long()));
 2299   match(ConL);
 2300 
 2301   op_cost(15);
 2302   format %{ %}
 2303   interface(CONST_INTER);
 2304 %}
 2305 
 2306 operand immL_Pow2()
 2307 %{
 2308   predicate(is_power_of_2((julong)n->get_long()));
 2309   match(ConL);
 2310 
 2311   op_cost(15);
 2312   format %{ %}
 2313   interface(CONST_INTER);
 2314 %}
 2315 
 2316 operand immL_NotPow2()
 2317 %{
 2318   predicate(is_power_of_2((julong)~n->get_long()));
 2319   match(ConL);
 2320 
 2321   op_cost(15);
 2322   format %{ %}
 2323   interface(CONST_INTER);
 2324 %}
 2325 
 2326 // Long Immediate zero
 2327 operand immL0()
 2328 %{
 2329   predicate(n->get_long() == 0L);
 2330   match(ConL);
 2331 
 2332   op_cost(10);
 2333   format %{ %}
 2334   interface(CONST_INTER);
 2335 %}
 2336 
 2337 // Constant for increment
 2338 operand immL1()
 2339 %{
 2340   predicate(n->get_long() == 1);
 2341   match(ConL);
 2342 
 2343   format %{ %}
 2344   interface(CONST_INTER);
 2345 %}
 2346 
 2347 // Constant for decrement
 2348 operand immL_M1()
 2349 %{
 2350   predicate(n->get_long() == -1);
 2351   match(ConL);
 2352 
 2353   format %{ %}
 2354   interface(CONST_INTER);
 2355 %}
 2356 
 2357 // Long Immediate: low 32-bit mask
 2358 operand immL_32bits()
 2359 %{
 2360   predicate(n->get_long() == 0xFFFFFFFFL);
 2361   match(ConL);
 2362   op_cost(20);
 2363 
 2364   format %{ %}
 2365   interface(CONST_INTER);
 2366 %}
 2367 
 2368 // Int Immediate: 2^n-1, positive
 2369 operand immI_Pow2M1()
 2370 %{
 2371   predicate((n->get_int() > 0)
 2372             && is_power_of_2((juint)n->get_int() + 1));
 2373   match(ConI);
 2374 
 2375   op_cost(20);
 2376   format %{ %}
 2377   interface(CONST_INTER);
 2378 %}
 2379 
 2380 // Float Immediate zero
 2381 operand immF0()
 2382 %{
 2383   predicate(jint_cast(n->getf()) == 0);
 2384   match(ConF);
 2385 
 2386   op_cost(5);
 2387   format %{ %}
 2388   interface(CONST_INTER);
 2389 %}
 2390 
 2391 // Float Immediate
 2392 operand immF()
 2393 %{
 2394   match(ConF);
 2395 
 2396   op_cost(15);
 2397   format %{ %}
 2398   interface(CONST_INTER);
 2399 %}
 2400 
 2401 // Half Float Immediate
 2402 operand immH()
 2403 %{
 2404   match(ConH);
 2405 
 2406   op_cost(15);
 2407   format %{ %}
 2408   interface(CONST_INTER);
 2409 %}
 2410 
 2411 // Double Immediate zero
 2412 operand immD0()
 2413 %{
 2414   predicate(jlong_cast(n->getd()) == 0);
 2415   match(ConD);
 2416 
 2417   op_cost(5);
 2418   format %{ %}
 2419   interface(CONST_INTER);
 2420 %}
 2421 
 2422 // Double Immediate
 2423 operand immD()
 2424 %{
 2425   match(ConD);
 2426 
 2427   op_cost(15);
 2428   format %{ %}
 2429   interface(CONST_INTER);
 2430 %}
 2431 
 2432 // Immediates for special shifts (sign extend)
 2433 
 2434 // Constants for increment
 2435 operand immI_16()
 2436 %{
 2437   predicate(n->get_int() == 16);
 2438   match(ConI);
 2439 
 2440   format %{ %}
 2441   interface(CONST_INTER);
 2442 %}
 2443 
 2444 operand immI_24()
 2445 %{
 2446   predicate(n->get_int() == 24);
 2447   match(ConI);
 2448 
 2449   format %{ %}
 2450   interface(CONST_INTER);
 2451 %}
 2452 
 2453 // Constant for byte-wide masking
 2454 operand immI_255()
 2455 %{
 2456   predicate(n->get_int() == 255);
 2457   match(ConI);
 2458 
 2459   format %{ %}
 2460   interface(CONST_INTER);
 2461 %}
 2462 
 2463 // Constant for short-wide masking
 2464 operand immI_65535()
 2465 %{
 2466   predicate(n->get_int() == 65535);
 2467   match(ConI);
 2468 
 2469   format %{ %}
 2470   interface(CONST_INTER);
 2471 %}
 2472 
 2473 // Constant for byte-wide masking
 2474 operand immL_255()
 2475 %{
 2476   predicate(n->get_long() == 255);
 2477   match(ConL);
 2478 
 2479   format %{ %}
 2480   interface(CONST_INTER);
 2481 %}
 2482 
 2483 // Constant for short-wide masking
 2484 operand immL_65535()
 2485 %{
 2486   predicate(n->get_long() == 65535);
 2487   match(ConL);
 2488 
 2489   format %{ %}
 2490   interface(CONST_INTER);
 2491 %}
 2492 
 2493 operand kReg()
 2494 %{
 2495   constraint(ALLOC_IN_RC(vectmask_reg));
 2496   match(RegVectMask);
 2497   format %{%}
 2498   interface(REG_INTER);
 2499 %}
 2500 
 2501 // Register Operands
 2502 // Integer Register
 2503 operand rRegI()
 2504 %{
 2505   constraint(ALLOC_IN_RC(int_reg));
 2506   match(RegI);
 2507 
 2508   match(rax_RegI);
 2509   match(rbx_RegI);
 2510   match(rcx_RegI);
 2511   match(rdx_RegI);
 2512   match(rdi_RegI);
 2513 
 2514   format %{ %}
 2515   interface(REG_INTER);
 2516 %}
 2517 
 2518 // Special Registers
 2519 operand rax_RegI()
 2520 %{
 2521   constraint(ALLOC_IN_RC(int_rax_reg));
 2522   match(RegI);
 2523   match(rRegI);
 2524 
 2525   format %{ "RAX" %}
 2526   interface(REG_INTER);
 2527 %}
 2528 
 2529 // Special Registers
 2530 operand rbx_RegI()
 2531 %{
 2532   constraint(ALLOC_IN_RC(int_rbx_reg));
 2533   match(RegI);
 2534   match(rRegI);
 2535 
 2536   format %{ "RBX" %}
 2537   interface(REG_INTER);
 2538 %}
 2539 
 2540 operand rcx_RegI()
 2541 %{
 2542   constraint(ALLOC_IN_RC(int_rcx_reg));
 2543   match(RegI);
 2544   match(rRegI);
 2545 
 2546   format %{ "RCX" %}
 2547   interface(REG_INTER);
 2548 %}
 2549 
 2550 operand rdx_RegI()
 2551 %{
 2552   constraint(ALLOC_IN_RC(int_rdx_reg));
 2553   match(RegI);
 2554   match(rRegI);
 2555 
 2556   format %{ "RDX" %}
 2557   interface(REG_INTER);
 2558 %}
 2559 
 2560 operand rdi_RegI()
 2561 %{
 2562   constraint(ALLOC_IN_RC(int_rdi_reg));
 2563   match(RegI);
 2564   match(rRegI);
 2565 
 2566   format %{ "RDI" %}
 2567   interface(REG_INTER);
 2568 %}
 2569 
 2570 operand no_rax_rdx_RegI()
 2571 %{
 2572   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2573   match(RegI);
 2574   match(rbx_RegI);
 2575   match(rcx_RegI);
 2576   match(rdi_RegI);
 2577 
 2578   format %{ %}
 2579   interface(REG_INTER);
 2580 %}
 2581 
 2582 operand no_rbp_r13_RegI()
 2583 %{
 2584   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2585   match(RegI);
 2586   match(rRegI);
 2587   match(rax_RegI);
 2588   match(rbx_RegI);
 2589   match(rcx_RegI);
 2590   match(rdx_RegI);
 2591   match(rdi_RegI);
 2592 
 2593   format %{ %}
 2594   interface(REG_INTER);
 2595 %}
 2596 
 2597 // Pointer Register
 2598 operand any_RegP()
 2599 %{
 2600   constraint(ALLOC_IN_RC(any_reg));
 2601   match(RegP);
 2602   match(rax_RegP);
 2603   match(rbx_RegP);
 2604   match(rdi_RegP);
 2605   match(rsi_RegP);
 2606   match(rbp_RegP);
 2607   match(r15_RegP);
 2608   match(rRegP);
 2609 
 2610   format %{ %}
 2611   interface(REG_INTER);
 2612 %}
 2613 
 2614 operand rRegP()
 2615 %{
 2616   constraint(ALLOC_IN_RC(ptr_reg));
 2617   match(RegP);
 2618   match(rax_RegP);
 2619   match(rbx_RegP);
 2620   match(rdi_RegP);
 2621   match(rsi_RegP);
 2622   match(rbp_RegP);  // See Q&A below about
 2623   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2624 
 2625   format %{ %}
 2626   interface(REG_INTER);
 2627 %}
 2628 
 2629 operand rRegN() %{
 2630   constraint(ALLOC_IN_RC(int_reg));
 2631   match(RegN);
 2632 
 2633   format %{ %}
 2634   interface(REG_INTER);
 2635 %}
 2636 
 2637 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2638 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2639 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2640 // The output of an instruction is controlled by the allocator, which respects
 2641 // register class masks, not match rules.  Unless an instruction mentions
 2642 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2643 // by the allocator as an input.
 2644 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2645 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2646 // result, RBP is not included in the output of the instruction either.
 2647 
 2648 // This operand is not allowed to use RBP even if
 2649 // RBP is not used to hold the frame pointer.
 2650 operand no_rbp_RegP()
 2651 %{
 2652   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2653   match(RegP);
 2654   match(rbx_RegP);
 2655   match(rsi_RegP);
 2656   match(rdi_RegP);
 2657 
 2658   format %{ %}
 2659   interface(REG_INTER);
 2660 %}
 2661 
 2662 // Special Registers
 2663 // Return a pointer value
 2664 operand rax_RegP()
 2665 %{
 2666   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2667   match(RegP);
 2668   match(rRegP);
 2669 
 2670   format %{ %}
 2671   interface(REG_INTER);
 2672 %}
 2673 
 2674 // Special Registers
 2675 // Return a compressed pointer value
 2676 operand rax_RegN()
 2677 %{
 2678   constraint(ALLOC_IN_RC(int_rax_reg));
 2679   match(RegN);
 2680   match(rRegN);
 2681 
 2682   format %{ %}
 2683   interface(REG_INTER);
 2684 %}
 2685 
 2686 // Used in AtomicAdd
 2687 operand rbx_RegP()
 2688 %{
 2689   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2690   match(RegP);
 2691   match(rRegP);
 2692 
 2693   format %{ %}
 2694   interface(REG_INTER);
 2695 %}
 2696 
 2697 operand rsi_RegP()
 2698 %{
 2699   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2700   match(RegP);
 2701   match(rRegP);
 2702 
 2703   format %{ %}
 2704   interface(REG_INTER);
 2705 %}
 2706 
 2707 operand rbp_RegP()
 2708 %{
 2709   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2710   match(RegP);
 2711   match(rRegP);
 2712 
 2713   format %{ %}
 2714   interface(REG_INTER);
 2715 %}
 2716 
 2717 // Used in rep stosq
 2718 operand rdi_RegP()
 2719 %{
 2720   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2721   match(RegP);
 2722   match(rRegP);
 2723 
 2724   format %{ %}
 2725   interface(REG_INTER);
 2726 %}
 2727 
 2728 operand r15_RegP()
 2729 %{
 2730   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2731   match(RegP);
 2732   match(rRegP);
 2733 
 2734   format %{ %}
 2735   interface(REG_INTER);
 2736 %}
 2737 
 2738 operand rRegL()
 2739 %{
 2740   constraint(ALLOC_IN_RC(long_reg));
 2741   match(RegL);
 2742   match(rax_RegL);
 2743   match(rdx_RegL);
 2744 
 2745   format %{ %}
 2746   interface(REG_INTER);
 2747 %}
 2748 
 2749 // Special Registers
 2750 operand no_rax_rdx_RegL()
 2751 %{
 2752   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2753   match(RegL);
 2754   match(rRegL);
 2755 
 2756   format %{ %}
 2757   interface(REG_INTER);
 2758 %}
 2759 
 2760 operand rax_RegL()
 2761 %{
 2762   constraint(ALLOC_IN_RC(long_rax_reg));
 2763   match(RegL);
 2764   match(rRegL);
 2765 
 2766   format %{ "RAX" %}
 2767   interface(REG_INTER);
 2768 %}
 2769 
 2770 operand rcx_RegL()
 2771 %{
 2772   constraint(ALLOC_IN_RC(long_rcx_reg));
 2773   match(RegL);
 2774   match(rRegL);
 2775 
 2776   format %{ %}
 2777   interface(REG_INTER);
 2778 %}
 2779 
 2780 operand rdx_RegL()
 2781 %{
 2782   constraint(ALLOC_IN_RC(long_rdx_reg));
 2783   match(RegL);
 2784   match(rRegL);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 operand r11_RegL()
 2791 %{
 2792   constraint(ALLOC_IN_RC(long_r11_reg));
 2793   match(RegL);
 2794   match(rRegL);
 2795 
 2796   format %{ %}
 2797   interface(REG_INTER);
 2798 %}
 2799 
 2800 operand no_rbp_r13_RegL()
 2801 %{
 2802   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2803   match(RegL);
 2804   match(rRegL);
 2805   match(rax_RegL);
 2806   match(rcx_RegL);
 2807   match(rdx_RegL);
 2808 
 2809   format %{ %}
 2810   interface(REG_INTER);
 2811 %}
 2812 
 2813 // Flags register, used as output of compare instructions
 2814 operand rFlagsReg()
 2815 %{
 2816   constraint(ALLOC_IN_RC(int_flags));
 2817   match(RegFlags);
 2818 
 2819   format %{ "RFLAGS" %}
 2820   interface(REG_INTER);
 2821 %}
 2822 
 2823 // Flags register, used as output of FLOATING POINT compare instructions
 2824 operand rFlagsRegU()
 2825 %{
 2826   constraint(ALLOC_IN_RC(int_flags));
 2827   match(RegFlags);
 2828 
 2829   format %{ "RFLAGS_U" %}
 2830   interface(REG_INTER);
 2831 %}
 2832 
 2833 operand rFlagsRegUCF() %{
 2834   constraint(ALLOC_IN_RC(int_flags));
 2835   match(RegFlags);
 2836   predicate(false);
 2837 
 2838   format %{ "RFLAGS_U_CF" %}
 2839   interface(REG_INTER);
 2840 %}
 2841 
 2842 // Float register operands
 2843 operand regF() %{
 2844    constraint(ALLOC_IN_RC(float_reg));
 2845    match(RegF);
 2846 
 2847    format %{ %}
 2848    interface(REG_INTER);
 2849 %}
 2850 
 2851 // Float register operands
 2852 operand legRegF() %{
 2853    constraint(ALLOC_IN_RC(float_reg_legacy));
 2854    match(RegF);
 2855 
 2856    format %{ %}
 2857    interface(REG_INTER);
 2858 %}
 2859 
 2860 // Float register operands
 2861 operand vlRegF() %{
 2862    constraint(ALLOC_IN_RC(float_reg_vl));
 2863    match(RegF);
 2864 
 2865    format %{ %}
 2866    interface(REG_INTER);
 2867 %}
 2868 
 2869 // Double register operands
 2870 operand regD() %{
 2871    constraint(ALLOC_IN_RC(double_reg));
 2872    match(RegD);
 2873 
 2874    format %{ %}
 2875    interface(REG_INTER);
 2876 %}
 2877 
 2878 // Double register operands
 2879 operand legRegD() %{
 2880    constraint(ALLOC_IN_RC(double_reg_legacy));
 2881    match(RegD);
 2882 
 2883    format %{ %}
 2884    interface(REG_INTER);
 2885 %}
 2886 
 2887 // Double register operands
 2888 operand vlRegD() %{
 2889    constraint(ALLOC_IN_RC(double_reg_vl));
 2890    match(RegD);
 2891 
 2892    format %{ %}
 2893    interface(REG_INTER);
 2894 %}
 2895 
 2896 //----------Memory Operands----------------------------------------------------
 2897 // Direct Memory Operand
 2898 // operand direct(immP addr)
 2899 // %{
 2900 //   match(addr);
 2901 
 2902 //   format %{ "[$addr]" %}
 2903 //   interface(MEMORY_INTER) %{
 2904 //     base(0xFFFFFFFF);
 2905 //     index(0x4);
 2906 //     scale(0x0);
 2907 //     disp($addr);
 2908 //   %}
 2909 // %}
 2910 
 2911 // Indirect Memory Operand
 2912 operand indirect(any_RegP reg)
 2913 %{
 2914   constraint(ALLOC_IN_RC(ptr_reg));
 2915   match(reg);
 2916 
 2917   format %{ "[$reg]" %}
 2918   interface(MEMORY_INTER) %{
 2919     base($reg);
 2920     index(0x4);
 2921     scale(0x0);
 2922     disp(0x0);
 2923   %}
 2924 %}
 2925 
 2926 // Indirect Memory Plus Short Offset Operand
 2927 operand indOffset8(any_RegP reg, immL8 off)
 2928 %{
 2929   constraint(ALLOC_IN_RC(ptr_reg));
 2930   match(AddP reg off);
 2931 
 2932   format %{ "[$reg + $off (8-bit)]" %}
 2933   interface(MEMORY_INTER) %{
 2934     base($reg);
 2935     index(0x4);
 2936     scale(0x0);
 2937     disp($off);
 2938   %}
 2939 %}
 2940 
 2941 // Indirect Memory Plus Long Offset Operand
 2942 operand indOffset32(any_RegP reg, immL32 off)
 2943 %{
 2944   constraint(ALLOC_IN_RC(ptr_reg));
 2945   match(AddP reg off);
 2946 
 2947   format %{ "[$reg + $off (32-bit)]" %}
 2948   interface(MEMORY_INTER) %{
 2949     base($reg);
 2950     index(0x4);
 2951     scale(0x0);
 2952     disp($off);
 2953   %}
 2954 %}
 2955 
 2956 // Indirect Memory Plus Index Register Plus Offset Operand
 2957 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2958 %{
 2959   constraint(ALLOC_IN_RC(ptr_reg));
 2960   match(AddP (AddP reg lreg) off);
 2961 
 2962   op_cost(10);
 2963   format %{"[$reg + $off + $lreg]" %}
 2964   interface(MEMORY_INTER) %{
 2965     base($reg);
 2966     index($lreg);
 2967     scale(0x0);
 2968     disp($off);
 2969   %}
 2970 %}
 2971 
 2972 // Indirect Memory Plus Index Register Plus Offset Operand
 2973 operand indIndex(any_RegP reg, rRegL lreg)
 2974 %{
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(AddP reg lreg);
 2977 
 2978   op_cost(10);
 2979   format %{"[$reg + $lreg]" %}
 2980   interface(MEMORY_INTER) %{
 2981     base($reg);
 2982     index($lreg);
 2983     scale(0x0);
 2984     disp(0x0);
 2985   %}
 2986 %}
 2987 
 2988 // Indirect Memory Times Scale Plus Index Register
 2989 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2990 %{
 2991   constraint(ALLOC_IN_RC(ptr_reg));
 2992   match(AddP reg (LShiftL lreg scale));
 2993 
 2994   op_cost(10);
 2995   format %{"[$reg + $lreg << $scale]" %}
 2996   interface(MEMORY_INTER) %{
 2997     base($reg);
 2998     index($lreg);
 2999     scale($scale);
 3000     disp(0x0);
 3001   %}
 3002 %}
 3003 
 3004 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3005 %{
 3006   constraint(ALLOC_IN_RC(ptr_reg));
 3007   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3008   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3009 
 3010   op_cost(10);
 3011   format %{"[$reg + pos $idx << $scale]" %}
 3012   interface(MEMORY_INTER) %{
 3013     base($reg);
 3014     index($idx);
 3015     scale($scale);
 3016     disp(0x0);
 3017   %}
 3018 %}
 3019 
 3020 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3021 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3022 %{
 3023   constraint(ALLOC_IN_RC(ptr_reg));
 3024   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3025 
 3026   op_cost(10);
 3027   format %{"[$reg + $off + $lreg << $scale]" %}
 3028   interface(MEMORY_INTER) %{
 3029     base($reg);
 3030     index($lreg);
 3031     scale($scale);
 3032     disp($off);
 3033   %}
 3034 %}
 3035 
 3036 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3037 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3038 %{
 3039   constraint(ALLOC_IN_RC(ptr_reg));
 3040   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3041   match(AddP (AddP reg (ConvI2L idx)) off);
 3042 
 3043   op_cost(10);
 3044   format %{"[$reg + $off + $idx]" %}
 3045   interface(MEMORY_INTER) %{
 3046     base($reg);
 3047     index($idx);
 3048     scale(0x0);
 3049     disp($off);
 3050   %}
 3051 %}
 3052 
 3053 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3054 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3055 %{
 3056   constraint(ALLOC_IN_RC(ptr_reg));
 3057   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3058   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3059 
 3060   op_cost(10);
 3061   format %{"[$reg + $off + $idx << $scale]" %}
 3062   interface(MEMORY_INTER) %{
 3063     base($reg);
 3064     index($idx);
 3065     scale($scale);
 3066     disp($off);
 3067   %}
 3068 %}
 3069 
 3070 // Indirect Narrow Oop Plus Offset Operand
 3071 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3072 // we can't free r12 even with CompressedOops::base() == nullptr.
 3073 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3074   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   match(AddP (DecodeN reg) off);
 3077 
 3078   op_cost(10);
 3079   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3080   interface(MEMORY_INTER) %{
 3081     base(0xc); // R12
 3082     index($reg);
 3083     scale(0x3);
 3084     disp($off);
 3085   %}
 3086 %}
 3087 
 3088 // Indirect Memory Operand
 3089 operand indirectNarrow(rRegN reg)
 3090 %{
 3091   predicate(CompressedOops::shift() == 0);
 3092   constraint(ALLOC_IN_RC(ptr_reg));
 3093   match(DecodeN reg);
 3094 
 3095   format %{ "[$reg]" %}
 3096   interface(MEMORY_INTER) %{
 3097     base($reg);
 3098     index(0x4);
 3099     scale(0x0);
 3100     disp(0x0);
 3101   %}
 3102 %}
 3103 
 3104 // Indirect Memory Plus Short Offset Operand
 3105 operand indOffset8Narrow(rRegN reg, immL8 off)
 3106 %{
 3107   predicate(CompressedOops::shift() == 0);
 3108   constraint(ALLOC_IN_RC(ptr_reg));
 3109   match(AddP (DecodeN reg) off);
 3110 
 3111   format %{ "[$reg + $off (8-bit)]" %}
 3112   interface(MEMORY_INTER) %{
 3113     base($reg);
 3114     index(0x4);
 3115     scale(0x0);
 3116     disp($off);
 3117   %}
 3118 %}
 3119 
 3120 // Indirect Memory Plus Long Offset Operand
 3121 operand indOffset32Narrow(rRegN reg, immL32 off)
 3122 %{
 3123   predicate(CompressedOops::shift() == 0);
 3124   constraint(ALLOC_IN_RC(ptr_reg));
 3125   match(AddP (DecodeN reg) off);
 3126 
 3127   format %{ "[$reg + $off (32-bit)]" %}
 3128   interface(MEMORY_INTER) %{
 3129     base($reg);
 3130     index(0x4);
 3131     scale(0x0);
 3132     disp($off);
 3133   %}
 3134 %}
 3135 
 3136 // Indirect Memory Plus Index Register Plus Offset Operand
 3137 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3138 %{
 3139   predicate(CompressedOops::shift() == 0);
 3140   constraint(ALLOC_IN_RC(ptr_reg));
 3141   match(AddP (AddP (DecodeN reg) lreg) off);
 3142 
 3143   op_cost(10);
 3144   format %{"[$reg + $off + $lreg]" %}
 3145   interface(MEMORY_INTER) %{
 3146     base($reg);
 3147     index($lreg);
 3148     scale(0x0);
 3149     disp($off);
 3150   %}
 3151 %}
 3152 
 3153 // Indirect Memory Plus Index Register Plus Offset Operand
 3154 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3155 %{
 3156   predicate(CompressedOops::shift() == 0);
 3157   constraint(ALLOC_IN_RC(ptr_reg));
 3158   match(AddP (DecodeN reg) lreg);
 3159 
 3160   op_cost(10);
 3161   format %{"[$reg + $lreg]" %}
 3162   interface(MEMORY_INTER) %{
 3163     base($reg);
 3164     index($lreg);
 3165     scale(0x0);
 3166     disp(0x0);
 3167   %}
 3168 %}
 3169 
 3170 // Indirect Memory Times Scale Plus Index Register
 3171 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3172 %{
 3173   predicate(CompressedOops::shift() == 0);
 3174   constraint(ALLOC_IN_RC(ptr_reg));
 3175   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3176 
 3177   op_cost(10);
 3178   format %{"[$reg + $lreg << $scale]" %}
 3179   interface(MEMORY_INTER) %{
 3180     base($reg);
 3181     index($lreg);
 3182     scale($scale);
 3183     disp(0x0);
 3184   %}
 3185 %}
 3186 
 3187 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3188 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3189 %{
 3190   predicate(CompressedOops::shift() == 0);
 3191   constraint(ALLOC_IN_RC(ptr_reg));
 3192   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3193 
 3194   op_cost(10);
 3195   format %{"[$reg + $off + $lreg << $scale]" %}
 3196   interface(MEMORY_INTER) %{
 3197     base($reg);
 3198     index($lreg);
 3199     scale($scale);
 3200     disp($off);
 3201   %}
 3202 %}
 3203 
 3204 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3205 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3206 %{
 3207   constraint(ALLOC_IN_RC(ptr_reg));
 3208   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3209   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3210 
 3211   op_cost(10);
 3212   format %{"[$reg + $off + $idx]" %}
 3213   interface(MEMORY_INTER) %{
 3214     base($reg);
 3215     index($idx);
 3216     scale(0x0);
 3217     disp($off);
 3218   %}
 3219 %}
 3220 
 3221 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3222 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3223 %{
 3224   constraint(ALLOC_IN_RC(ptr_reg));
 3225   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3226   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3227 
 3228   op_cost(10);
 3229   format %{"[$reg + $off + $idx << $scale]" %}
 3230   interface(MEMORY_INTER) %{
 3231     base($reg);
 3232     index($idx);
 3233     scale($scale);
 3234     disp($off);
 3235   %}
 3236 %}
 3237 
 3238 //----------Special Memory Operands--------------------------------------------
 3239 // Stack Slot Operand - This operand is used for loading and storing temporary
 3240 //                      values on the stack where a match requires a value to
 3241 //                      flow through memory.
 3242 operand stackSlotP(sRegP reg)
 3243 %{
 3244   constraint(ALLOC_IN_RC(stack_slots));
 3245   // No match rule because this operand is only generated in matching
 3246 
 3247   format %{ "[$reg]" %}
 3248   interface(MEMORY_INTER) %{
 3249     base(0x4);   // RSP
 3250     index(0x4);  // No Index
 3251     scale(0x0);  // No Scale
 3252     disp($reg);  // Stack Offset
 3253   %}
 3254 %}
 3255 
 3256 operand stackSlotI(sRegI reg)
 3257 %{
 3258   constraint(ALLOC_IN_RC(stack_slots));
 3259   // No match rule because this operand is only generated in matching
 3260 
 3261   format %{ "[$reg]" %}
 3262   interface(MEMORY_INTER) %{
 3263     base(0x4);   // RSP
 3264     index(0x4);  // No Index
 3265     scale(0x0);  // No Scale
 3266     disp($reg);  // Stack Offset
 3267   %}
 3268 %}
 3269 
 3270 operand stackSlotF(sRegF reg)
 3271 %{
 3272   constraint(ALLOC_IN_RC(stack_slots));
 3273   // No match rule because this operand is only generated in matching
 3274 
 3275   format %{ "[$reg]" %}
 3276   interface(MEMORY_INTER) %{
 3277     base(0x4);   // RSP
 3278     index(0x4);  // No Index
 3279     scale(0x0);  // No Scale
 3280     disp($reg);  // Stack Offset
 3281   %}
 3282 %}
 3283 
 3284 operand stackSlotD(sRegD reg)
 3285 %{
 3286   constraint(ALLOC_IN_RC(stack_slots));
 3287   // No match rule because this operand is only generated in matching
 3288 
 3289   format %{ "[$reg]" %}
 3290   interface(MEMORY_INTER) %{
 3291     base(0x4);   // RSP
 3292     index(0x4);  // No Index
 3293     scale(0x0);  // No Scale
 3294     disp($reg);  // Stack Offset
 3295   %}
 3296 %}
 3297 operand stackSlotL(sRegL reg)
 3298 %{
 3299   constraint(ALLOC_IN_RC(stack_slots));
 3300   // No match rule because this operand is only generated in matching
 3301 
 3302   format %{ "[$reg]" %}
 3303   interface(MEMORY_INTER) %{
 3304     base(0x4);   // RSP
 3305     index(0x4);  // No Index
 3306     scale(0x0);  // No Scale
 3307     disp($reg);  // Stack Offset
 3308   %}
 3309 %}
 3310 
 3311 //----------Conditional Branch Operands----------------------------------------
 3312 // Comparison Op  - This is the operation of the comparison, and is limited to
 3313 //                  the following set of codes:
 3314 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3315 //
 3316 // Other attributes of the comparison, such as unsignedness, are specified
 3317 // by the comparison instruction that sets a condition code flags register.
 3318 // That result is represented by a flags operand whose subtype is appropriate
 3319 // to the unsignedness (etc.) of the comparison.
 3320 //
 3321 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3322 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3323 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3324 
 3325 // Comparison Code
 3326 operand cmpOp()
 3327 %{
 3328   match(Bool);
 3329 
 3330   format %{ "" %}
 3331   interface(COND_INTER) %{
 3332     equal(0x4, "e");
 3333     not_equal(0x5, "ne");
 3334     less(0xC, "l");
 3335     greater_equal(0xD, "ge");
 3336     less_equal(0xE, "le");
 3337     greater(0xF, "g");
 3338     overflow(0x0, "o");
 3339     no_overflow(0x1, "no");
 3340   %}
 3341 %}
 3342 
 3343 // Comparison Code, unsigned compare.  Used by FP also, with
 3344 // C2 (unordered) turned into GT or LT already.  The other bits
 3345 // C0 and C3 are turned into Carry & Zero flags.
 3346 operand cmpOpU()
 3347 %{
 3348   match(Bool);
 3349 
 3350   format %{ "" %}
 3351   interface(COND_INTER) %{
 3352     equal(0x4, "e");
 3353     not_equal(0x5, "ne");
 3354     less(0x2, "b");
 3355     greater_equal(0x3, "ae");
 3356     less_equal(0x6, "be");
 3357     greater(0x7, "a");
 3358     overflow(0x0, "o");
 3359     no_overflow(0x1, "no");
 3360   %}
 3361 %}
 3362 
 3363 
 3364 // Floating comparisons that don't require any fixup for the unordered case,
 3365 // If both inputs of the comparison are the same, ZF is always set so we
 3366 // don't need to use cmpOpUCF2 for eq/ne
 3367 operand cmpOpUCF() %{
 3368   match(Bool);
 3369   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3370             n->as_Bool()->_test._test == BoolTest::ge ||
 3371             n->as_Bool()->_test._test == BoolTest::le ||
 3372             n->as_Bool()->_test._test == BoolTest::gt ||
 3373             n->in(1)->in(1) == n->in(1)->in(2));
 3374   format %{ "" %}
 3375   interface(COND_INTER) %{
 3376     equal(0xb, "np");
 3377     not_equal(0xa, "p");
 3378     less(0x2, "b");
 3379     greater_equal(0x3, "ae");
 3380     less_equal(0x6, "be");
 3381     greater(0x7, "a");
 3382     overflow(0x0, "o");
 3383     no_overflow(0x1, "no");
 3384   %}
 3385 %}
 3386 
 3387 
 3388 // Floating comparisons that can be fixed up with extra conditional jumps
 3389 operand cmpOpUCF2() %{
 3390   match(Bool);
 3391   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3392              n->as_Bool()->_test._test == BoolTest::eq) &&
 3393             n->in(1)->in(1) != n->in(1)->in(2));
 3394   format %{ "" %}
 3395   interface(COND_INTER) %{
 3396     equal(0x4, "e");
 3397     not_equal(0x5, "ne");
 3398     less(0x2, "b");
 3399     greater_equal(0x3, "ae");
 3400     less_equal(0x6, "be");
 3401     greater(0x7, "a");
 3402     overflow(0x0, "o");
 3403     no_overflow(0x1, "no");
 3404   %}
 3405 %}
 3406 
 3407 //----------OPERAND CLASSES----------------------------------------------------
 3408 // Operand Classes are groups of operands that are used as to simplify
 3409 // instruction definitions by not requiring the AD writer to specify separate
 3410 // instructions for every form of operand when the instruction accepts
 3411 // multiple operand types with the same basic encoding and format.  The classic
 3412 // case of this is memory operands.
 3413 
 3414 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3415                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3416                indCompressedOopOffset,
 3417                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3418                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3419                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3420 
 3421 //----------PIPELINE-----------------------------------------------------------
 3422 // Rules which define the behavior of the target architectures pipeline.
 3423 pipeline %{
 3424 
 3425 //----------ATTRIBUTES---------------------------------------------------------
 3426 attributes %{
 3427   variable_size_instructions;        // Fixed size instructions
 3428   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3429   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3430   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3431   instruction_fetch_units = 1;       // of 16 bytes
 3432 %}
 3433 
 3434 //----------RESOURCES----------------------------------------------------------
 3435 // Resources are the functional units available to the machine
 3436 
 3437 // Generic P2/P3 pipeline
 3438 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3439 // 3 instructions decoded per cycle.
 3440 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3441 // 3 ALU op, only ALU0 handles mul instructions.
 3442 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3443            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3444            BR, FPU,
 3445            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3446 
 3447 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3448 // Pipeline Description specifies the stages in the machine's pipeline
 3449 
 3450 // Generic P2/P3 pipeline
 3451 pipe_desc(S0, S1, S2, S3, S4, S5);
 3452 
 3453 //----------PIPELINE CLASSES---------------------------------------------------
 3454 // Pipeline Classes describe the stages in which input and output are
 3455 // referenced by the hardware pipeline.
 3456 
 3457 // Naming convention: ialu or fpu
 3458 // Then: _reg
 3459 // Then: _reg if there is a 2nd register
 3460 // Then: _long if it's a pair of instructions implementing a long
 3461 // Then: _fat if it requires the big decoder
 3462 //   Or: _mem if it requires the big decoder and a memory unit.
 3463 
 3464 // Integer ALU reg operation
 3465 pipe_class ialu_reg(rRegI dst)
 3466 %{
 3467     single_instruction;
 3468     dst    : S4(write);
 3469     dst    : S3(read);
 3470     DECODE : S0;        // any decoder
 3471     ALU    : S3;        // any alu
 3472 %}
 3473 
 3474 // Long ALU reg operation
 3475 pipe_class ialu_reg_long(rRegL dst)
 3476 %{
 3477     instruction_count(2);
 3478     dst    : S4(write);
 3479     dst    : S3(read);
 3480     DECODE : S0(2);     // any 2 decoders
 3481     ALU    : S3(2);     // both alus
 3482 %}
 3483 
 3484 // Integer ALU reg operation using big decoder
 3485 pipe_class ialu_reg_fat(rRegI dst)
 3486 %{
 3487     single_instruction;
 3488     dst    : S4(write);
 3489     dst    : S3(read);
 3490     D0     : S0;        // big decoder only
 3491     ALU    : S3;        // any alu
 3492 %}
 3493 
 3494 // Integer ALU reg-reg operation
 3495 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3496 %{
 3497     single_instruction;
 3498     dst    : S4(write);
 3499     src    : S3(read);
 3500     DECODE : S0;        // any decoder
 3501     ALU    : S3;        // any alu
 3502 %}
 3503 
 3504 // Integer ALU reg-reg operation
 3505 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3506 %{
 3507     single_instruction;
 3508     dst    : S4(write);
 3509     src    : S3(read);
 3510     D0     : S0;        // big decoder only
 3511     ALU    : S3;        // any alu
 3512 %}
 3513 
 3514 // Integer ALU reg-mem operation
 3515 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3516 %{
 3517     single_instruction;
 3518     dst    : S5(write);
 3519     mem    : S3(read);
 3520     D0     : S0;        // big decoder only
 3521     ALU    : S4;        // any alu
 3522     MEM    : S3;        // any mem
 3523 %}
 3524 
 3525 // Integer mem operation (prefetch)
 3526 pipe_class ialu_mem(memory mem)
 3527 %{
 3528     single_instruction;
 3529     mem    : S3(read);
 3530     D0     : S0;        // big decoder only
 3531     MEM    : S3;        // any mem
 3532 %}
 3533 
 3534 // Integer Store to Memory
 3535 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3536 %{
 3537     single_instruction;
 3538     mem    : S3(read);
 3539     src    : S5(read);
 3540     D0     : S0;        // big decoder only
 3541     ALU    : S4;        // any alu
 3542     MEM    : S3;
 3543 %}
 3544 
 3545 // // Long Store to Memory
 3546 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3547 // %{
 3548 //     instruction_count(2);
 3549 //     mem    : S3(read);
 3550 //     src    : S5(read);
 3551 //     D0     : S0(2);          // big decoder only; twice
 3552 //     ALU    : S4(2);     // any 2 alus
 3553 //     MEM    : S3(2);  // Both mems
 3554 // %}
 3555 
 3556 // Integer Store to Memory
 3557 pipe_class ialu_mem_imm(memory mem)
 3558 %{
 3559     single_instruction;
 3560     mem    : S3(read);
 3561     D0     : S0;        // big decoder only
 3562     ALU    : S4;        // any alu
 3563     MEM    : S3;
 3564 %}
 3565 
 3566 // Integer ALU0 reg-reg operation
 3567 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3568 %{
 3569     single_instruction;
 3570     dst    : S4(write);
 3571     src    : S3(read);
 3572     D0     : S0;        // Big decoder only
 3573     ALU0   : S3;        // only alu0
 3574 %}
 3575 
 3576 // Integer ALU0 reg-mem operation
 3577 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3578 %{
 3579     single_instruction;
 3580     dst    : S5(write);
 3581     mem    : S3(read);
 3582     D0     : S0;        // big decoder only
 3583     ALU0   : S4;        // ALU0 only
 3584     MEM    : S3;        // any mem
 3585 %}
 3586 
 3587 // Integer ALU reg-reg operation
 3588 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3589 %{
 3590     single_instruction;
 3591     cr     : S4(write);
 3592     src1   : S3(read);
 3593     src2   : S3(read);
 3594     DECODE : S0;        // any decoder
 3595     ALU    : S3;        // any alu
 3596 %}
 3597 
 3598 // Integer ALU reg-imm operation
 3599 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3600 %{
 3601     single_instruction;
 3602     cr     : S4(write);
 3603     src1   : S3(read);
 3604     DECODE : S0;        // any decoder
 3605     ALU    : S3;        // any alu
 3606 %}
 3607 
 3608 // Integer ALU reg-mem operation
 3609 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3610 %{
 3611     single_instruction;
 3612     cr     : S4(write);
 3613     src1   : S3(read);
 3614     src2   : S3(read);
 3615     D0     : S0;        // big decoder only
 3616     ALU    : S4;        // any alu
 3617     MEM    : S3;
 3618 %}
 3619 
 3620 // Conditional move reg-reg
 3621 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3622 %{
 3623     instruction_count(4);
 3624     y      : S4(read);
 3625     q      : S3(read);
 3626     p      : S3(read);
 3627     DECODE : S0(4);     // any decoder
 3628 %}
 3629 
 3630 // Conditional move reg-reg
 3631 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3632 %{
 3633     single_instruction;
 3634     dst    : S4(write);
 3635     src    : S3(read);
 3636     cr     : S3(read);
 3637     DECODE : S0;        // any decoder
 3638 %}
 3639 
 3640 // Conditional move reg-mem
 3641 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3642 %{
 3643     single_instruction;
 3644     dst    : S4(write);
 3645     src    : S3(read);
 3646     cr     : S3(read);
 3647     DECODE : S0;        // any decoder
 3648     MEM    : S3;
 3649 %}
 3650 
 3651 // Conditional move reg-reg long
 3652 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3653 %{
 3654     single_instruction;
 3655     dst    : S4(write);
 3656     src    : S3(read);
 3657     cr     : S3(read);
 3658     DECODE : S0(2);     // any 2 decoders
 3659 %}
 3660 
 3661 // Float reg-reg operation
 3662 pipe_class fpu_reg(regD dst)
 3663 %{
 3664     instruction_count(2);
 3665     dst    : S3(read);
 3666     DECODE : S0(2);     // any 2 decoders
 3667     FPU    : S3;
 3668 %}
 3669 
 3670 // Float reg-reg operation
 3671 pipe_class fpu_reg_reg(regD dst, regD src)
 3672 %{
 3673     instruction_count(2);
 3674     dst    : S4(write);
 3675     src    : S3(read);
 3676     DECODE : S0(2);     // any 2 decoders
 3677     FPU    : S3;
 3678 %}
 3679 
 3680 // Float reg-reg operation
 3681 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3682 %{
 3683     instruction_count(3);
 3684     dst    : S4(write);
 3685     src1   : S3(read);
 3686     src2   : S3(read);
 3687     DECODE : S0(3);     // any 3 decoders
 3688     FPU    : S3(2);
 3689 %}
 3690 
 3691 // Float reg-reg operation
 3692 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3693 %{
 3694     instruction_count(4);
 3695     dst    : S4(write);
 3696     src1   : S3(read);
 3697     src2   : S3(read);
 3698     src3   : S3(read);
 3699     DECODE : S0(4);     // any 3 decoders
 3700     FPU    : S3(2);
 3701 %}
 3702 
 3703 // Float reg-reg operation
 3704 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3705 %{
 3706     instruction_count(4);
 3707     dst    : S4(write);
 3708     src1   : S3(read);
 3709     src2   : S3(read);
 3710     src3   : S3(read);
 3711     DECODE : S1(3);     // any 3 decoders
 3712     D0     : S0;        // Big decoder only
 3713     FPU    : S3(2);
 3714     MEM    : S3;
 3715 %}
 3716 
 3717 // Float reg-mem operation
 3718 pipe_class fpu_reg_mem(regD dst, memory mem)
 3719 %{
 3720     instruction_count(2);
 3721     dst    : S5(write);
 3722     mem    : S3(read);
 3723     D0     : S0;        // big decoder only
 3724     DECODE : S1;        // any decoder for FPU POP
 3725     FPU    : S4;
 3726     MEM    : S3;        // any mem
 3727 %}
 3728 
 3729 // Float reg-mem operation
 3730 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3731 %{
 3732     instruction_count(3);
 3733     dst    : S5(write);
 3734     src1   : S3(read);
 3735     mem    : S3(read);
 3736     D0     : S0;        // big decoder only
 3737     DECODE : S1(2);     // any decoder for FPU POP
 3738     FPU    : S4;
 3739     MEM    : S3;        // any mem
 3740 %}
 3741 
 3742 // Float mem-reg operation
 3743 pipe_class fpu_mem_reg(memory mem, regD src)
 3744 %{
 3745     instruction_count(2);
 3746     src    : S5(read);
 3747     mem    : S3(read);
 3748     DECODE : S0;        // any decoder for FPU PUSH
 3749     D0     : S1;        // big decoder only
 3750     FPU    : S4;
 3751     MEM    : S3;        // any mem
 3752 %}
 3753 
 3754 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3755 %{
 3756     instruction_count(3);
 3757     src1   : S3(read);
 3758     src2   : S3(read);
 3759     mem    : S3(read);
 3760     DECODE : S0(2);     // any decoder for FPU PUSH
 3761     D0     : S1;        // big decoder only
 3762     FPU    : S4;
 3763     MEM    : S3;        // any mem
 3764 %}
 3765 
 3766 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3767 %{
 3768     instruction_count(3);
 3769     src1   : S3(read);
 3770     src2   : S3(read);
 3771     mem    : S4(read);
 3772     DECODE : S0;        // any decoder for FPU PUSH
 3773     D0     : S0(2);     // big decoder only
 3774     FPU    : S4;
 3775     MEM    : S3(2);     // any mem
 3776 %}
 3777 
 3778 pipe_class fpu_mem_mem(memory dst, memory src1)
 3779 %{
 3780     instruction_count(2);
 3781     src1   : S3(read);
 3782     dst    : S4(read);
 3783     D0     : S0(2);     // big decoder only
 3784     MEM    : S3(2);     // any mem
 3785 %}
 3786 
 3787 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3788 %{
 3789     instruction_count(3);
 3790     src1   : S3(read);
 3791     src2   : S3(read);
 3792     dst    : S4(read);
 3793     D0     : S0(3);     // big decoder only
 3794     FPU    : S4;
 3795     MEM    : S3(3);     // any mem
 3796 %}
 3797 
 3798 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3799 %{
 3800     instruction_count(3);
 3801     src1   : S4(read);
 3802     mem    : S4(read);
 3803     DECODE : S0;        // any decoder for FPU PUSH
 3804     D0     : S0(2);     // big decoder only
 3805     FPU    : S4;
 3806     MEM    : S3(2);     // any mem
 3807 %}
 3808 
 3809 // Float load constant
 3810 pipe_class fpu_reg_con(regD dst)
 3811 %{
 3812     instruction_count(2);
 3813     dst    : S5(write);
 3814     D0     : S0;        // big decoder only for the load
 3815     DECODE : S1;        // any decoder for FPU POP
 3816     FPU    : S4;
 3817     MEM    : S3;        // any mem
 3818 %}
 3819 
 3820 // Float load constant
 3821 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3822 %{
 3823     instruction_count(3);
 3824     dst    : S5(write);
 3825     src    : S3(read);
 3826     D0     : S0;        // big decoder only for the load
 3827     DECODE : S1(2);     // any decoder for FPU POP
 3828     FPU    : S4;
 3829     MEM    : S3;        // any mem
 3830 %}
 3831 
 3832 // UnConditional branch
 3833 pipe_class pipe_jmp(label labl)
 3834 %{
 3835     single_instruction;
 3836     BR   : S3;
 3837 %}
 3838 
 3839 // Conditional branch
 3840 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3841 %{
 3842     single_instruction;
 3843     cr    : S1(read);
 3844     BR    : S3;
 3845 %}
 3846 
 3847 // Allocation idiom
 3848 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3849 %{
 3850     instruction_count(1); force_serialization;
 3851     fixed_latency(6);
 3852     heap_ptr : S3(read);
 3853     DECODE   : S0(3);
 3854     D0       : S2;
 3855     MEM      : S3;
 3856     ALU      : S3(2);
 3857     dst      : S5(write);
 3858     BR       : S5;
 3859 %}
 3860 
 3861 // Generic big/slow expanded idiom
 3862 pipe_class pipe_slow()
 3863 %{
 3864     instruction_count(10); multiple_bundles; force_serialization;
 3865     fixed_latency(100);
 3866     D0  : S0(2);
 3867     MEM : S3(2);
 3868 %}
 3869 
 3870 // The real do-nothing guy
 3871 pipe_class empty()
 3872 %{
 3873     instruction_count(0);
 3874 %}
 3875 
 3876 // Define the class for the Nop node
 3877 define
 3878 %{
 3879    MachNop = empty;
 3880 %}
 3881 
 3882 %}
 3883 
 3884 //----------INSTRUCTIONS-------------------------------------------------------
 3885 //
 3886 // match      -- States which machine-independent subtree may be replaced
 3887 //               by this instruction.
 3888 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3889 //               selection to identify a minimum cost tree of machine
 3890 //               instructions that matches a tree of machine-independent
 3891 //               instructions.
 3892 // format     -- A string providing the disassembly for this instruction.
 3893 //               The value of an instruction's operand may be inserted
 3894 //               by referring to it with a '$' prefix.
 3895 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3896 //               to within an encode class as $primary, $secondary, and $tertiary
 3897 //               rrspectively.  The primary opcode is commonly used to
 3898 //               indicate the type of machine instruction, while secondary
 3899 //               and tertiary are often used for prefix options or addressing
 3900 //               modes.
 3901 // ins_encode -- A list of encode classes with parameters. The encode class
 3902 //               name must have been defined in an 'enc_class' specification
 3903 //               in the encode section of the architecture description.
 3904 
 3905 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3906 // Load Float
 3907 instruct MoveF2VL(vlRegF dst, regF src) %{
 3908   match(Set dst src);
 3909   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3910   ins_encode %{
 3911     ShouldNotReachHere();
 3912   %}
 3913   ins_pipe( fpu_reg_reg );
 3914 %}
 3915 
 3916 // Load Float
 3917 instruct MoveF2LEG(legRegF dst, regF src) %{
 3918   match(Set dst src);
 3919   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3920   ins_encode %{
 3921     ShouldNotReachHere();
 3922   %}
 3923   ins_pipe( fpu_reg_reg );
 3924 %}
 3925 
 3926 // Load Float
 3927 instruct MoveVL2F(regF dst, vlRegF src) %{
 3928   match(Set dst src);
 3929   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3930   ins_encode %{
 3931     ShouldNotReachHere();
 3932   %}
 3933   ins_pipe( fpu_reg_reg );
 3934 %}
 3935 
 3936 // Load Float
 3937 instruct MoveLEG2F(regF dst, legRegF src) %{
 3938   match(Set dst src);
 3939   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3940   ins_encode %{
 3941     ShouldNotReachHere();
 3942   %}
 3943   ins_pipe( fpu_reg_reg );
 3944 %}
 3945 
 3946 // Load Double
 3947 instruct MoveD2VL(vlRegD dst, regD src) %{
 3948   match(Set dst src);
 3949   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3950   ins_encode %{
 3951     ShouldNotReachHere();
 3952   %}
 3953   ins_pipe( fpu_reg_reg );
 3954 %}
 3955 
 3956 // Load Double
 3957 instruct MoveD2LEG(legRegD dst, regD src) %{
 3958   match(Set dst src);
 3959   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3960   ins_encode %{
 3961     ShouldNotReachHere();
 3962   %}
 3963   ins_pipe( fpu_reg_reg );
 3964 %}
 3965 
 3966 // Load Double
 3967 instruct MoveVL2D(regD dst, vlRegD src) %{
 3968   match(Set dst src);
 3969   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3970   ins_encode %{
 3971     ShouldNotReachHere();
 3972   %}
 3973   ins_pipe( fpu_reg_reg );
 3974 %}
 3975 
 3976 // Load Double
 3977 instruct MoveLEG2D(regD dst, legRegD src) %{
 3978   match(Set dst src);
 3979   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3980   ins_encode %{
 3981     ShouldNotReachHere();
 3982   %}
 3983   ins_pipe( fpu_reg_reg );
 3984 %}
 3985 
 3986 //----------Load/Store/Move Instructions---------------------------------------
 3987 //----------Load Instructions--------------------------------------------------
 3988 
 3989 // Load Byte (8 bit signed)
 3990 instruct loadB(rRegI dst, memory mem)
 3991 %{
 3992   match(Set dst (LoadB mem));
 3993 
 3994   ins_cost(125);
 3995   format %{ "movsbl  $dst, $mem\t# byte" %}
 3996 
 3997   ins_encode %{
 3998     __ movsbl($dst$$Register, $mem$$Address);
 3999   %}
 4000 
 4001   ins_pipe(ialu_reg_mem);
 4002 %}
 4003 
 4004 // Load Byte (8 bit signed) into Long Register
 4005 instruct loadB2L(rRegL dst, memory mem)
 4006 %{
 4007   match(Set dst (ConvI2L (LoadB mem)));
 4008 
 4009   ins_cost(125);
 4010   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4011 
 4012   ins_encode %{
 4013     __ movsbq($dst$$Register, $mem$$Address);
 4014   %}
 4015 
 4016   ins_pipe(ialu_reg_mem);
 4017 %}
 4018 
 4019 // Load Unsigned Byte (8 bit UNsigned)
 4020 instruct loadUB(rRegI dst, memory mem)
 4021 %{
 4022   match(Set dst (LoadUB mem));
 4023 
 4024   ins_cost(125);
 4025   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4026 
 4027   ins_encode %{
 4028     __ movzbl($dst$$Register, $mem$$Address);
 4029   %}
 4030 
 4031   ins_pipe(ialu_reg_mem);
 4032 %}
 4033 
 4034 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4035 instruct loadUB2L(rRegL dst, memory mem)
 4036 %{
 4037   match(Set dst (ConvI2L (LoadUB mem)));
 4038 
 4039   ins_cost(125);
 4040   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4041 
 4042   ins_encode %{
 4043     __ movzbq($dst$$Register, $mem$$Address);
 4044   %}
 4045 
 4046   ins_pipe(ialu_reg_mem);
 4047 %}
 4048 
 4049 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4050 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4051   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4052   effect(KILL cr);
 4053 
 4054   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4055             "andl    $dst, right_n_bits($mask, 8)" %}
 4056   ins_encode %{
 4057     Register Rdst = $dst$$Register;
 4058     __ movzbq(Rdst, $mem$$Address);
 4059     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4060   %}
 4061   ins_pipe(ialu_reg_mem);
 4062 %}
 4063 
 4064 // Load Short (16 bit signed)
 4065 instruct loadS(rRegI dst, memory mem)
 4066 %{
 4067   match(Set dst (LoadS mem));
 4068 
 4069   ins_cost(125);
 4070   format %{ "movswl $dst, $mem\t# short" %}
 4071 
 4072   ins_encode %{
 4073     __ movswl($dst$$Register, $mem$$Address);
 4074   %}
 4075 
 4076   ins_pipe(ialu_reg_mem);
 4077 %}
 4078 
 4079 // Load Short (16 bit signed) to Byte (8 bit signed)
 4080 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4081   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4082 
 4083   ins_cost(125);
 4084   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4085   ins_encode %{
 4086     __ movsbl($dst$$Register, $mem$$Address);
 4087   %}
 4088   ins_pipe(ialu_reg_mem);
 4089 %}
 4090 
 4091 // Load Short (16 bit signed) into Long Register
 4092 instruct loadS2L(rRegL dst, memory mem)
 4093 %{
 4094   match(Set dst (ConvI2L (LoadS mem)));
 4095 
 4096   ins_cost(125);
 4097   format %{ "movswq $dst, $mem\t# short -> long" %}
 4098 
 4099   ins_encode %{
 4100     __ movswq($dst$$Register, $mem$$Address);
 4101   %}
 4102 
 4103   ins_pipe(ialu_reg_mem);
 4104 %}
 4105 
 4106 // Load Unsigned Short/Char (16 bit UNsigned)
 4107 instruct loadUS(rRegI dst, memory mem)
 4108 %{
 4109   match(Set dst (LoadUS mem));
 4110 
 4111   ins_cost(125);
 4112   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4113 
 4114   ins_encode %{
 4115     __ movzwl($dst$$Register, $mem$$Address);
 4116   %}
 4117 
 4118   ins_pipe(ialu_reg_mem);
 4119 %}
 4120 
 4121 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4122 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4123   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4124 
 4125   ins_cost(125);
 4126   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4127   ins_encode %{
 4128     __ movsbl($dst$$Register, $mem$$Address);
 4129   %}
 4130   ins_pipe(ialu_reg_mem);
 4131 %}
 4132 
 4133 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4134 instruct loadUS2L(rRegL dst, memory mem)
 4135 %{
 4136   match(Set dst (ConvI2L (LoadUS mem)));
 4137 
 4138   ins_cost(125);
 4139   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4140 
 4141   ins_encode %{
 4142     __ movzwq($dst$$Register, $mem$$Address);
 4143   %}
 4144 
 4145   ins_pipe(ialu_reg_mem);
 4146 %}
 4147 
 4148 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4149 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4150   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4151 
 4152   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4153   ins_encode %{
 4154     __ movzbq($dst$$Register, $mem$$Address);
 4155   %}
 4156   ins_pipe(ialu_reg_mem);
 4157 %}
 4158 
 4159 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4160 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4161   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4162   effect(KILL cr);
 4163 
 4164   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4165             "andl    $dst, right_n_bits($mask, 16)" %}
 4166   ins_encode %{
 4167     Register Rdst = $dst$$Register;
 4168     __ movzwq(Rdst, $mem$$Address);
 4169     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4170   %}
 4171   ins_pipe(ialu_reg_mem);
 4172 %}
 4173 
 4174 // Load Integer
 4175 instruct loadI(rRegI dst, memory mem)
 4176 %{
 4177   match(Set dst (LoadI mem));
 4178 
 4179   ins_cost(125);
 4180   format %{ "movl    $dst, $mem\t# int" %}
 4181 
 4182   ins_encode %{
 4183     __ movl($dst$$Register, $mem$$Address);
 4184   %}
 4185 
 4186   ins_pipe(ialu_reg_mem);
 4187 %}
 4188 
 4189 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4190 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4191   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4192 
 4193   ins_cost(125);
 4194   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4195   ins_encode %{
 4196     __ movsbl($dst$$Register, $mem$$Address);
 4197   %}
 4198   ins_pipe(ialu_reg_mem);
 4199 %}
 4200 
 4201 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4202 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4203   match(Set dst (AndI (LoadI mem) mask));
 4204 
 4205   ins_cost(125);
 4206   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4207   ins_encode %{
 4208     __ movzbl($dst$$Register, $mem$$Address);
 4209   %}
 4210   ins_pipe(ialu_reg_mem);
 4211 %}
 4212 
 4213 // Load Integer (32 bit signed) to Short (16 bit signed)
 4214 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4215   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4216 
 4217   ins_cost(125);
 4218   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4219   ins_encode %{
 4220     __ movswl($dst$$Register, $mem$$Address);
 4221   %}
 4222   ins_pipe(ialu_reg_mem);
 4223 %}
 4224 
 4225 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4226 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4227   match(Set dst (AndI (LoadI mem) mask));
 4228 
 4229   ins_cost(125);
 4230   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4231   ins_encode %{
 4232     __ movzwl($dst$$Register, $mem$$Address);
 4233   %}
 4234   ins_pipe(ialu_reg_mem);
 4235 %}
 4236 
 4237 // Load Integer into Long Register
 4238 instruct loadI2L(rRegL dst, memory mem)
 4239 %{
 4240   match(Set dst (ConvI2L (LoadI mem)));
 4241 
 4242   ins_cost(125);
 4243   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4244 
 4245   ins_encode %{
 4246     __ movslq($dst$$Register, $mem$$Address);
 4247   %}
 4248 
 4249   ins_pipe(ialu_reg_mem);
 4250 %}
 4251 
 4252 // Load Integer with mask 0xFF into Long Register
 4253 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4254   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4255 
 4256   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4257   ins_encode %{
 4258     __ movzbq($dst$$Register, $mem$$Address);
 4259   %}
 4260   ins_pipe(ialu_reg_mem);
 4261 %}
 4262 
 4263 // Load Integer with mask 0xFFFF into Long Register
 4264 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4265   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4266 
 4267   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4268   ins_encode %{
 4269     __ movzwq($dst$$Register, $mem$$Address);
 4270   %}
 4271   ins_pipe(ialu_reg_mem);
 4272 %}
 4273 
 4274 // Load Integer with a 31-bit mask into Long Register
 4275 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4276   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4277   effect(KILL cr);
 4278 
 4279   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4280             "andl    $dst, $mask" %}
 4281   ins_encode %{
 4282     Register Rdst = $dst$$Register;
 4283     __ movl(Rdst, $mem$$Address);
 4284     __ andl(Rdst, $mask$$constant);
 4285   %}
 4286   ins_pipe(ialu_reg_mem);
 4287 %}
 4288 
 4289 // Load Unsigned Integer into Long Register
 4290 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4291 %{
 4292   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4293 
 4294   ins_cost(125);
 4295   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4296 
 4297   ins_encode %{
 4298     __ movl($dst$$Register, $mem$$Address);
 4299   %}
 4300 
 4301   ins_pipe(ialu_reg_mem);
 4302 %}
 4303 
 4304 // Load Long
 4305 instruct loadL(rRegL dst, memory mem)
 4306 %{
 4307   match(Set dst (LoadL mem));
 4308 
 4309   ins_cost(125);
 4310   format %{ "movq    $dst, $mem\t# long" %}
 4311 
 4312   ins_encode %{
 4313     __ movq($dst$$Register, $mem$$Address);
 4314   %}
 4315 
 4316   ins_pipe(ialu_reg_mem); // XXX
 4317 %}
 4318 
 4319 // Load Range
 4320 instruct loadRange(rRegI dst, memory mem)
 4321 %{
 4322   match(Set dst (LoadRange mem));
 4323 
 4324   ins_cost(125); // XXX
 4325   format %{ "movl    $dst, $mem\t# range" %}
 4326   ins_encode %{
 4327     __ movl($dst$$Register, $mem$$Address);
 4328   %}
 4329   ins_pipe(ialu_reg_mem);
 4330 %}
 4331 
 4332 // Load Pointer
 4333 instruct loadP(rRegP dst, memory mem)
 4334 %{
 4335   match(Set dst (LoadP mem));
 4336   predicate(n->as_Load()->barrier_data() == 0);
 4337 
 4338   ins_cost(125); // XXX
 4339   format %{ "movq    $dst, $mem\t# ptr" %}
 4340   ins_encode %{
 4341     __ movq($dst$$Register, $mem$$Address);
 4342   %}
 4343   ins_pipe(ialu_reg_mem); // XXX
 4344 %}
 4345 
 4346 // Load Compressed Pointer
 4347 instruct loadN(rRegN dst, memory mem)
 4348 %{
 4349    predicate(n->as_Load()->barrier_data() == 0);
 4350    match(Set dst (LoadN mem));
 4351 
 4352    ins_cost(125); // XXX
 4353    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4354    ins_encode %{
 4355      __ movl($dst$$Register, $mem$$Address);
 4356    %}
 4357    ins_pipe(ialu_reg_mem); // XXX
 4358 %}
 4359 
 4360 
 4361 // Load Klass Pointer
 4362 instruct loadKlass(rRegP dst, memory mem)
 4363 %{
 4364   match(Set dst (LoadKlass mem));
 4365 
 4366   ins_cost(125); // XXX
 4367   format %{ "movq    $dst, $mem\t# class" %}
 4368   ins_encode %{
 4369     __ movq($dst$$Register, $mem$$Address);
 4370   %}
 4371   ins_pipe(ialu_reg_mem); // XXX
 4372 %}
 4373 
 4374 // Load narrow Klass Pointer
 4375 instruct loadNKlass(rRegN dst, memory mem)
 4376 %{
 4377   predicate(!UseCompactObjectHeaders);
 4378   match(Set dst (LoadNKlass mem));
 4379 
 4380   ins_cost(125); // XXX
 4381   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4382   ins_encode %{
 4383     __ movl($dst$$Register, $mem$$Address);
 4384   %}
 4385   ins_pipe(ialu_reg_mem); // XXX
 4386 %}
 4387 
 4388 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4389 %{
 4390   predicate(UseCompactObjectHeaders);
 4391   match(Set dst (LoadNKlass mem));
 4392   effect(KILL cr);
 4393   ins_cost(125);
 4394   format %{
 4395     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4396     "shrl    $dst, markWord::klass_shift_at_offset"
 4397   %}
 4398   ins_encode %{
 4399     if (UseAPX) {
 4400       __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
 4401     }
 4402     else {
 4403       __ movl($dst$$Register, $mem$$Address);
 4404       __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4405     }
 4406   %}
 4407   ins_pipe(ialu_reg_mem);
 4408 %}
 4409 
 4410 // Load Float
 4411 instruct loadF(regF dst, memory mem)
 4412 %{
 4413   match(Set dst (LoadF mem));
 4414 
 4415   ins_cost(145); // XXX
 4416   format %{ "movss   $dst, $mem\t# float" %}
 4417   ins_encode %{
 4418     __ movflt($dst$$XMMRegister, $mem$$Address);
 4419   %}
 4420   ins_pipe(pipe_slow); // XXX
 4421 %}
 4422 
 4423 // Load Double
 4424 instruct loadD_partial(regD dst, memory mem)
 4425 %{
 4426   predicate(!UseXmmLoadAndClearUpper);
 4427   match(Set dst (LoadD mem));
 4428 
 4429   ins_cost(145); // XXX
 4430   format %{ "movlpd  $dst, $mem\t# double" %}
 4431   ins_encode %{
 4432     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4433   %}
 4434   ins_pipe(pipe_slow); // XXX
 4435 %}
 4436 
 4437 instruct loadD(regD dst, memory mem)
 4438 %{
 4439   predicate(UseXmmLoadAndClearUpper);
 4440   match(Set dst (LoadD mem));
 4441 
 4442   ins_cost(145); // XXX
 4443   format %{ "movsd   $dst, $mem\t# double" %}
 4444   ins_encode %{
 4445     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4446   %}
 4447   ins_pipe(pipe_slow); // XXX
 4448 %}
 4449 
 4450 // max = java.lang.Math.max(float a, float b)
 4451 instruct maxF_avx10_reg(regF dst, regF a, regF b) %{
 4452   predicate(VM_Version::supports_avx10_2());
 4453   match(Set dst (MaxF a b));
 4454   format %{ "maxF $dst, $a, $b" %}
 4455   ins_encode %{
 4456     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4457   %}
 4458   ins_pipe( pipe_slow );
 4459 %}
 4460 
 4461 // max = java.lang.Math.max(float a, float b)
 4462 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4463   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4464   match(Set dst (MaxF a b));
 4465   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4466   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4467   ins_encode %{
 4468     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4469   %}
 4470   ins_pipe( pipe_slow );
 4471 %}
 4472 
 4473 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4474   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4475   match(Set dst (MaxF a b));
 4476   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4477 
 4478   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4479   ins_encode %{
 4480     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4481                     false /*min*/, true /*single*/);
 4482   %}
 4483   ins_pipe( pipe_slow );
 4484 %}
 4485 
 4486 // max = java.lang.Math.max(double a, double b)
 4487 instruct maxD_avx10_reg(regD dst, regD a, regD b) %{
 4488   predicate(VM_Version::supports_avx10_2());
 4489   match(Set dst (MaxD a b));
 4490   format %{ "maxD $dst, $a, $b" %}
 4491   ins_encode %{
 4492     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4493   %}
 4494   ins_pipe( pipe_slow );
 4495 %}
 4496 
 4497 // max = java.lang.Math.max(double a, double b)
 4498 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4499   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4500   match(Set dst (MaxD a b));
 4501   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4502   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4503   ins_encode %{
 4504     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4505   %}
 4506   ins_pipe( pipe_slow );
 4507 %}
 4508 
 4509 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4510   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4511   match(Set dst (MaxD a b));
 4512   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4513 
 4514   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4515   ins_encode %{
 4516     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4517                     false /*min*/, false /*single*/);
 4518   %}
 4519   ins_pipe( pipe_slow );
 4520 %}
 4521 
 4522 // max = java.lang.Math.min(float a, float b)
 4523 instruct minF_avx10_reg(regF dst, regF a, regF b) %{
 4524   predicate(VM_Version::supports_avx10_2());
 4525   match(Set dst (MinF a b));
 4526   format %{ "minF $dst, $a, $b" %}
 4527   ins_encode %{
 4528     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4529   %}
 4530   ins_pipe( pipe_slow );
 4531 %}
 4532 
 4533 // min = java.lang.Math.min(float a, float b)
 4534 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4535   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4536   match(Set dst (MinF a b));
 4537   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4538   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4539   ins_encode %{
 4540     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4541   %}
 4542   ins_pipe( pipe_slow );
 4543 %}
 4544 
 4545 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4546   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4547   match(Set dst (MinF a b));
 4548   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4549 
 4550   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4551   ins_encode %{
 4552     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4553                     true /*min*/, true /*single*/);
 4554   %}
 4555   ins_pipe( pipe_slow );
 4556 %}
 4557 
 4558 // max = java.lang.Math.min(double a, double b)
 4559 instruct minD_avx10_reg(regD dst, regD a, regD b) %{
 4560   predicate(VM_Version::supports_avx10_2());
 4561   match(Set dst (MinD a b));
 4562   format %{ "minD $dst, $a, $b" %}
 4563   ins_encode %{
 4564     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4565   %}
 4566   ins_pipe( pipe_slow );
 4567 %}
 4568 
 4569 // min = java.lang.Math.min(double a, double b)
 4570 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4571   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4572   match(Set dst (MinD a b));
 4573   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4574     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4575   ins_encode %{
 4576     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4577   %}
 4578   ins_pipe( pipe_slow );
 4579 %}
 4580 
 4581 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4582   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4583   match(Set dst (MinD a b));
 4584   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4585 
 4586   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4587   ins_encode %{
 4588     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4589                     true /*min*/, false /*single*/);
 4590   %}
 4591   ins_pipe( pipe_slow );
 4592 %}
 4593 
 4594 // Load Effective Address
 4595 instruct leaP8(rRegP dst, indOffset8 mem)
 4596 %{
 4597   match(Set dst mem);
 4598 
 4599   ins_cost(110); // XXX
 4600   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4601   ins_encode %{
 4602     __ leaq($dst$$Register, $mem$$Address);
 4603   %}
 4604   ins_pipe(ialu_reg_reg_fat);
 4605 %}
 4606 
 4607 instruct leaP32(rRegP dst, indOffset32 mem)
 4608 %{
 4609   match(Set dst mem);
 4610 
 4611   ins_cost(110);
 4612   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4613   ins_encode %{
 4614     __ leaq($dst$$Register, $mem$$Address);
 4615   %}
 4616   ins_pipe(ialu_reg_reg_fat);
 4617 %}
 4618 
 4619 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4620 %{
 4621   match(Set dst mem);
 4622 
 4623   ins_cost(110);
 4624   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4625   ins_encode %{
 4626     __ leaq($dst$$Register, $mem$$Address);
 4627   %}
 4628   ins_pipe(ialu_reg_reg_fat);
 4629 %}
 4630 
 4631 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4632 %{
 4633   match(Set dst mem);
 4634 
 4635   ins_cost(110);
 4636   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4637   ins_encode %{
 4638     __ leaq($dst$$Register, $mem$$Address);
 4639   %}
 4640   ins_pipe(ialu_reg_reg_fat);
 4641 %}
 4642 
 4643 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4644 %{
 4645   match(Set dst mem);
 4646 
 4647   ins_cost(110);
 4648   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4649   ins_encode %{
 4650     __ leaq($dst$$Register, $mem$$Address);
 4651   %}
 4652   ins_pipe(ialu_reg_reg_fat);
 4653 %}
 4654 
 4655 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4656 %{
 4657   match(Set dst mem);
 4658 
 4659   ins_cost(110);
 4660   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4661   ins_encode %{
 4662     __ leaq($dst$$Register, $mem$$Address);
 4663   %}
 4664   ins_pipe(ialu_reg_reg_fat);
 4665 %}
 4666 
 4667 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4668 %{
 4669   match(Set dst mem);
 4670 
 4671   ins_cost(110);
 4672   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4673   ins_encode %{
 4674     __ leaq($dst$$Register, $mem$$Address);
 4675   %}
 4676   ins_pipe(ialu_reg_reg_fat);
 4677 %}
 4678 
 4679 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4680 %{
 4681   match(Set dst mem);
 4682 
 4683   ins_cost(110);
 4684   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4685   ins_encode %{
 4686     __ leaq($dst$$Register, $mem$$Address);
 4687   %}
 4688   ins_pipe(ialu_reg_reg_fat);
 4689 %}
 4690 
 4691 // Load Effective Address which uses Narrow (32-bits) oop
 4692 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4693 %{
 4694   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4695   match(Set dst mem);
 4696 
 4697   ins_cost(110);
 4698   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4699   ins_encode %{
 4700     __ leaq($dst$$Register, $mem$$Address);
 4701   %}
 4702   ins_pipe(ialu_reg_reg_fat);
 4703 %}
 4704 
 4705 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4706 %{
 4707   predicate(CompressedOops::shift() == 0);
 4708   match(Set dst mem);
 4709 
 4710   ins_cost(110); // XXX
 4711   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4712   ins_encode %{
 4713     __ leaq($dst$$Register, $mem$$Address);
 4714   %}
 4715   ins_pipe(ialu_reg_reg_fat);
 4716 %}
 4717 
 4718 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4719 %{
 4720   predicate(CompressedOops::shift() == 0);
 4721   match(Set dst mem);
 4722 
 4723   ins_cost(110);
 4724   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4725   ins_encode %{
 4726     __ leaq($dst$$Register, $mem$$Address);
 4727   %}
 4728   ins_pipe(ialu_reg_reg_fat);
 4729 %}
 4730 
 4731 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4732 %{
 4733   predicate(CompressedOops::shift() == 0);
 4734   match(Set dst mem);
 4735 
 4736   ins_cost(110);
 4737   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4738   ins_encode %{
 4739     __ leaq($dst$$Register, $mem$$Address);
 4740   %}
 4741   ins_pipe(ialu_reg_reg_fat);
 4742 %}
 4743 
 4744 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4745 %{
 4746   predicate(CompressedOops::shift() == 0);
 4747   match(Set dst mem);
 4748 
 4749   ins_cost(110);
 4750   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4751   ins_encode %{
 4752     __ leaq($dst$$Register, $mem$$Address);
 4753   %}
 4754   ins_pipe(ialu_reg_reg_fat);
 4755 %}
 4756 
 4757 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4758 %{
 4759   predicate(CompressedOops::shift() == 0);
 4760   match(Set dst mem);
 4761 
 4762   ins_cost(110);
 4763   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4764   ins_encode %{
 4765     __ leaq($dst$$Register, $mem$$Address);
 4766   %}
 4767   ins_pipe(ialu_reg_reg_fat);
 4768 %}
 4769 
 4770 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4771 %{
 4772   predicate(CompressedOops::shift() == 0);
 4773   match(Set dst mem);
 4774 
 4775   ins_cost(110);
 4776   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4777   ins_encode %{
 4778     __ leaq($dst$$Register, $mem$$Address);
 4779   %}
 4780   ins_pipe(ialu_reg_reg_fat);
 4781 %}
 4782 
 4783 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4784 %{
 4785   predicate(CompressedOops::shift() == 0);
 4786   match(Set dst mem);
 4787 
 4788   ins_cost(110);
 4789   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4790   ins_encode %{
 4791     __ leaq($dst$$Register, $mem$$Address);
 4792   %}
 4793   ins_pipe(ialu_reg_reg_fat);
 4794 %}
 4795 
 4796 instruct loadConI(rRegI dst, immI src)
 4797 %{
 4798   match(Set dst src);
 4799 
 4800   format %{ "movl    $dst, $src\t# int" %}
 4801   ins_encode %{
 4802     __ movl($dst$$Register, $src$$constant);
 4803   %}
 4804   ins_pipe(ialu_reg_fat); // XXX
 4805 %}
 4806 
 4807 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4808 %{
 4809   match(Set dst src);
 4810   effect(KILL cr);
 4811 
 4812   ins_cost(50);
 4813   format %{ "xorl    $dst, $dst\t# int" %}
 4814   ins_encode %{
 4815     __ xorl($dst$$Register, $dst$$Register);
 4816   %}
 4817   ins_pipe(ialu_reg);
 4818 %}
 4819 
 4820 instruct loadConL(rRegL dst, immL src)
 4821 %{
 4822   match(Set dst src);
 4823 
 4824   ins_cost(150);
 4825   format %{ "movq    $dst, $src\t# long" %}
 4826   ins_encode %{
 4827     __ mov64($dst$$Register, $src$$constant);
 4828   %}
 4829   ins_pipe(ialu_reg);
 4830 %}
 4831 
 4832 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4833 %{
 4834   match(Set dst src);
 4835   effect(KILL cr);
 4836 
 4837   ins_cost(50);
 4838   format %{ "xorl    $dst, $dst\t# long" %}
 4839   ins_encode %{
 4840     __ xorl($dst$$Register, $dst$$Register);
 4841   %}
 4842   ins_pipe(ialu_reg); // XXX
 4843 %}
 4844 
 4845 instruct loadConUL32(rRegL dst, immUL32 src)
 4846 %{
 4847   match(Set dst src);
 4848 
 4849   ins_cost(60);
 4850   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4851   ins_encode %{
 4852     __ movl($dst$$Register, $src$$constant);
 4853   %}
 4854   ins_pipe(ialu_reg);
 4855 %}
 4856 
 4857 instruct loadConL32(rRegL dst, immL32 src)
 4858 %{
 4859   match(Set dst src);
 4860 
 4861   ins_cost(70);
 4862   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4863   ins_encode %{
 4864     __ movq($dst$$Register, $src$$constant);
 4865   %}
 4866   ins_pipe(ialu_reg);
 4867 %}
 4868 
 4869 instruct loadConP(rRegP dst, immP con) %{
 4870   match(Set dst con);
 4871 
 4872   format %{ "movq    $dst, $con\t# ptr" %}
 4873   ins_encode %{
 4874     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4875   %}
 4876   ins_pipe(ialu_reg_fat); // XXX
 4877 %}
 4878 
 4879 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4880 %{
 4881   match(Set dst src);
 4882   effect(KILL cr);
 4883 
 4884   ins_cost(50);
 4885   format %{ "xorl    $dst, $dst\t# ptr" %}
 4886   ins_encode %{
 4887     __ xorl($dst$$Register, $dst$$Register);
 4888   %}
 4889   ins_pipe(ialu_reg);
 4890 %}
 4891 
 4892 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4893 %{
 4894   match(Set dst src);
 4895   effect(KILL cr);
 4896 
 4897   ins_cost(60);
 4898   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4899   ins_encode %{
 4900     __ movl($dst$$Register, $src$$constant);
 4901   %}
 4902   ins_pipe(ialu_reg);
 4903 %}
 4904 
 4905 instruct loadConF(regF dst, immF con) %{
 4906   match(Set dst con);
 4907   ins_cost(125);
 4908   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4909   ins_encode %{
 4910     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4911   %}
 4912   ins_pipe(pipe_slow);
 4913 %}
 4914 
 4915 instruct loadConH(regF dst, immH con) %{
 4916   match(Set dst con);
 4917   ins_cost(125);
 4918   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4919   ins_encode %{
 4920     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4921   %}
 4922   ins_pipe(pipe_slow);
 4923 %}
 4924 
 4925 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4926   match(Set dst src);
 4927   effect(KILL cr);
 4928   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4929   ins_encode %{
 4930     __ xorq($dst$$Register, $dst$$Register);
 4931   %}
 4932   ins_pipe(ialu_reg);
 4933 %}
 4934 
 4935 instruct loadConN(rRegN dst, immN src) %{
 4936   match(Set dst src);
 4937 
 4938   ins_cost(125);
 4939   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4940   ins_encode %{
 4941     address con = (address)$src$$constant;
 4942     if (con == nullptr) {
 4943       ShouldNotReachHere();
 4944     } else {
 4945       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4946     }
 4947   %}
 4948   ins_pipe(ialu_reg_fat); // XXX
 4949 %}
 4950 
 4951 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4952   match(Set dst src);
 4953 
 4954   ins_cost(125);
 4955   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4956   ins_encode %{
 4957     address con = (address)$src$$constant;
 4958     if (con == nullptr) {
 4959       ShouldNotReachHere();
 4960     } else {
 4961       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4962     }
 4963   %}
 4964   ins_pipe(ialu_reg_fat); // XXX
 4965 %}
 4966 
 4967 instruct loadConF0(regF dst, immF0 src)
 4968 %{
 4969   match(Set dst src);
 4970   ins_cost(100);
 4971 
 4972   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4973   ins_encode %{
 4974     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4975   %}
 4976   ins_pipe(pipe_slow);
 4977 %}
 4978 
 4979 // Use the same format since predicate() can not be used here.
 4980 instruct loadConD(regD dst, immD con) %{
 4981   match(Set dst con);
 4982   ins_cost(125);
 4983   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4984   ins_encode %{
 4985     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4986   %}
 4987   ins_pipe(pipe_slow);
 4988 %}
 4989 
 4990 instruct loadConD0(regD dst, immD0 src)
 4991 %{
 4992   match(Set dst src);
 4993   ins_cost(100);
 4994 
 4995   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 4996   ins_encode %{
 4997     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 4998   %}
 4999   ins_pipe(pipe_slow);
 5000 %}
 5001 
 5002 instruct loadSSI(rRegI dst, stackSlotI src)
 5003 %{
 5004   match(Set dst src);
 5005 
 5006   ins_cost(125);
 5007   format %{ "movl    $dst, $src\t# int stk" %}
 5008   ins_encode %{
 5009     __ movl($dst$$Register, $src$$Address);
 5010   %}
 5011   ins_pipe(ialu_reg_mem);
 5012 %}
 5013 
 5014 instruct loadSSL(rRegL dst, stackSlotL src)
 5015 %{
 5016   match(Set dst src);
 5017 
 5018   ins_cost(125);
 5019   format %{ "movq    $dst, $src\t# long stk" %}
 5020   ins_encode %{
 5021     __ movq($dst$$Register, $src$$Address);
 5022   %}
 5023   ins_pipe(ialu_reg_mem);
 5024 %}
 5025 
 5026 instruct loadSSP(rRegP dst, stackSlotP src)
 5027 %{
 5028   match(Set dst src);
 5029 
 5030   ins_cost(125);
 5031   format %{ "movq    $dst, $src\t# ptr stk" %}
 5032   ins_encode %{
 5033     __ movq($dst$$Register, $src$$Address);
 5034   %}
 5035   ins_pipe(ialu_reg_mem);
 5036 %}
 5037 
 5038 instruct loadSSF(regF dst, stackSlotF src)
 5039 %{
 5040   match(Set dst src);
 5041 
 5042   ins_cost(125);
 5043   format %{ "movss   $dst, $src\t# float stk" %}
 5044   ins_encode %{
 5045     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5046   %}
 5047   ins_pipe(pipe_slow); // XXX
 5048 %}
 5049 
 5050 // Use the same format since predicate() can not be used here.
 5051 instruct loadSSD(regD dst, stackSlotD src)
 5052 %{
 5053   match(Set dst src);
 5054 
 5055   ins_cost(125);
 5056   format %{ "movsd   $dst, $src\t# double stk" %}
 5057   ins_encode  %{
 5058     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5059   %}
 5060   ins_pipe(pipe_slow); // XXX
 5061 %}
 5062 
 5063 // Prefetch instructions for allocation.
 5064 // Must be safe to execute with invalid address (cannot fault).
 5065 
 5066 instruct prefetchAlloc( memory mem ) %{
 5067   predicate(AllocatePrefetchInstr==3);
 5068   match(PrefetchAllocation mem);
 5069   ins_cost(125);
 5070 
 5071   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5072   ins_encode %{
 5073     __ prefetchw($mem$$Address);
 5074   %}
 5075   ins_pipe(ialu_mem);
 5076 %}
 5077 
 5078 instruct prefetchAllocNTA( memory mem ) %{
 5079   predicate(AllocatePrefetchInstr==0);
 5080   match(PrefetchAllocation mem);
 5081   ins_cost(125);
 5082 
 5083   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5084   ins_encode %{
 5085     __ prefetchnta($mem$$Address);
 5086   %}
 5087   ins_pipe(ialu_mem);
 5088 %}
 5089 
 5090 instruct prefetchAllocT0( memory mem ) %{
 5091   predicate(AllocatePrefetchInstr==1);
 5092   match(PrefetchAllocation mem);
 5093   ins_cost(125);
 5094 
 5095   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5096   ins_encode %{
 5097     __ prefetcht0($mem$$Address);
 5098   %}
 5099   ins_pipe(ialu_mem);
 5100 %}
 5101 
 5102 instruct prefetchAllocT2( memory mem ) %{
 5103   predicate(AllocatePrefetchInstr==2);
 5104   match(PrefetchAllocation mem);
 5105   ins_cost(125);
 5106 
 5107   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5108   ins_encode %{
 5109     __ prefetcht2($mem$$Address);
 5110   %}
 5111   ins_pipe(ialu_mem);
 5112 %}
 5113 
 5114 //----------Store Instructions-------------------------------------------------
 5115 
 5116 // Store Byte
 5117 instruct storeB(memory mem, rRegI src)
 5118 %{
 5119   match(Set mem (StoreB mem src));
 5120 
 5121   ins_cost(125); // XXX
 5122   format %{ "movb    $mem, $src\t# byte" %}
 5123   ins_encode %{
 5124     __ movb($mem$$Address, $src$$Register);
 5125   %}
 5126   ins_pipe(ialu_mem_reg);
 5127 %}
 5128 
 5129 // Store Char/Short
 5130 instruct storeC(memory mem, rRegI src)
 5131 %{
 5132   match(Set mem (StoreC mem src));
 5133 
 5134   ins_cost(125); // XXX
 5135   format %{ "movw    $mem, $src\t# char/short" %}
 5136   ins_encode %{
 5137     __ movw($mem$$Address, $src$$Register);
 5138   %}
 5139   ins_pipe(ialu_mem_reg);
 5140 %}
 5141 
 5142 // Store Integer
 5143 instruct storeI(memory mem, rRegI src)
 5144 %{
 5145   match(Set mem (StoreI mem src));
 5146 
 5147   ins_cost(125); // XXX
 5148   format %{ "movl    $mem, $src\t# int" %}
 5149   ins_encode %{
 5150     __ movl($mem$$Address, $src$$Register);
 5151   %}
 5152   ins_pipe(ialu_mem_reg);
 5153 %}
 5154 
 5155 // Store Long
 5156 instruct storeL(memory mem, rRegL src)
 5157 %{
 5158   match(Set mem (StoreL mem src));
 5159 
 5160   ins_cost(125); // XXX
 5161   format %{ "movq    $mem, $src\t# long" %}
 5162   ins_encode %{
 5163     __ movq($mem$$Address, $src$$Register);
 5164   %}
 5165   ins_pipe(ialu_mem_reg); // XXX
 5166 %}
 5167 
 5168 // Store Pointer
 5169 instruct storeP(memory mem, any_RegP src)
 5170 %{
 5171   predicate(n->as_Store()->barrier_data() == 0);
 5172   match(Set mem (StoreP mem src));
 5173 
 5174   ins_cost(125); // XXX
 5175   format %{ "movq    $mem, $src\t# ptr" %}
 5176   ins_encode %{
 5177     __ movq($mem$$Address, $src$$Register);
 5178   %}
 5179   ins_pipe(ialu_mem_reg);
 5180 %}
 5181 
 5182 instruct storeImmP0(memory mem, immP0 zero)
 5183 %{
 5184   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5185   match(Set mem (StoreP mem zero));
 5186 
 5187   ins_cost(125); // XXX
 5188   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5189   ins_encode %{
 5190     __ movq($mem$$Address, r12);
 5191   %}
 5192   ins_pipe(ialu_mem_reg);
 5193 %}
 5194 
 5195 // Store Null Pointer, mark word, or other simple pointer constant.
 5196 instruct storeImmP(memory mem, immP31 src)
 5197 %{
 5198   predicate(n->as_Store()->barrier_data() == 0);
 5199   match(Set mem (StoreP mem src));
 5200 
 5201   ins_cost(150); // XXX
 5202   format %{ "movq    $mem, $src\t# ptr" %}
 5203   ins_encode %{
 5204     __ movq($mem$$Address, $src$$constant);
 5205   %}
 5206   ins_pipe(ialu_mem_imm);
 5207 %}
 5208 
 5209 // Store Compressed Pointer
 5210 instruct storeN(memory mem, rRegN src)
 5211 %{
 5212   predicate(n->as_Store()->barrier_data() == 0);
 5213   match(Set mem (StoreN mem src));
 5214 
 5215   ins_cost(125); // XXX
 5216   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5217   ins_encode %{
 5218     __ movl($mem$$Address, $src$$Register);
 5219   %}
 5220   ins_pipe(ialu_mem_reg);
 5221 %}
 5222 
 5223 instruct storeNKlass(memory mem, rRegN src)
 5224 %{
 5225   match(Set mem (StoreNKlass mem src));
 5226 
 5227   ins_cost(125); // XXX
 5228   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5229   ins_encode %{
 5230     __ movl($mem$$Address, $src$$Register);
 5231   %}
 5232   ins_pipe(ialu_mem_reg);
 5233 %}
 5234 
 5235 instruct storeImmN0(memory mem, immN0 zero)
 5236 %{
 5237   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5238   match(Set mem (StoreN mem zero));
 5239 
 5240   ins_cost(125); // XXX
 5241   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5242   ins_encode %{
 5243     __ movl($mem$$Address, r12);
 5244   %}
 5245   ins_pipe(ialu_mem_reg);
 5246 %}
 5247 
 5248 instruct storeImmN(memory mem, immN src)
 5249 %{
 5250   predicate(n->as_Store()->barrier_data() == 0);
 5251   match(Set mem (StoreN mem src));
 5252 
 5253   ins_cost(150); // XXX
 5254   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5255   ins_encode %{
 5256     address con = (address)$src$$constant;
 5257     if (con == nullptr) {
 5258       __ movl($mem$$Address, 0);
 5259     } else {
 5260       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5261     }
 5262   %}
 5263   ins_pipe(ialu_mem_imm);
 5264 %}
 5265 
 5266 instruct storeImmNKlass(memory mem, immNKlass src)
 5267 %{
 5268   match(Set mem (StoreNKlass mem src));
 5269 
 5270   ins_cost(150); // XXX
 5271   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5272   ins_encode %{
 5273     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5274   %}
 5275   ins_pipe(ialu_mem_imm);
 5276 %}
 5277 
 5278 // Store Integer Immediate
 5279 instruct storeImmI0(memory mem, immI_0 zero)
 5280 %{
 5281   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5282   match(Set mem (StoreI mem zero));
 5283 
 5284   ins_cost(125); // XXX
 5285   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5286   ins_encode %{
 5287     __ movl($mem$$Address, r12);
 5288   %}
 5289   ins_pipe(ialu_mem_reg);
 5290 %}
 5291 
 5292 instruct storeImmI(memory mem, immI src)
 5293 %{
 5294   match(Set mem (StoreI mem src));
 5295 
 5296   ins_cost(150);
 5297   format %{ "movl    $mem, $src\t# int" %}
 5298   ins_encode %{
 5299     __ movl($mem$$Address, $src$$constant);
 5300   %}
 5301   ins_pipe(ialu_mem_imm);
 5302 %}
 5303 
 5304 // Store Long Immediate
 5305 instruct storeImmL0(memory mem, immL0 zero)
 5306 %{
 5307   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5308   match(Set mem (StoreL mem zero));
 5309 
 5310   ins_cost(125); // XXX
 5311   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5312   ins_encode %{
 5313     __ movq($mem$$Address, r12);
 5314   %}
 5315   ins_pipe(ialu_mem_reg);
 5316 %}
 5317 
 5318 instruct storeImmL(memory mem, immL32 src)
 5319 %{
 5320   match(Set mem (StoreL mem src));
 5321 
 5322   ins_cost(150);
 5323   format %{ "movq    $mem, $src\t# long" %}
 5324   ins_encode %{
 5325     __ movq($mem$$Address, $src$$constant);
 5326   %}
 5327   ins_pipe(ialu_mem_imm);
 5328 %}
 5329 
 5330 // Store Short/Char Immediate
 5331 instruct storeImmC0(memory mem, immI_0 zero)
 5332 %{
 5333   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5334   match(Set mem (StoreC mem zero));
 5335 
 5336   ins_cost(125); // XXX
 5337   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5338   ins_encode %{
 5339     __ movw($mem$$Address, r12);
 5340   %}
 5341   ins_pipe(ialu_mem_reg);
 5342 %}
 5343 
 5344 instruct storeImmI16(memory mem, immI16 src)
 5345 %{
 5346   predicate(UseStoreImmI16);
 5347   match(Set mem (StoreC mem src));
 5348 
 5349   ins_cost(150);
 5350   format %{ "movw    $mem, $src\t# short/char" %}
 5351   ins_encode %{
 5352     __ movw($mem$$Address, $src$$constant);
 5353   %}
 5354   ins_pipe(ialu_mem_imm);
 5355 %}
 5356 
 5357 // Store Byte Immediate
 5358 instruct storeImmB0(memory mem, immI_0 zero)
 5359 %{
 5360   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5361   match(Set mem (StoreB mem zero));
 5362 
 5363   ins_cost(125); // XXX
 5364   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5365   ins_encode %{
 5366     __ movb($mem$$Address, r12);
 5367   %}
 5368   ins_pipe(ialu_mem_reg);
 5369 %}
 5370 
 5371 instruct storeImmB(memory mem, immI8 src)
 5372 %{
 5373   match(Set mem (StoreB mem src));
 5374 
 5375   ins_cost(150); // XXX
 5376   format %{ "movb    $mem, $src\t# byte" %}
 5377   ins_encode %{
 5378     __ movb($mem$$Address, $src$$constant);
 5379   %}
 5380   ins_pipe(ialu_mem_imm);
 5381 %}
 5382 
 5383 // Store Float
 5384 instruct storeF(memory mem, regF src)
 5385 %{
 5386   match(Set mem (StoreF mem src));
 5387 
 5388   ins_cost(95); // XXX
 5389   format %{ "movss   $mem, $src\t# float" %}
 5390   ins_encode %{
 5391     __ movflt($mem$$Address, $src$$XMMRegister);
 5392   %}
 5393   ins_pipe(pipe_slow); // XXX
 5394 %}
 5395 
 5396 // Store immediate Float value (it is faster than store from XMM register)
 5397 instruct storeF0(memory mem, immF0 zero)
 5398 %{
 5399   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5400   match(Set mem (StoreF mem zero));
 5401 
 5402   ins_cost(25); // XXX
 5403   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5404   ins_encode %{
 5405     __ movl($mem$$Address, r12);
 5406   %}
 5407   ins_pipe(ialu_mem_reg);
 5408 %}
 5409 
 5410 instruct storeF_imm(memory mem, immF src)
 5411 %{
 5412   match(Set mem (StoreF mem src));
 5413 
 5414   ins_cost(50);
 5415   format %{ "movl    $mem, $src\t# float" %}
 5416   ins_encode %{
 5417     __ movl($mem$$Address, jint_cast($src$$constant));
 5418   %}
 5419   ins_pipe(ialu_mem_imm);
 5420 %}
 5421 
 5422 // Store Double
 5423 instruct storeD(memory mem, regD src)
 5424 %{
 5425   match(Set mem (StoreD mem src));
 5426 
 5427   ins_cost(95); // XXX
 5428   format %{ "movsd   $mem, $src\t# double" %}
 5429   ins_encode %{
 5430     __ movdbl($mem$$Address, $src$$XMMRegister);
 5431   %}
 5432   ins_pipe(pipe_slow); // XXX
 5433 %}
 5434 
 5435 // Store immediate double 0.0 (it is faster than store from XMM register)
 5436 instruct storeD0_imm(memory mem, immD0 src)
 5437 %{
 5438   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5439   match(Set mem (StoreD mem src));
 5440 
 5441   ins_cost(50);
 5442   format %{ "movq    $mem, $src\t# double 0." %}
 5443   ins_encode %{
 5444     __ movq($mem$$Address, $src$$constant);
 5445   %}
 5446   ins_pipe(ialu_mem_imm);
 5447 %}
 5448 
 5449 instruct storeD0(memory mem, immD0 zero)
 5450 %{
 5451   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5452   match(Set mem (StoreD mem zero));
 5453 
 5454   ins_cost(25); // XXX
 5455   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5456   ins_encode %{
 5457     __ movq($mem$$Address, r12);
 5458   %}
 5459   ins_pipe(ialu_mem_reg);
 5460 %}
 5461 
 5462 instruct storeSSI(stackSlotI dst, rRegI src)
 5463 %{
 5464   match(Set dst src);
 5465 
 5466   ins_cost(100);
 5467   format %{ "movl    $dst, $src\t# int stk" %}
 5468   ins_encode %{
 5469     __ movl($dst$$Address, $src$$Register);
 5470   %}
 5471   ins_pipe( ialu_mem_reg );
 5472 %}
 5473 
 5474 instruct storeSSL(stackSlotL dst, rRegL src)
 5475 %{
 5476   match(Set dst src);
 5477 
 5478   ins_cost(100);
 5479   format %{ "movq    $dst, $src\t# long stk" %}
 5480   ins_encode %{
 5481     __ movq($dst$$Address, $src$$Register);
 5482   %}
 5483   ins_pipe(ialu_mem_reg);
 5484 %}
 5485 
 5486 instruct storeSSP(stackSlotP dst, rRegP src)
 5487 %{
 5488   match(Set dst src);
 5489 
 5490   ins_cost(100);
 5491   format %{ "movq    $dst, $src\t# ptr stk" %}
 5492   ins_encode %{
 5493     __ movq($dst$$Address, $src$$Register);
 5494   %}
 5495   ins_pipe(ialu_mem_reg);
 5496 %}
 5497 
 5498 instruct storeSSF(stackSlotF dst, regF src)
 5499 %{
 5500   match(Set dst src);
 5501 
 5502   ins_cost(95); // XXX
 5503   format %{ "movss   $dst, $src\t# float stk" %}
 5504   ins_encode %{
 5505     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5506   %}
 5507   ins_pipe(pipe_slow); // XXX
 5508 %}
 5509 
 5510 instruct storeSSD(stackSlotD dst, regD src)
 5511 %{
 5512   match(Set dst src);
 5513 
 5514   ins_cost(95); // XXX
 5515   format %{ "movsd   $dst, $src\t# double stk" %}
 5516   ins_encode %{
 5517     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5518   %}
 5519   ins_pipe(pipe_slow); // XXX
 5520 %}
 5521 
 5522 instruct cacheWB(indirect addr)
 5523 %{
 5524   predicate(VM_Version::supports_data_cache_line_flush());
 5525   match(CacheWB addr);
 5526 
 5527   ins_cost(100);
 5528   format %{"cache wb $addr" %}
 5529   ins_encode %{
 5530     assert($addr->index_position() < 0, "should be");
 5531     assert($addr$$disp == 0, "should be");
 5532     __ cache_wb(Address($addr$$base$$Register, 0));
 5533   %}
 5534   ins_pipe(pipe_slow); // XXX
 5535 %}
 5536 
 5537 instruct cacheWBPreSync()
 5538 %{
 5539   predicate(VM_Version::supports_data_cache_line_flush());
 5540   match(CacheWBPreSync);
 5541 
 5542   ins_cost(100);
 5543   format %{"cache wb presync" %}
 5544   ins_encode %{
 5545     __ cache_wbsync(true);
 5546   %}
 5547   ins_pipe(pipe_slow); // XXX
 5548 %}
 5549 
 5550 instruct cacheWBPostSync()
 5551 %{
 5552   predicate(VM_Version::supports_data_cache_line_flush());
 5553   match(CacheWBPostSync);
 5554 
 5555   ins_cost(100);
 5556   format %{"cache wb postsync" %}
 5557   ins_encode %{
 5558     __ cache_wbsync(false);
 5559   %}
 5560   ins_pipe(pipe_slow); // XXX
 5561 %}
 5562 
 5563 //----------BSWAP Instructions-------------------------------------------------
 5564 instruct bytes_reverse_int(rRegI dst) %{
 5565   match(Set dst (ReverseBytesI dst));
 5566 
 5567   format %{ "bswapl  $dst" %}
 5568   ins_encode %{
 5569     __ bswapl($dst$$Register);
 5570   %}
 5571   ins_pipe( ialu_reg );
 5572 %}
 5573 
 5574 instruct bytes_reverse_long(rRegL dst) %{
 5575   match(Set dst (ReverseBytesL dst));
 5576 
 5577   format %{ "bswapq  $dst" %}
 5578   ins_encode %{
 5579     __ bswapq($dst$$Register);
 5580   %}
 5581   ins_pipe( ialu_reg);
 5582 %}
 5583 
 5584 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5585   match(Set dst (ReverseBytesUS dst));
 5586   effect(KILL cr);
 5587 
 5588   format %{ "bswapl  $dst\n\t"
 5589             "shrl    $dst,16\n\t" %}
 5590   ins_encode %{
 5591     __ bswapl($dst$$Register);
 5592     __ shrl($dst$$Register, 16);
 5593   %}
 5594   ins_pipe( ialu_reg );
 5595 %}
 5596 
 5597 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5598   match(Set dst (ReverseBytesS dst));
 5599   effect(KILL cr);
 5600 
 5601   format %{ "bswapl  $dst\n\t"
 5602             "sar     $dst,16\n\t" %}
 5603   ins_encode %{
 5604     __ bswapl($dst$$Register);
 5605     __ sarl($dst$$Register, 16);
 5606   %}
 5607   ins_pipe( ialu_reg );
 5608 %}
 5609 
 5610 //---------- Zeros Count Instructions ------------------------------------------
 5611 
 5612 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5613   predicate(UseCountLeadingZerosInstruction);
 5614   match(Set dst (CountLeadingZerosI src));
 5615   effect(KILL cr);
 5616 
 5617   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5618   ins_encode %{
 5619     __ lzcntl($dst$$Register, $src$$Register);
 5620   %}
 5621   ins_pipe(ialu_reg);
 5622 %}
 5623 
 5624 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5625   predicate(UseCountLeadingZerosInstruction);
 5626   match(Set dst (CountLeadingZerosI (LoadI src)));
 5627   effect(KILL cr);
 5628   ins_cost(175);
 5629   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5630   ins_encode %{
 5631     __ lzcntl($dst$$Register, $src$$Address);
 5632   %}
 5633   ins_pipe(ialu_reg_mem);
 5634 %}
 5635 
 5636 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5637   predicate(!UseCountLeadingZerosInstruction);
 5638   match(Set dst (CountLeadingZerosI src));
 5639   effect(KILL cr);
 5640 
 5641   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5642             "jnz     skip\n\t"
 5643             "movl    $dst, -1\n"
 5644       "skip:\n\t"
 5645             "negl    $dst\n\t"
 5646             "addl    $dst, 31" %}
 5647   ins_encode %{
 5648     Register Rdst = $dst$$Register;
 5649     Register Rsrc = $src$$Register;
 5650     Label skip;
 5651     __ bsrl(Rdst, Rsrc);
 5652     __ jccb(Assembler::notZero, skip);
 5653     __ movl(Rdst, -1);
 5654     __ bind(skip);
 5655     __ negl(Rdst);
 5656     __ addl(Rdst, BitsPerInt - 1);
 5657   %}
 5658   ins_pipe(ialu_reg);
 5659 %}
 5660 
 5661 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5662   predicate(UseCountLeadingZerosInstruction);
 5663   match(Set dst (CountLeadingZerosL src));
 5664   effect(KILL cr);
 5665 
 5666   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5667   ins_encode %{
 5668     __ lzcntq($dst$$Register, $src$$Register);
 5669   %}
 5670   ins_pipe(ialu_reg);
 5671 %}
 5672 
 5673 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5674   predicate(UseCountLeadingZerosInstruction);
 5675   match(Set dst (CountLeadingZerosL (LoadL src)));
 5676   effect(KILL cr);
 5677   ins_cost(175);
 5678   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5679   ins_encode %{
 5680     __ lzcntq($dst$$Register, $src$$Address);
 5681   %}
 5682   ins_pipe(ialu_reg_mem);
 5683 %}
 5684 
 5685 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5686   predicate(!UseCountLeadingZerosInstruction);
 5687   match(Set dst (CountLeadingZerosL src));
 5688   effect(KILL cr);
 5689 
 5690   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5691             "jnz     skip\n\t"
 5692             "movl    $dst, -1\n"
 5693       "skip:\n\t"
 5694             "negl    $dst\n\t"
 5695             "addl    $dst, 63" %}
 5696   ins_encode %{
 5697     Register Rdst = $dst$$Register;
 5698     Register Rsrc = $src$$Register;
 5699     Label skip;
 5700     __ bsrq(Rdst, Rsrc);
 5701     __ jccb(Assembler::notZero, skip);
 5702     __ movl(Rdst, -1);
 5703     __ bind(skip);
 5704     __ negl(Rdst);
 5705     __ addl(Rdst, BitsPerLong - 1);
 5706   %}
 5707   ins_pipe(ialu_reg);
 5708 %}
 5709 
 5710 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5711   predicate(UseCountTrailingZerosInstruction);
 5712   match(Set dst (CountTrailingZerosI src));
 5713   effect(KILL cr);
 5714 
 5715   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5716   ins_encode %{
 5717     __ tzcntl($dst$$Register, $src$$Register);
 5718   %}
 5719   ins_pipe(ialu_reg);
 5720 %}
 5721 
 5722 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5723   predicate(UseCountTrailingZerosInstruction);
 5724   match(Set dst (CountTrailingZerosI (LoadI src)));
 5725   effect(KILL cr);
 5726   ins_cost(175);
 5727   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5728   ins_encode %{
 5729     __ tzcntl($dst$$Register, $src$$Address);
 5730   %}
 5731   ins_pipe(ialu_reg_mem);
 5732 %}
 5733 
 5734 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5735   predicate(!UseCountTrailingZerosInstruction);
 5736   match(Set dst (CountTrailingZerosI src));
 5737   effect(KILL cr);
 5738 
 5739   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5740             "jnz     done\n\t"
 5741             "movl    $dst, 32\n"
 5742       "done:" %}
 5743   ins_encode %{
 5744     Register Rdst = $dst$$Register;
 5745     Label done;
 5746     __ bsfl(Rdst, $src$$Register);
 5747     __ jccb(Assembler::notZero, done);
 5748     __ movl(Rdst, BitsPerInt);
 5749     __ bind(done);
 5750   %}
 5751   ins_pipe(ialu_reg);
 5752 %}
 5753 
 5754 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5755   predicate(UseCountTrailingZerosInstruction);
 5756   match(Set dst (CountTrailingZerosL src));
 5757   effect(KILL cr);
 5758 
 5759   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5760   ins_encode %{
 5761     __ tzcntq($dst$$Register, $src$$Register);
 5762   %}
 5763   ins_pipe(ialu_reg);
 5764 %}
 5765 
 5766 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5767   predicate(UseCountTrailingZerosInstruction);
 5768   match(Set dst (CountTrailingZerosL (LoadL src)));
 5769   effect(KILL cr);
 5770   ins_cost(175);
 5771   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5772   ins_encode %{
 5773     __ tzcntq($dst$$Register, $src$$Address);
 5774   %}
 5775   ins_pipe(ialu_reg_mem);
 5776 %}
 5777 
 5778 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5779   predicate(!UseCountTrailingZerosInstruction);
 5780   match(Set dst (CountTrailingZerosL src));
 5781   effect(KILL cr);
 5782 
 5783   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5784             "jnz     done\n\t"
 5785             "movl    $dst, 64\n"
 5786       "done:" %}
 5787   ins_encode %{
 5788     Register Rdst = $dst$$Register;
 5789     Label done;
 5790     __ bsfq(Rdst, $src$$Register);
 5791     __ jccb(Assembler::notZero, done);
 5792     __ movl(Rdst, BitsPerLong);
 5793     __ bind(done);
 5794   %}
 5795   ins_pipe(ialu_reg);
 5796 %}
 5797 
 5798 //--------------- Reverse Operation Instructions ----------------
 5799 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5800   predicate(!VM_Version::supports_gfni());
 5801   match(Set dst (ReverseI src));
 5802   effect(TEMP dst, TEMP rtmp, KILL cr);
 5803   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5804   ins_encode %{
 5805     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5806   %}
 5807   ins_pipe( ialu_reg );
 5808 %}
 5809 
 5810 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5811   predicate(VM_Version::supports_gfni());
 5812   match(Set dst (ReverseI src));
 5813   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5814   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5815   ins_encode %{
 5816     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5817   %}
 5818   ins_pipe( ialu_reg );
 5819 %}
 5820 
 5821 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5822   predicate(!VM_Version::supports_gfni());
 5823   match(Set dst (ReverseL src));
 5824   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5825   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5826   ins_encode %{
 5827     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5828   %}
 5829   ins_pipe( ialu_reg );
 5830 %}
 5831 
 5832 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5833   predicate(VM_Version::supports_gfni());
 5834   match(Set dst (ReverseL src));
 5835   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5836   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5837   ins_encode %{
 5838     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5839   %}
 5840   ins_pipe( ialu_reg );
 5841 %}
 5842 
 5843 //---------- Population Count Instructions -------------------------------------
 5844 
 5845 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5846   predicate(UsePopCountInstruction);
 5847   match(Set dst (PopCountI src));
 5848   effect(KILL cr);
 5849 
 5850   format %{ "popcnt  $dst, $src" %}
 5851   ins_encode %{
 5852     __ popcntl($dst$$Register, $src$$Register);
 5853   %}
 5854   ins_pipe(ialu_reg);
 5855 %}
 5856 
 5857 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5858   predicate(UsePopCountInstruction);
 5859   match(Set dst (PopCountI (LoadI mem)));
 5860   effect(KILL cr);
 5861 
 5862   format %{ "popcnt  $dst, $mem" %}
 5863   ins_encode %{
 5864     __ popcntl($dst$$Register, $mem$$Address);
 5865   %}
 5866   ins_pipe(ialu_reg);
 5867 %}
 5868 
 5869 // Note: Long.bitCount(long) returns an int.
 5870 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5871   predicate(UsePopCountInstruction);
 5872   match(Set dst (PopCountL src));
 5873   effect(KILL cr);
 5874 
 5875   format %{ "popcnt  $dst, $src" %}
 5876   ins_encode %{
 5877     __ popcntq($dst$$Register, $src$$Register);
 5878   %}
 5879   ins_pipe(ialu_reg);
 5880 %}
 5881 
 5882 // Note: Long.bitCount(long) returns an int.
 5883 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5884   predicate(UsePopCountInstruction);
 5885   match(Set dst (PopCountL (LoadL mem)));
 5886   effect(KILL cr);
 5887 
 5888   format %{ "popcnt  $dst, $mem" %}
 5889   ins_encode %{
 5890     __ popcntq($dst$$Register, $mem$$Address);
 5891   %}
 5892   ins_pipe(ialu_reg);
 5893 %}
 5894 
 5895 
 5896 //----------MemBar Instructions-----------------------------------------------
 5897 // Memory barrier flavors
 5898 
 5899 instruct membar_acquire()
 5900 %{
 5901   match(MemBarAcquire);
 5902   match(LoadFence);
 5903   ins_cost(0);
 5904 
 5905   size(0);
 5906   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5907   ins_encode();
 5908   ins_pipe(empty);
 5909 %}
 5910 
 5911 instruct membar_acquire_lock()
 5912 %{
 5913   match(MemBarAcquireLock);
 5914   ins_cost(0);
 5915 
 5916   size(0);
 5917   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5918   ins_encode();
 5919   ins_pipe(empty);
 5920 %}
 5921 
 5922 instruct membar_release()
 5923 %{
 5924   match(MemBarRelease);
 5925   match(StoreFence);
 5926   ins_cost(0);
 5927 
 5928   size(0);
 5929   format %{ "MEMBAR-release ! (empty encoding)" %}
 5930   ins_encode();
 5931   ins_pipe(empty);
 5932 %}
 5933 
 5934 instruct membar_release_lock()
 5935 %{
 5936   match(MemBarReleaseLock);
 5937   ins_cost(0);
 5938 
 5939   size(0);
 5940   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5941   ins_encode();
 5942   ins_pipe(empty);
 5943 %}
 5944 
 5945 instruct membar_volatile(rFlagsReg cr) %{
 5946   match(MemBarVolatile);
 5947   effect(KILL cr);
 5948   ins_cost(400);
 5949 
 5950   format %{
 5951     $$template
 5952     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5953   %}
 5954   ins_encode %{
 5955     __ membar(Assembler::StoreLoad);
 5956   %}
 5957   ins_pipe(pipe_slow);
 5958 %}
 5959 
 5960 instruct unnecessary_membar_volatile()
 5961 %{
 5962   match(MemBarVolatile);
 5963   predicate(Matcher::post_store_load_barrier(n));
 5964   ins_cost(0);
 5965 
 5966   size(0);
 5967   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5968   ins_encode();
 5969   ins_pipe(empty);
 5970 %}
 5971 
 5972 instruct membar_storestore() %{
 5973   match(MemBarStoreStore);
 5974   match(StoreStoreFence);
 5975   ins_cost(0);
 5976 
 5977   size(0);
 5978   format %{ "MEMBAR-storestore (empty encoding)" %}
 5979   ins_encode( );
 5980   ins_pipe(empty);
 5981 %}
 5982 
 5983 //----------Move Instructions--------------------------------------------------
 5984 
 5985 instruct castX2P(rRegP dst, rRegL src)
 5986 %{
 5987   match(Set dst (CastX2P src));
 5988 
 5989   format %{ "movq    $dst, $src\t# long->ptr" %}
 5990   ins_encode %{
 5991     if ($dst$$reg != $src$$reg) {
 5992       __ movptr($dst$$Register, $src$$Register);
 5993     }
 5994   %}
 5995   ins_pipe(ialu_reg_reg); // XXX
 5996 %}
 5997 
 5998 instruct castP2X(rRegL dst, rRegP src)
 5999 %{
 6000   match(Set dst (CastP2X src));
 6001 
 6002   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6003   ins_encode %{
 6004     if ($dst$$reg != $src$$reg) {
 6005       __ movptr($dst$$Register, $src$$Register);
 6006     }
 6007   %}
 6008   ins_pipe(ialu_reg_reg); // XXX
 6009 %}
 6010 
 6011 // Convert oop into int for vectors alignment masking
 6012 instruct convP2I(rRegI dst, rRegP src)
 6013 %{
 6014   match(Set dst (ConvL2I (CastP2X src)));
 6015 
 6016   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6017   ins_encode %{
 6018     __ movl($dst$$Register, $src$$Register);
 6019   %}
 6020   ins_pipe(ialu_reg_reg); // XXX
 6021 %}
 6022 
 6023 // Convert compressed oop into int for vectors alignment masking
 6024 // in case of 32bit oops (heap < 4Gb).
 6025 instruct convN2I(rRegI dst, rRegN src)
 6026 %{
 6027   predicate(CompressedOops::shift() == 0);
 6028   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6029 
 6030   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6031   ins_encode %{
 6032     __ movl($dst$$Register, $src$$Register);
 6033   %}
 6034   ins_pipe(ialu_reg_reg); // XXX
 6035 %}
 6036 
 6037 // Convert oop pointer into compressed form
 6038 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6039   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6040   match(Set dst (EncodeP src));
 6041   effect(KILL cr);
 6042   format %{ "encode_heap_oop $dst,$src" %}
 6043   ins_encode %{
 6044     Register s = $src$$Register;
 6045     Register d = $dst$$Register;
 6046     if (s != d) {
 6047       __ movq(d, s);
 6048     }
 6049     __ encode_heap_oop(d);
 6050   %}
 6051   ins_pipe(ialu_reg_long);
 6052 %}
 6053 
 6054 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6055   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6056   match(Set dst (EncodeP src));
 6057   effect(KILL cr);
 6058   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6059   ins_encode %{
 6060     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6061   %}
 6062   ins_pipe(ialu_reg_long);
 6063 %}
 6064 
 6065 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6066   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6067             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6068   match(Set dst (DecodeN src));
 6069   effect(KILL cr);
 6070   format %{ "decode_heap_oop $dst,$src" %}
 6071   ins_encode %{
 6072     Register s = $src$$Register;
 6073     Register d = $dst$$Register;
 6074     if (s != d) {
 6075       __ movq(d, s);
 6076     }
 6077     __ decode_heap_oop(d);
 6078   %}
 6079   ins_pipe(ialu_reg_long);
 6080 %}
 6081 
 6082 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6083   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6084             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6085   match(Set dst (DecodeN src));
 6086   effect(KILL cr);
 6087   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6088   ins_encode %{
 6089     Register s = $src$$Register;
 6090     Register d = $dst$$Register;
 6091     if (s != d) {
 6092       __ decode_heap_oop_not_null(d, s);
 6093     } else {
 6094       __ decode_heap_oop_not_null(d);
 6095     }
 6096   %}
 6097   ins_pipe(ialu_reg_long);
 6098 %}
 6099 
 6100 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6101   match(Set dst (EncodePKlass src));
 6102   effect(TEMP dst, KILL cr);
 6103   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6104   ins_encode %{
 6105     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6106   %}
 6107   ins_pipe(ialu_reg_long);
 6108 %}
 6109 
 6110 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6111   match(Set dst (DecodeNKlass src));
 6112   effect(TEMP dst, KILL cr);
 6113   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6114   ins_encode %{
 6115     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6116   %}
 6117   ins_pipe(ialu_reg_long);
 6118 %}
 6119 
 6120 //----------Conditional Move---------------------------------------------------
 6121 // Jump
 6122 // dummy instruction for generating temp registers
 6123 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6124   match(Jump (LShiftL switch_val shift));
 6125   ins_cost(350);
 6126   predicate(false);
 6127   effect(TEMP dest);
 6128 
 6129   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6130             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6131   ins_encode %{
 6132     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6133     // to do that and the compiler is using that register as one it can allocate.
 6134     // So we build it all by hand.
 6135     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6136     // ArrayAddress dispatch(table, index);
 6137     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6138     __ lea($dest$$Register, $constantaddress);
 6139     __ jmp(dispatch);
 6140   %}
 6141   ins_pipe(pipe_jmp);
 6142 %}
 6143 
 6144 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6145   match(Jump (AddL (LShiftL switch_val shift) offset));
 6146   ins_cost(350);
 6147   effect(TEMP dest);
 6148 
 6149   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6150             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6151   ins_encode %{
 6152     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6153     // to do that and the compiler is using that register as one it can allocate.
 6154     // So we build it all by hand.
 6155     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6156     // ArrayAddress dispatch(table, index);
 6157     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6158     __ lea($dest$$Register, $constantaddress);
 6159     __ jmp(dispatch);
 6160   %}
 6161   ins_pipe(pipe_jmp);
 6162 %}
 6163 
 6164 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6165   match(Jump switch_val);
 6166   ins_cost(350);
 6167   effect(TEMP dest);
 6168 
 6169   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6170             "jmp     [$dest + $switch_val]\n\t" %}
 6171   ins_encode %{
 6172     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6173     // to do that and the compiler is using that register as one it can allocate.
 6174     // So we build it all by hand.
 6175     // Address index(noreg, switch_reg, Address::times_1);
 6176     // ArrayAddress dispatch(table, index);
 6177     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6178     __ lea($dest$$Register, $constantaddress);
 6179     __ jmp(dispatch);
 6180   %}
 6181   ins_pipe(pipe_jmp);
 6182 %}
 6183 
 6184 // Conditional move
 6185 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6186 %{
 6187   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6188   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6189 
 6190   ins_cost(100); // XXX
 6191   format %{ "setbn$cop $dst\t# signed, int" %}
 6192   ins_encode %{
 6193     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6194     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6195   %}
 6196   ins_pipe(ialu_reg);
 6197 %}
 6198 
 6199 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6200 %{
 6201   predicate(!UseAPX);
 6202   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6203 
 6204   ins_cost(200); // XXX
 6205   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6206   ins_encode %{
 6207     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6208   %}
 6209   ins_pipe(pipe_cmov_reg);
 6210 %}
 6211 
 6212 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6213 %{
 6214   predicate(UseAPX);
 6215   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6216 
 6217   ins_cost(200);
 6218   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6219   ins_encode %{
 6220     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6221   %}
 6222   ins_pipe(pipe_cmov_reg);
 6223 %}
 6224 
 6225 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6226 %{
 6227   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6228   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6229 
 6230   ins_cost(100); // XXX
 6231   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6232   ins_encode %{
 6233     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6234     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6235   %}
 6236   ins_pipe(ialu_reg);
 6237 %}
 6238 
 6239 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6240   predicate(!UseAPX);
 6241   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6242 
 6243   ins_cost(200); // XXX
 6244   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6245   ins_encode %{
 6246     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6247   %}
 6248   ins_pipe(pipe_cmov_reg);
 6249 %}
 6250 
 6251 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6252   predicate(UseAPX);
 6253   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6254 
 6255   ins_cost(200);
 6256   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6257   ins_encode %{
 6258     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6259   %}
 6260   ins_pipe(pipe_cmov_reg);
 6261 %}
 6262 
 6263 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6264 %{
 6265   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6266   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6267 
 6268   ins_cost(100); // XXX
 6269   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6270   ins_encode %{
 6271     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6272     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6273   %}
 6274   ins_pipe(ialu_reg);
 6275 %}
 6276 
 6277 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6278   predicate(!UseAPX);
 6279   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6280   ins_cost(200);
 6281   expand %{
 6282     cmovI_regU(cop, cr, dst, src);
 6283   %}
 6284 %}
 6285 
 6286 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6287   predicate(UseAPX);
 6288   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6289   ins_cost(200);
 6290   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6291   ins_encode %{
 6292     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6293   %}
 6294   ins_pipe(pipe_cmov_reg);
 6295 %}
 6296 
 6297 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6298   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6299   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6300 
 6301   ins_cost(200); // XXX
 6302   format %{ "cmovpl  $dst, $src\n\t"
 6303             "cmovnel $dst, $src" %}
 6304   ins_encode %{
 6305     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6306     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6307   %}
 6308   ins_pipe(pipe_cmov_reg);
 6309 %}
 6310 
 6311 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6312   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6313   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6314   effect(TEMP dst);
 6315 
 6316   ins_cost(200);
 6317   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6318             "cmovnel  $dst, $src2" %}
 6319   ins_encode %{
 6320     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6321     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6322   %}
 6323   ins_pipe(pipe_cmov_reg);
 6324 %}
 6325 
 6326 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6327 // inputs of the CMove
 6328 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6329   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6330   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6331   effect(TEMP dst);
 6332 
 6333   ins_cost(200); // XXX
 6334   format %{ "cmovpl  $dst, $src\n\t"
 6335             "cmovnel $dst, $src" %}
 6336   ins_encode %{
 6337     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6338     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6339   %}
 6340   ins_pipe(pipe_cmov_reg);
 6341 %}
 6342 
 6343 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6344 // and parity flag bit is set if any of the operand is a NaN.
 6345 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6346   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6347   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6348   effect(TEMP dst);
 6349 
 6350   ins_cost(200);
 6351   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6352             "cmovnel  $dst, $src2" %}
 6353   ins_encode %{
 6354     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6355     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6356   %}
 6357   ins_pipe(pipe_cmov_reg);
 6358 %}
 6359 
 6360 // Conditional move
 6361 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6362   predicate(!UseAPX);
 6363   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6364 
 6365   ins_cost(250); // XXX
 6366   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6367   ins_encode %{
 6368     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6369   %}
 6370   ins_pipe(pipe_cmov_mem);
 6371 %}
 6372 
 6373 // Conditional move
 6374 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6375 %{
 6376   predicate(UseAPX);
 6377   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6378 
 6379   ins_cost(250);
 6380   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6381   ins_encode %{
 6382     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6383   %}
 6384   ins_pipe(pipe_cmov_mem);
 6385 %}
 6386 
 6387 // Conditional move
 6388 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6389 %{
 6390   predicate(!UseAPX);
 6391   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6392 
 6393   ins_cost(250); // XXX
 6394   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6395   ins_encode %{
 6396     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6397   %}
 6398   ins_pipe(pipe_cmov_mem);
 6399 %}
 6400 
 6401 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6402   predicate(!UseAPX);
 6403   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6404   ins_cost(250);
 6405   expand %{
 6406     cmovI_memU(cop, cr, dst, src);
 6407   %}
 6408 %}
 6409 
 6410 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6411 %{
 6412   predicate(UseAPX);
 6413   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6414 
 6415   ins_cost(250);
 6416   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6417   ins_encode %{
 6418     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6419   %}
 6420   ins_pipe(pipe_cmov_mem);
 6421 %}
 6422 
 6423 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
 6424 %{
 6425   predicate(UseAPX);
 6426   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6427   ins_cost(250);
 6428   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6429   ins_encode %{
 6430     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6431   %}
 6432   ins_pipe(pipe_cmov_mem);
 6433 %}
 6434 
 6435 // Conditional move
 6436 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6437 %{
 6438   predicate(!UseAPX);
 6439   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6440 
 6441   ins_cost(200); // XXX
 6442   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6443   ins_encode %{
 6444     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6445   %}
 6446   ins_pipe(pipe_cmov_reg);
 6447 %}
 6448 
 6449 // Conditional move ndd
 6450 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6451 %{
 6452   predicate(UseAPX);
 6453   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6454 
 6455   ins_cost(200);
 6456   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6457   ins_encode %{
 6458     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6459   %}
 6460   ins_pipe(pipe_cmov_reg);
 6461 %}
 6462 
 6463 // Conditional move
 6464 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6465 %{
 6466   predicate(!UseAPX);
 6467   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6468 
 6469   ins_cost(200); // XXX
 6470   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6471   ins_encode %{
 6472     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6473   %}
 6474   ins_pipe(pipe_cmov_reg);
 6475 %}
 6476 
 6477 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6478   predicate(!UseAPX);
 6479   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6480   ins_cost(200);
 6481   expand %{
 6482     cmovN_regU(cop, cr, dst, src);
 6483   %}
 6484 %}
 6485 
 6486 // Conditional move ndd
 6487 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6488 %{
 6489   predicate(UseAPX);
 6490   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6491 
 6492   ins_cost(200);
 6493   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6494   ins_encode %{
 6495     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6496   %}
 6497   ins_pipe(pipe_cmov_reg);
 6498 %}
 6499 
 6500 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6501   predicate(UseAPX);
 6502   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6503   ins_cost(200);
 6504   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6505   ins_encode %{
 6506     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6507   %}
 6508   ins_pipe(pipe_cmov_reg);
 6509 %}
 6510 
 6511 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6512   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6513   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6514 
 6515   ins_cost(200); // XXX
 6516   format %{ "cmovpl  $dst, $src\n\t"
 6517             "cmovnel $dst, $src" %}
 6518   ins_encode %{
 6519     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6520     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6521   %}
 6522   ins_pipe(pipe_cmov_reg);
 6523 %}
 6524 
 6525 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6526 // inputs of the CMove
 6527 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6528   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6529   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6530 
 6531   ins_cost(200); // XXX
 6532   format %{ "cmovpl  $dst, $src\n\t"
 6533             "cmovnel $dst, $src" %}
 6534   ins_encode %{
 6535     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6536     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6537   %}
 6538   ins_pipe(pipe_cmov_reg);
 6539 %}
 6540 
 6541 // Conditional move
 6542 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6543 %{
 6544   predicate(!UseAPX);
 6545   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6546 
 6547   ins_cost(200); // XXX
 6548   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6549   ins_encode %{
 6550     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6551   %}
 6552   ins_pipe(pipe_cmov_reg);  // XXX
 6553 %}
 6554 
 6555 // Conditional move ndd
 6556 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6557 %{
 6558   predicate(UseAPX);
 6559   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6560 
 6561   ins_cost(200);
 6562   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6563   ins_encode %{
 6564     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6565   %}
 6566   ins_pipe(pipe_cmov_reg);
 6567 %}
 6568 
 6569 // Conditional move
 6570 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6571 %{
 6572   predicate(!UseAPX);
 6573   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6574 
 6575   ins_cost(200); // XXX
 6576   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6577   ins_encode %{
 6578     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6579   %}
 6580   ins_pipe(pipe_cmov_reg); // XXX
 6581 %}
 6582 
 6583 // Conditional move ndd
 6584 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6585 %{
 6586   predicate(UseAPX);
 6587   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6588 
 6589   ins_cost(200);
 6590   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6591   ins_encode %{
 6592     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6593   %}
 6594   ins_pipe(pipe_cmov_reg);
 6595 %}
 6596 
 6597 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6598   predicate(!UseAPX);
 6599   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6600   ins_cost(200);
 6601   expand %{
 6602     cmovP_regU(cop, cr, dst, src);
 6603   %}
 6604 %}
 6605 
 6606 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6607   predicate(UseAPX);
 6608   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6609   ins_cost(200);
 6610   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6611   ins_encode %{
 6612     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6613   %}
 6614   ins_pipe(pipe_cmov_reg);
 6615 %}
 6616 
 6617 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6618   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6619   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6620 
 6621   ins_cost(200); // XXX
 6622   format %{ "cmovpq  $dst, $src\n\t"
 6623             "cmovneq $dst, $src" %}
 6624   ins_encode %{
 6625     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6626     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6627   %}
 6628   ins_pipe(pipe_cmov_reg);
 6629 %}
 6630 
 6631 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6632   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6633   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6634   effect(TEMP dst);
 6635 
 6636   ins_cost(200);
 6637   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6638             "cmovneq  $dst, $src2" %}
 6639   ins_encode %{
 6640     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6641     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6642   %}
 6643   ins_pipe(pipe_cmov_reg);
 6644 %}
 6645 
 6646 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6647 // inputs of the CMove
 6648 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6649   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6650   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6651 
 6652   ins_cost(200); // XXX
 6653   format %{ "cmovpq  $dst, $src\n\t"
 6654             "cmovneq $dst, $src" %}
 6655   ins_encode %{
 6656     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6657     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6658   %}
 6659   ins_pipe(pipe_cmov_reg);
 6660 %}
 6661 
 6662 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6663   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6664   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6665   effect(TEMP dst);
 6666 
 6667   ins_cost(200);
 6668   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6669             "cmovneq  $dst, $src2" %}
 6670   ins_encode %{
 6671     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6672     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6673   %}
 6674   ins_pipe(pipe_cmov_reg);
 6675 %}
 6676 
 6677 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6678 %{
 6679   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6680   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6681 
 6682   ins_cost(100); // XXX
 6683   format %{ "setbn$cop $dst\t# signed, long" %}
 6684   ins_encode %{
 6685     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6686     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6687   %}
 6688   ins_pipe(ialu_reg);
 6689 %}
 6690 
 6691 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6692 %{
 6693   predicate(!UseAPX);
 6694   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6695 
 6696   ins_cost(200); // XXX
 6697   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6698   ins_encode %{
 6699     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6700   %}
 6701   ins_pipe(pipe_cmov_reg);  // XXX
 6702 %}
 6703 
 6704 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6705 %{
 6706   predicate(UseAPX);
 6707   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6708 
 6709   ins_cost(200);
 6710   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6711   ins_encode %{
 6712     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6713   %}
 6714   ins_pipe(pipe_cmov_reg);
 6715 %}
 6716 
 6717 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6718 %{
 6719   predicate(!UseAPX);
 6720   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6721 
 6722   ins_cost(200); // XXX
 6723   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6724   ins_encode %{
 6725     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6726   %}
 6727   ins_pipe(pipe_cmov_mem);  // XXX
 6728 %}
 6729 
 6730 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6731 %{
 6732   predicate(UseAPX);
 6733   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6734 
 6735   ins_cost(200);
 6736   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6737   ins_encode %{
 6738     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6739   %}
 6740   ins_pipe(pipe_cmov_mem);
 6741 %}
 6742 
 6743 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6744 %{
 6745   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6746   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6747 
 6748   ins_cost(100); // XXX
 6749   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6750   ins_encode %{
 6751     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6752     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6753   %}
 6754   ins_pipe(ialu_reg);
 6755 %}
 6756 
 6757 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6758 %{
 6759   predicate(!UseAPX);
 6760   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6761 
 6762   ins_cost(200); // XXX
 6763   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6764   ins_encode %{
 6765     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6766   %}
 6767   ins_pipe(pipe_cmov_reg); // XXX
 6768 %}
 6769 
 6770 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6771 %{
 6772   predicate(UseAPX);
 6773   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6774 
 6775   ins_cost(200);
 6776   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6777   ins_encode %{
 6778     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6779   %}
 6780   ins_pipe(pipe_cmov_reg);
 6781 %}
 6782 
 6783 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6784 %{
 6785   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6786   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6787 
 6788   ins_cost(100); // XXX
 6789   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6790   ins_encode %{
 6791     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6792     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6793   %}
 6794   ins_pipe(ialu_reg);
 6795 %}
 6796 
 6797 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6798   predicate(!UseAPX);
 6799   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6800   ins_cost(200);
 6801   expand %{
 6802     cmovL_regU(cop, cr, dst, src);
 6803   %}
 6804 %}
 6805 
 6806 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
 6807 %{
 6808   predicate(UseAPX);
 6809   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6810   ins_cost(200);
 6811   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6812   ins_encode %{
 6813     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6814   %}
 6815   ins_pipe(pipe_cmov_reg);
 6816 %}
 6817 
 6818 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6819   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6820   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6821 
 6822   ins_cost(200); // XXX
 6823   format %{ "cmovpq  $dst, $src\n\t"
 6824             "cmovneq $dst, $src" %}
 6825   ins_encode %{
 6826     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6827     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6828   %}
 6829   ins_pipe(pipe_cmov_reg);
 6830 %}
 6831 
 6832 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6833   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6834   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6835   effect(TEMP dst);
 6836 
 6837   ins_cost(200);
 6838   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6839             "cmovneq  $dst, $src2" %}
 6840   ins_encode %{
 6841     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6842     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6843   %}
 6844   ins_pipe(pipe_cmov_reg);
 6845 %}
 6846 
 6847 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6848 // inputs of the CMove
 6849 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6850   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6851   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6852 
 6853   ins_cost(200); // XXX
 6854   format %{ "cmovpq  $dst, $src\n\t"
 6855             "cmovneq $dst, $src" %}
 6856   ins_encode %{
 6857     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6858     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6859   %}
 6860   ins_pipe(pipe_cmov_reg);
 6861 %}
 6862 
 6863 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6864   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6865   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6866   effect(TEMP dst);
 6867 
 6868   ins_cost(200);
 6869   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6870             "cmovneq $dst, $src2" %}
 6871   ins_encode %{
 6872     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6873     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6874   %}
 6875   ins_pipe(pipe_cmov_reg);
 6876 %}
 6877 
 6878 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6879 %{
 6880   predicate(!UseAPX);
 6881   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6882 
 6883   ins_cost(200); // XXX
 6884   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6885   ins_encode %{
 6886     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6887   %}
 6888   ins_pipe(pipe_cmov_mem); // XXX
 6889 %}
 6890 
 6891 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6892   predicate(!UseAPX);
 6893   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6894   ins_cost(200);
 6895   expand %{
 6896     cmovL_memU(cop, cr, dst, src);
 6897   %}
 6898 %}
 6899 
 6900 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6901 %{
 6902   predicate(UseAPX);
 6903   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6904 
 6905   ins_cost(200);
 6906   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6907   ins_encode %{
 6908     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6909   %}
 6910   ins_pipe(pipe_cmov_mem);
 6911 %}
 6912 
 6913 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
 6914 %{
 6915   predicate(UseAPX);
 6916   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6917   ins_cost(200);
 6918   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6919   ins_encode %{
 6920     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6921   %}
 6922   ins_pipe(pipe_cmov_mem);
 6923 %}
 6924 
 6925 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6926 %{
 6927   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6928 
 6929   ins_cost(200); // XXX
 6930   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6931             "movss     $dst, $src\n"
 6932     "skip:" %}
 6933   ins_encode %{
 6934     Label Lskip;
 6935     // Invert sense of branch from sense of CMOV
 6936     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6937     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6938     __ bind(Lskip);
 6939   %}
 6940   ins_pipe(pipe_slow);
 6941 %}
 6942 
 6943 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6944 %{
 6945   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6946 
 6947   ins_cost(200); // XXX
 6948   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6949             "movss     $dst, $src\n"
 6950     "skip:" %}
 6951   ins_encode %{
 6952     Label Lskip;
 6953     // Invert sense of branch from sense of CMOV
 6954     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6955     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6956     __ bind(Lskip);
 6957   %}
 6958   ins_pipe(pipe_slow);
 6959 %}
 6960 
 6961 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6962   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6963   ins_cost(200);
 6964   expand %{
 6965     cmovF_regU(cop, cr, dst, src);
 6966   %}
 6967 %}
 6968 
 6969 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6970 %{
 6971   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6972 
 6973   ins_cost(200); // XXX
 6974   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6975             "movsd     $dst, $src\n"
 6976     "skip:" %}
 6977   ins_encode %{
 6978     Label Lskip;
 6979     // Invert sense of branch from sense of CMOV
 6980     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6981     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6982     __ bind(Lskip);
 6983   %}
 6984   ins_pipe(pipe_slow);
 6985 %}
 6986 
 6987 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6988 %{
 6989   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6990 
 6991   ins_cost(200); // XXX
 6992   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6993             "movsd     $dst, $src\n"
 6994     "skip:" %}
 6995   ins_encode %{
 6996     Label Lskip;
 6997     // Invert sense of branch from sense of CMOV
 6998     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6999     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7000     __ bind(Lskip);
 7001   %}
 7002   ins_pipe(pipe_slow);
 7003 %}
 7004 
 7005 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7006   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7007   ins_cost(200);
 7008   expand %{
 7009     cmovD_regU(cop, cr, dst, src);
 7010   %}
 7011 %}
 7012 
 7013 //----------Arithmetic Instructions--------------------------------------------
 7014 //----------Addition Instructions----------------------------------------------
 7015 
 7016 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7017 %{
 7018   predicate(!UseAPX);
 7019   match(Set dst (AddI dst src));
 7020   effect(KILL cr);
 7021   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7022   format %{ "addl    $dst, $src\t# int" %}
 7023   ins_encode %{
 7024     __ addl($dst$$Register, $src$$Register);
 7025   %}
 7026   ins_pipe(ialu_reg_reg);
 7027 %}
 7028 
 7029 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 7030 %{
 7031   predicate(UseAPX);
 7032   match(Set dst (AddI src1 src2));
 7033   effect(KILL cr);
 7034   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7035 
 7036   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7037   ins_encode %{
 7038     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 7039   %}
 7040   ins_pipe(ialu_reg_reg);
 7041 %}
 7042 
 7043 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7044 %{
 7045   predicate(!UseAPX);
 7046   match(Set dst (AddI dst src));
 7047   effect(KILL cr);
 7048   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7049 
 7050   format %{ "addl    $dst, $src\t# int" %}
 7051   ins_encode %{
 7052     __ addl($dst$$Register, $src$$constant);
 7053   %}
 7054   ins_pipe( ialu_reg );
 7055 %}
 7056 
 7057 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7058 %{
 7059   predicate(UseAPX);
 7060   match(Set dst (AddI src1 src2));
 7061   effect(KILL cr);
 7062   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7063 
 7064   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7065   ins_encode %{
 7066     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7067   %}
 7068   ins_pipe( ialu_reg );
 7069 %}
 7070 
 7071 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7072 %{
 7073   predicate(UseAPX);
 7074   match(Set dst (AddI (LoadI src1) src2));
 7075   effect(KILL cr);
 7076   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7077 
 7078   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7079   ins_encode %{
 7080     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7081   %}
 7082   ins_pipe( ialu_reg );
 7083 %}
 7084 
 7085 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7086 %{
 7087   predicate(!UseAPX);
 7088   match(Set dst (AddI dst (LoadI src)));
 7089   effect(KILL cr);
 7090   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7091 
 7092   ins_cost(150); // XXX
 7093   format %{ "addl    $dst, $src\t# int" %}
 7094   ins_encode %{
 7095     __ addl($dst$$Register, $src$$Address);
 7096   %}
 7097   ins_pipe(ialu_reg_mem);
 7098 %}
 7099 
 7100 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7101 %{
 7102   predicate(UseAPX);
 7103   match(Set dst (AddI src1 (LoadI src2)));
 7104   effect(KILL cr);
 7105   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7106 
 7107   ins_cost(150);
 7108   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7109   ins_encode %{
 7110     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7111   %}
 7112   ins_pipe(ialu_reg_mem);
 7113 %}
 7114 
 7115 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7116 %{
 7117   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7118   effect(KILL cr);
 7119   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7120 
 7121   ins_cost(150); // XXX
 7122   format %{ "addl    $dst, $src\t# int" %}
 7123   ins_encode %{
 7124     __ addl($dst$$Address, $src$$Register);
 7125   %}
 7126   ins_pipe(ialu_mem_reg);
 7127 %}
 7128 
 7129 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7130 %{
 7131   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7132   effect(KILL cr);
 7133   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7134 
 7135 
 7136   ins_cost(125); // XXX
 7137   format %{ "addl    $dst, $src\t# int" %}
 7138   ins_encode %{
 7139     __ addl($dst$$Address, $src$$constant);
 7140   %}
 7141   ins_pipe(ialu_mem_imm);
 7142 %}
 7143 
 7144 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7145 %{
 7146   predicate(!UseAPX && UseIncDec);
 7147   match(Set dst (AddI dst src));
 7148   effect(KILL cr);
 7149 
 7150   format %{ "incl    $dst\t# int" %}
 7151   ins_encode %{
 7152     __ incrementl($dst$$Register);
 7153   %}
 7154   ins_pipe(ialu_reg);
 7155 %}
 7156 
 7157 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7158 %{
 7159   predicate(UseAPX && UseIncDec);
 7160   match(Set dst (AddI src val));
 7161   effect(KILL cr);
 7162 
 7163   format %{ "eincl    $dst, $src\t# int ndd" %}
 7164   ins_encode %{
 7165     __ eincl($dst$$Register, $src$$Register, false);
 7166   %}
 7167   ins_pipe(ialu_reg);
 7168 %}
 7169 
 7170 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7171 %{
 7172   predicate(UseAPX && UseIncDec);
 7173   match(Set dst (AddI (LoadI src) val));
 7174   effect(KILL cr);
 7175 
 7176   format %{ "eincl    $dst, $src\t# int ndd" %}
 7177   ins_encode %{
 7178     __ eincl($dst$$Register, $src$$Address, false);
 7179   %}
 7180   ins_pipe(ialu_reg);
 7181 %}
 7182 
 7183 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7184 %{
 7185   predicate(UseIncDec);
 7186   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7187   effect(KILL cr);
 7188 
 7189   ins_cost(125); // XXX
 7190   format %{ "incl    $dst\t# int" %}
 7191   ins_encode %{
 7192     __ incrementl($dst$$Address);
 7193   %}
 7194   ins_pipe(ialu_mem_imm);
 7195 %}
 7196 
 7197 // XXX why does that use AddI
 7198 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7199 %{
 7200   predicate(!UseAPX && UseIncDec);
 7201   match(Set dst (AddI dst src));
 7202   effect(KILL cr);
 7203 
 7204   format %{ "decl    $dst\t# int" %}
 7205   ins_encode %{
 7206     __ decrementl($dst$$Register);
 7207   %}
 7208   ins_pipe(ialu_reg);
 7209 %}
 7210 
 7211 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7212 %{
 7213   predicate(UseAPX && UseIncDec);
 7214   match(Set dst (AddI src val));
 7215   effect(KILL cr);
 7216 
 7217   format %{ "edecl    $dst, $src\t# int ndd" %}
 7218   ins_encode %{
 7219     __ edecl($dst$$Register, $src$$Register, false);
 7220   %}
 7221   ins_pipe(ialu_reg);
 7222 %}
 7223 
 7224 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7225 %{
 7226   predicate(UseAPX && UseIncDec);
 7227   match(Set dst (AddI (LoadI src) val));
 7228   effect(KILL cr);
 7229 
 7230   format %{ "edecl    $dst, $src\t# int ndd" %}
 7231   ins_encode %{
 7232     __ edecl($dst$$Register, $src$$Address, false);
 7233   %}
 7234   ins_pipe(ialu_reg);
 7235 %}
 7236 
 7237 // XXX why does that use AddI
 7238 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7239 %{
 7240   predicate(UseIncDec);
 7241   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7242   effect(KILL cr);
 7243 
 7244   ins_cost(125); // XXX
 7245   format %{ "decl    $dst\t# int" %}
 7246   ins_encode %{
 7247     __ decrementl($dst$$Address);
 7248   %}
 7249   ins_pipe(ialu_mem_imm);
 7250 %}
 7251 
 7252 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7253 %{
 7254   predicate(VM_Version::supports_fast_2op_lea());
 7255   match(Set dst (AddI (LShiftI index scale) disp));
 7256 
 7257   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7258   ins_encode %{
 7259     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7260     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7261   %}
 7262   ins_pipe(ialu_reg_reg);
 7263 %}
 7264 
 7265 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7266 %{
 7267   predicate(VM_Version::supports_fast_3op_lea());
 7268   match(Set dst (AddI (AddI base index) disp));
 7269 
 7270   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7271   ins_encode %{
 7272     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7273   %}
 7274   ins_pipe(ialu_reg_reg);
 7275 %}
 7276 
 7277 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7278 %{
 7279   predicate(VM_Version::supports_fast_2op_lea());
 7280   match(Set dst (AddI base (LShiftI index scale)));
 7281 
 7282   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7283   ins_encode %{
 7284     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7285     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7286   %}
 7287   ins_pipe(ialu_reg_reg);
 7288 %}
 7289 
 7290 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7291 %{
 7292   predicate(VM_Version::supports_fast_3op_lea());
 7293   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7294 
 7295   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7296   ins_encode %{
 7297     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7298     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7299   %}
 7300   ins_pipe(ialu_reg_reg);
 7301 %}
 7302 
 7303 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7304 %{
 7305   predicate(!UseAPX);
 7306   match(Set dst (AddL dst src));
 7307   effect(KILL cr);
 7308   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7309 
 7310   format %{ "addq    $dst, $src\t# long" %}
 7311   ins_encode %{
 7312     __ addq($dst$$Register, $src$$Register);
 7313   %}
 7314   ins_pipe(ialu_reg_reg);
 7315 %}
 7316 
 7317 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7318 %{
 7319   predicate(UseAPX);
 7320   match(Set dst (AddL src1 src2));
 7321   effect(KILL cr);
 7322   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7323 
 7324   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7325   ins_encode %{
 7326     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7327   %}
 7328   ins_pipe(ialu_reg_reg);
 7329 %}
 7330 
 7331 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7332 %{
 7333   predicate(!UseAPX);
 7334   match(Set dst (AddL dst src));
 7335   effect(KILL cr);
 7336   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7337 
 7338   format %{ "addq    $dst, $src\t# long" %}
 7339   ins_encode %{
 7340     __ addq($dst$$Register, $src$$constant);
 7341   %}
 7342   ins_pipe( ialu_reg );
 7343 %}
 7344 
 7345 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7346 %{
 7347   predicate(UseAPX);
 7348   match(Set dst (AddL src1 src2));
 7349   effect(KILL cr);
 7350   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7351 
 7352   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7353   ins_encode %{
 7354     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7355   %}
 7356   ins_pipe( ialu_reg );
 7357 %}
 7358 
 7359 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7360 %{
 7361   predicate(UseAPX);
 7362   match(Set dst (AddL (LoadL src1) src2));
 7363   effect(KILL cr);
 7364   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7365 
 7366   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7367   ins_encode %{
 7368     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7369   %}
 7370   ins_pipe( ialu_reg );
 7371 %}
 7372 
 7373 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7374 %{
 7375   predicate(!UseAPX);
 7376   match(Set dst (AddL dst (LoadL src)));
 7377   effect(KILL cr);
 7378   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7379 
 7380   ins_cost(150); // XXX
 7381   format %{ "addq    $dst, $src\t# long" %}
 7382   ins_encode %{
 7383     __ addq($dst$$Register, $src$$Address);
 7384   %}
 7385   ins_pipe(ialu_reg_mem);
 7386 %}
 7387 
 7388 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7389 %{
 7390   predicate(UseAPX);
 7391   match(Set dst (AddL src1 (LoadL src2)));
 7392   effect(KILL cr);
 7393   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7394 
 7395   ins_cost(150);
 7396   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7397   ins_encode %{
 7398     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7399   %}
 7400   ins_pipe(ialu_reg_mem);
 7401 %}
 7402 
 7403 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7404 %{
 7405   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7406   effect(KILL cr);
 7407   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7408 
 7409   ins_cost(150); // XXX
 7410   format %{ "addq    $dst, $src\t# long" %}
 7411   ins_encode %{
 7412     __ addq($dst$$Address, $src$$Register);
 7413   %}
 7414   ins_pipe(ialu_mem_reg);
 7415 %}
 7416 
 7417 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7418 %{
 7419   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7420   effect(KILL cr);
 7421   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7422 
 7423   ins_cost(125); // XXX
 7424   format %{ "addq    $dst, $src\t# long" %}
 7425   ins_encode %{
 7426     __ addq($dst$$Address, $src$$constant);
 7427   %}
 7428   ins_pipe(ialu_mem_imm);
 7429 %}
 7430 
 7431 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7432 %{
 7433   predicate(!UseAPX && UseIncDec);
 7434   match(Set dst (AddL dst src));
 7435   effect(KILL cr);
 7436 
 7437   format %{ "incq    $dst\t# long" %}
 7438   ins_encode %{
 7439     __ incrementq($dst$$Register);
 7440   %}
 7441   ins_pipe(ialu_reg);
 7442 %}
 7443 
 7444 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7445 %{
 7446   predicate(UseAPX && UseIncDec);
 7447   match(Set dst (AddL src val));
 7448   effect(KILL cr);
 7449 
 7450   format %{ "eincq    $dst, $src\t# long ndd" %}
 7451   ins_encode %{
 7452     __ eincq($dst$$Register, $src$$Register, false);
 7453   %}
 7454   ins_pipe(ialu_reg);
 7455 %}
 7456 
 7457 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7458 %{
 7459   predicate(UseAPX && UseIncDec);
 7460   match(Set dst (AddL (LoadL src) val));
 7461   effect(KILL cr);
 7462 
 7463   format %{ "eincq    $dst, $src\t# long ndd" %}
 7464   ins_encode %{
 7465     __ eincq($dst$$Register, $src$$Address, false);
 7466   %}
 7467   ins_pipe(ialu_reg);
 7468 %}
 7469 
 7470 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7471 %{
 7472   predicate(UseIncDec);
 7473   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7474   effect(KILL cr);
 7475 
 7476   ins_cost(125); // XXX
 7477   format %{ "incq    $dst\t# long" %}
 7478   ins_encode %{
 7479     __ incrementq($dst$$Address);
 7480   %}
 7481   ins_pipe(ialu_mem_imm);
 7482 %}
 7483 
 7484 // XXX why does that use AddL
 7485 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7486 %{
 7487   predicate(!UseAPX && UseIncDec);
 7488   match(Set dst (AddL dst src));
 7489   effect(KILL cr);
 7490 
 7491   format %{ "decq    $dst\t# long" %}
 7492   ins_encode %{
 7493     __ decrementq($dst$$Register);
 7494   %}
 7495   ins_pipe(ialu_reg);
 7496 %}
 7497 
 7498 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7499 %{
 7500   predicate(UseAPX && UseIncDec);
 7501   match(Set dst (AddL src val));
 7502   effect(KILL cr);
 7503 
 7504   format %{ "edecq    $dst, $src\t# long ndd" %}
 7505   ins_encode %{
 7506     __ edecq($dst$$Register, $src$$Register, false);
 7507   %}
 7508   ins_pipe(ialu_reg);
 7509 %}
 7510 
 7511 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7512 %{
 7513   predicate(UseAPX && UseIncDec);
 7514   match(Set dst (AddL (LoadL src) val));
 7515   effect(KILL cr);
 7516 
 7517   format %{ "edecq    $dst, $src\t# long ndd" %}
 7518   ins_encode %{
 7519     __ edecq($dst$$Register, $src$$Address, false);
 7520   %}
 7521   ins_pipe(ialu_reg);
 7522 %}
 7523 
 7524 // XXX why does that use AddL
 7525 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7526 %{
 7527   predicate(UseIncDec);
 7528   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7529   effect(KILL cr);
 7530 
 7531   ins_cost(125); // XXX
 7532   format %{ "decq    $dst\t# long" %}
 7533   ins_encode %{
 7534     __ decrementq($dst$$Address);
 7535   %}
 7536   ins_pipe(ialu_mem_imm);
 7537 %}
 7538 
 7539 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7540 %{
 7541   predicate(VM_Version::supports_fast_2op_lea());
 7542   match(Set dst (AddL (LShiftL index scale) disp));
 7543 
 7544   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7545   ins_encode %{
 7546     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7547     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7548   %}
 7549   ins_pipe(ialu_reg_reg);
 7550 %}
 7551 
 7552 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7553 %{
 7554   predicate(VM_Version::supports_fast_3op_lea());
 7555   match(Set dst (AddL (AddL base index) disp));
 7556 
 7557   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7558   ins_encode %{
 7559     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7560   %}
 7561   ins_pipe(ialu_reg_reg);
 7562 %}
 7563 
 7564 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7565 %{
 7566   predicate(VM_Version::supports_fast_2op_lea());
 7567   match(Set dst (AddL base (LShiftL index scale)));
 7568 
 7569   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7570   ins_encode %{
 7571     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7572     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7573   %}
 7574   ins_pipe(ialu_reg_reg);
 7575 %}
 7576 
 7577 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7578 %{
 7579   predicate(VM_Version::supports_fast_3op_lea());
 7580   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7581 
 7582   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7583   ins_encode %{
 7584     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7585     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7586   %}
 7587   ins_pipe(ialu_reg_reg);
 7588 %}
 7589 
 7590 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7591 %{
 7592   match(Set dst (AddP dst src));
 7593   effect(KILL cr);
 7594   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7595 
 7596   format %{ "addq    $dst, $src\t# ptr" %}
 7597   ins_encode %{
 7598     __ addq($dst$$Register, $src$$Register);
 7599   %}
 7600   ins_pipe(ialu_reg_reg);
 7601 %}
 7602 
 7603 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7604 %{
 7605   match(Set dst (AddP dst src));
 7606   effect(KILL cr);
 7607   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7608 
 7609   format %{ "addq    $dst, $src\t# ptr" %}
 7610   ins_encode %{
 7611     __ addq($dst$$Register, $src$$constant);
 7612   %}
 7613   ins_pipe( ialu_reg );
 7614 %}
 7615 
 7616 // XXX addP mem ops ????
 7617 
 7618 instruct checkCastPP(rRegP dst)
 7619 %{
 7620   match(Set dst (CheckCastPP dst));
 7621 
 7622   size(0);
 7623   format %{ "# checkcastPP of $dst" %}
 7624   ins_encode(/* empty encoding */);
 7625   ins_pipe(empty);
 7626 %}
 7627 
 7628 instruct castPP(rRegP dst)
 7629 %{
 7630   match(Set dst (CastPP dst));
 7631 
 7632   size(0);
 7633   format %{ "# castPP of $dst" %}
 7634   ins_encode(/* empty encoding */);
 7635   ins_pipe(empty);
 7636 %}
 7637 
 7638 instruct castII(rRegI dst)
 7639 %{
 7640   predicate(VerifyConstraintCasts == 0);
 7641   match(Set dst (CastII dst));
 7642 
 7643   size(0);
 7644   format %{ "# castII of $dst" %}
 7645   ins_encode(/* empty encoding */);
 7646   ins_cost(0);
 7647   ins_pipe(empty);
 7648 %}
 7649 
 7650 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7651 %{
 7652   predicate(VerifyConstraintCasts > 0);
 7653   match(Set dst (CastII dst));
 7654 
 7655   effect(KILL cr);
 7656   format %{ "# cast_checked_II $dst" %}
 7657   ins_encode %{
 7658     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7659   %}
 7660   ins_pipe(pipe_slow);
 7661 %}
 7662 
 7663 instruct castLL(rRegL dst)
 7664 %{
 7665   predicate(VerifyConstraintCasts == 0);
 7666   match(Set dst (CastLL dst));
 7667 
 7668   size(0);
 7669   format %{ "# castLL of $dst" %}
 7670   ins_encode(/* empty encoding */);
 7671   ins_cost(0);
 7672   ins_pipe(empty);
 7673 %}
 7674 
 7675 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7676 %{
 7677   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7678   match(Set dst (CastLL dst));
 7679 
 7680   effect(KILL cr);
 7681   format %{ "# cast_checked_LL $dst" %}
 7682   ins_encode %{
 7683     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7684   %}
 7685   ins_pipe(pipe_slow);
 7686 %}
 7687 
 7688 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7689 %{
 7690   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7691   match(Set dst (CastLL dst));
 7692 
 7693   effect(KILL cr, TEMP tmp);
 7694   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7695   ins_encode %{
 7696     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7697   %}
 7698   ins_pipe(pipe_slow);
 7699 %}
 7700 
 7701 instruct castFF(regF dst)
 7702 %{
 7703   match(Set dst (CastFF dst));
 7704 
 7705   size(0);
 7706   format %{ "# castFF of $dst" %}
 7707   ins_encode(/* empty encoding */);
 7708   ins_cost(0);
 7709   ins_pipe(empty);
 7710 %}
 7711 
 7712 instruct castHH(regF dst)
 7713 %{
 7714   match(Set dst (CastHH dst));
 7715 
 7716   size(0);
 7717   format %{ "# castHH of $dst" %}
 7718   ins_encode(/* empty encoding */);
 7719   ins_cost(0);
 7720   ins_pipe(empty);
 7721 %}
 7722 
 7723 instruct castDD(regD dst)
 7724 %{
 7725   match(Set dst (CastDD dst));
 7726 
 7727   size(0);
 7728   format %{ "# castDD of $dst" %}
 7729   ins_encode(/* empty encoding */);
 7730   ins_cost(0);
 7731   ins_pipe(empty);
 7732 %}
 7733 
 7734 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7735 instruct compareAndSwapP(rRegI res,
 7736                          memory mem_ptr,
 7737                          rax_RegP oldval, rRegP newval,
 7738                          rFlagsReg cr)
 7739 %{
 7740   predicate(n->as_LoadStore()->barrier_data() == 0);
 7741   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7742   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7743   effect(KILL cr, KILL oldval);
 7744 
 7745   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7746             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7747             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7748   ins_encode %{
 7749     __ lock();
 7750     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7751     __ setcc(Assembler::equal, $res$$Register);
 7752   %}
 7753   ins_pipe( pipe_cmpxchg );
 7754 %}
 7755 
 7756 instruct compareAndSwapL(rRegI res,
 7757                          memory mem_ptr,
 7758                          rax_RegL oldval, rRegL newval,
 7759                          rFlagsReg cr)
 7760 %{
 7761   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7762   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7763   effect(KILL cr, KILL oldval);
 7764 
 7765   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7766             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7767             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7768   ins_encode %{
 7769     __ lock();
 7770     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7771     __ setcc(Assembler::equal, $res$$Register);
 7772   %}
 7773   ins_pipe( pipe_cmpxchg );
 7774 %}
 7775 
 7776 instruct compareAndSwapI(rRegI res,
 7777                          memory mem_ptr,
 7778                          rax_RegI oldval, rRegI newval,
 7779                          rFlagsReg cr)
 7780 %{
 7781   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7782   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7783   effect(KILL cr, KILL oldval);
 7784 
 7785   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7786             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7787             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7788   ins_encode %{
 7789     __ lock();
 7790     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7791     __ setcc(Assembler::equal, $res$$Register);
 7792   %}
 7793   ins_pipe( pipe_cmpxchg );
 7794 %}
 7795 
 7796 instruct compareAndSwapB(rRegI res,
 7797                          memory mem_ptr,
 7798                          rax_RegI oldval, rRegI newval,
 7799                          rFlagsReg cr)
 7800 %{
 7801   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7802   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7803   effect(KILL cr, KILL oldval);
 7804 
 7805   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7806             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7807             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7808   ins_encode %{
 7809     __ lock();
 7810     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7811     __ setcc(Assembler::equal, $res$$Register);
 7812   %}
 7813   ins_pipe( pipe_cmpxchg );
 7814 %}
 7815 
 7816 instruct compareAndSwapS(rRegI res,
 7817                          memory mem_ptr,
 7818                          rax_RegI oldval, rRegI newval,
 7819                          rFlagsReg cr)
 7820 %{
 7821   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7822   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7823   effect(KILL cr, KILL oldval);
 7824 
 7825   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7826             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7827             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7828   ins_encode %{
 7829     __ lock();
 7830     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7831     __ setcc(Assembler::equal, $res$$Register);
 7832   %}
 7833   ins_pipe( pipe_cmpxchg );
 7834 %}
 7835 
 7836 instruct compareAndSwapN(rRegI res,
 7837                           memory mem_ptr,
 7838                           rax_RegN oldval, rRegN newval,
 7839                           rFlagsReg cr) %{
 7840   predicate(n->as_LoadStore()->barrier_data() == 0);
 7841   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7842   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7843   effect(KILL cr, KILL oldval);
 7844 
 7845   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7846             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7847             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7848   ins_encode %{
 7849     __ lock();
 7850     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7851     __ setcc(Assembler::equal, $res$$Register);
 7852   %}
 7853   ins_pipe( pipe_cmpxchg );
 7854 %}
 7855 
 7856 instruct compareAndExchangeB(
 7857                          memory mem_ptr,
 7858                          rax_RegI oldval, rRegI newval,
 7859                          rFlagsReg cr)
 7860 %{
 7861   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7862   effect(KILL cr);
 7863 
 7864   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7865             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7866   ins_encode %{
 7867     __ lock();
 7868     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7869   %}
 7870   ins_pipe( pipe_cmpxchg );
 7871 %}
 7872 
 7873 instruct compareAndExchangeS(
 7874                          memory mem_ptr,
 7875                          rax_RegI oldval, rRegI newval,
 7876                          rFlagsReg cr)
 7877 %{
 7878   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7879   effect(KILL cr);
 7880 
 7881   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7882             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7883   ins_encode %{
 7884     __ lock();
 7885     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7886   %}
 7887   ins_pipe( pipe_cmpxchg );
 7888 %}
 7889 
 7890 instruct compareAndExchangeI(
 7891                          memory mem_ptr,
 7892                          rax_RegI oldval, rRegI newval,
 7893                          rFlagsReg cr)
 7894 %{
 7895   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7896   effect(KILL cr);
 7897 
 7898   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7899             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7900   ins_encode %{
 7901     __ lock();
 7902     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7903   %}
 7904   ins_pipe( pipe_cmpxchg );
 7905 %}
 7906 
 7907 instruct compareAndExchangeL(
 7908                          memory mem_ptr,
 7909                          rax_RegL oldval, rRegL newval,
 7910                          rFlagsReg cr)
 7911 %{
 7912   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7913   effect(KILL cr);
 7914 
 7915   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7916             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7917   ins_encode %{
 7918     __ lock();
 7919     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7920   %}
 7921   ins_pipe( pipe_cmpxchg );
 7922 %}
 7923 
 7924 instruct compareAndExchangeN(
 7925                           memory mem_ptr,
 7926                           rax_RegN oldval, rRegN newval,
 7927                           rFlagsReg cr) %{
 7928   predicate(n->as_LoadStore()->barrier_data() == 0);
 7929   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7930   effect(KILL cr);
 7931 
 7932   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7933             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7934   ins_encode %{
 7935     __ lock();
 7936     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7937   %}
 7938   ins_pipe( pipe_cmpxchg );
 7939 %}
 7940 
 7941 instruct compareAndExchangeP(
 7942                          memory mem_ptr,
 7943                          rax_RegP oldval, rRegP newval,
 7944                          rFlagsReg cr)
 7945 %{
 7946   predicate(n->as_LoadStore()->barrier_data() == 0);
 7947   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7948   effect(KILL cr);
 7949 
 7950   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7951             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7952   ins_encode %{
 7953     __ lock();
 7954     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7955   %}
 7956   ins_pipe( pipe_cmpxchg );
 7957 %}
 7958 
 7959 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7960   predicate(n->as_LoadStore()->result_not_used());
 7961   match(Set dummy (GetAndAddB mem add));
 7962   effect(KILL cr);
 7963   format %{ "addb_lock   $mem, $add" %}
 7964   ins_encode %{
 7965     __ lock();
 7966     __ addb($mem$$Address, $add$$Register);
 7967   %}
 7968   ins_pipe(pipe_cmpxchg);
 7969 %}
 7970 
 7971 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7972   predicate(n->as_LoadStore()->result_not_used());
 7973   match(Set dummy (GetAndAddB mem add));
 7974   effect(KILL cr);
 7975   format %{ "addb_lock   $mem, $add" %}
 7976   ins_encode %{
 7977     __ lock();
 7978     __ addb($mem$$Address, $add$$constant);
 7979   %}
 7980   ins_pipe(pipe_cmpxchg);
 7981 %}
 7982 
 7983 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7984   predicate(!n->as_LoadStore()->result_not_used());
 7985   match(Set newval (GetAndAddB mem newval));
 7986   effect(KILL cr);
 7987   format %{ "xaddb_lock  $mem, $newval" %}
 7988   ins_encode %{
 7989     __ lock();
 7990     __ xaddb($mem$$Address, $newval$$Register);
 7991   %}
 7992   ins_pipe(pipe_cmpxchg);
 7993 %}
 7994 
 7995 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7996   predicate(n->as_LoadStore()->result_not_used());
 7997   match(Set dummy (GetAndAddS mem add));
 7998   effect(KILL cr);
 7999   format %{ "addw_lock   $mem, $add" %}
 8000   ins_encode %{
 8001     __ lock();
 8002     __ addw($mem$$Address, $add$$Register);
 8003   %}
 8004   ins_pipe(pipe_cmpxchg);
 8005 %}
 8006 
 8007 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8008   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 8009   match(Set dummy (GetAndAddS mem add));
 8010   effect(KILL cr);
 8011   format %{ "addw_lock   $mem, $add" %}
 8012   ins_encode %{
 8013     __ lock();
 8014     __ addw($mem$$Address, $add$$constant);
 8015   %}
 8016   ins_pipe(pipe_cmpxchg);
 8017 %}
 8018 
 8019 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8020   predicate(!n->as_LoadStore()->result_not_used());
 8021   match(Set newval (GetAndAddS mem newval));
 8022   effect(KILL cr);
 8023   format %{ "xaddw_lock  $mem, $newval" %}
 8024   ins_encode %{
 8025     __ lock();
 8026     __ xaddw($mem$$Address, $newval$$Register);
 8027   %}
 8028   ins_pipe(pipe_cmpxchg);
 8029 %}
 8030 
 8031 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8032   predicate(n->as_LoadStore()->result_not_used());
 8033   match(Set dummy (GetAndAddI mem add));
 8034   effect(KILL cr);
 8035   format %{ "addl_lock   $mem, $add" %}
 8036   ins_encode %{
 8037     __ lock();
 8038     __ addl($mem$$Address, $add$$Register);
 8039   %}
 8040   ins_pipe(pipe_cmpxchg);
 8041 %}
 8042 
 8043 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8044   predicate(n->as_LoadStore()->result_not_used());
 8045   match(Set dummy (GetAndAddI mem add));
 8046   effect(KILL cr);
 8047   format %{ "addl_lock   $mem, $add" %}
 8048   ins_encode %{
 8049     __ lock();
 8050     __ addl($mem$$Address, $add$$constant);
 8051   %}
 8052   ins_pipe(pipe_cmpxchg);
 8053 %}
 8054 
 8055 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8056   predicate(!n->as_LoadStore()->result_not_used());
 8057   match(Set newval (GetAndAddI mem newval));
 8058   effect(KILL cr);
 8059   format %{ "xaddl_lock  $mem, $newval" %}
 8060   ins_encode %{
 8061     __ lock();
 8062     __ xaddl($mem$$Address, $newval$$Register);
 8063   %}
 8064   ins_pipe(pipe_cmpxchg);
 8065 %}
 8066 
 8067 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8068   predicate(n->as_LoadStore()->result_not_used());
 8069   match(Set dummy (GetAndAddL mem add));
 8070   effect(KILL cr);
 8071   format %{ "addq_lock   $mem, $add" %}
 8072   ins_encode %{
 8073     __ lock();
 8074     __ addq($mem$$Address, $add$$Register);
 8075   %}
 8076   ins_pipe(pipe_cmpxchg);
 8077 %}
 8078 
 8079 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8080   predicate(n->as_LoadStore()->result_not_used());
 8081   match(Set dummy (GetAndAddL mem add));
 8082   effect(KILL cr);
 8083   format %{ "addq_lock   $mem, $add" %}
 8084   ins_encode %{
 8085     __ lock();
 8086     __ addq($mem$$Address, $add$$constant);
 8087   %}
 8088   ins_pipe(pipe_cmpxchg);
 8089 %}
 8090 
 8091 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8092   predicate(!n->as_LoadStore()->result_not_used());
 8093   match(Set newval (GetAndAddL mem newval));
 8094   effect(KILL cr);
 8095   format %{ "xaddq_lock  $mem, $newval" %}
 8096   ins_encode %{
 8097     __ lock();
 8098     __ xaddq($mem$$Address, $newval$$Register);
 8099   %}
 8100   ins_pipe(pipe_cmpxchg);
 8101 %}
 8102 
 8103 instruct xchgB( memory mem, rRegI newval) %{
 8104   match(Set newval (GetAndSetB mem newval));
 8105   format %{ "XCHGB  $newval,[$mem]" %}
 8106   ins_encode %{
 8107     __ xchgb($newval$$Register, $mem$$Address);
 8108   %}
 8109   ins_pipe( pipe_cmpxchg );
 8110 %}
 8111 
 8112 instruct xchgS( memory mem, rRegI newval) %{
 8113   match(Set newval (GetAndSetS mem newval));
 8114   format %{ "XCHGW  $newval,[$mem]" %}
 8115   ins_encode %{
 8116     __ xchgw($newval$$Register, $mem$$Address);
 8117   %}
 8118   ins_pipe( pipe_cmpxchg );
 8119 %}
 8120 
 8121 instruct xchgI( memory mem, rRegI newval) %{
 8122   match(Set newval (GetAndSetI mem newval));
 8123   format %{ "XCHGL  $newval,[$mem]" %}
 8124   ins_encode %{
 8125     __ xchgl($newval$$Register, $mem$$Address);
 8126   %}
 8127   ins_pipe( pipe_cmpxchg );
 8128 %}
 8129 
 8130 instruct xchgL( memory mem, rRegL newval) %{
 8131   match(Set newval (GetAndSetL mem newval));
 8132   format %{ "XCHGL  $newval,[$mem]" %}
 8133   ins_encode %{
 8134     __ xchgq($newval$$Register, $mem$$Address);
 8135   %}
 8136   ins_pipe( pipe_cmpxchg );
 8137 %}
 8138 
 8139 instruct xchgP( memory mem, rRegP newval) %{
 8140   match(Set newval (GetAndSetP mem newval));
 8141   predicate(n->as_LoadStore()->barrier_data() == 0);
 8142   format %{ "XCHGQ  $newval,[$mem]" %}
 8143   ins_encode %{
 8144     __ xchgq($newval$$Register, $mem$$Address);
 8145   %}
 8146   ins_pipe( pipe_cmpxchg );
 8147 %}
 8148 
 8149 instruct xchgN( memory mem, rRegN newval) %{
 8150   predicate(n->as_LoadStore()->barrier_data() == 0);
 8151   match(Set newval (GetAndSetN mem newval));
 8152   format %{ "XCHGL  $newval,$mem]" %}
 8153   ins_encode %{
 8154     __ xchgl($newval$$Register, $mem$$Address);
 8155   %}
 8156   ins_pipe( pipe_cmpxchg );
 8157 %}
 8158 
 8159 //----------Abs Instructions-------------------------------------------
 8160 
 8161 // Integer Absolute Instructions
 8162 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8163 %{
 8164   match(Set dst (AbsI src));
 8165   effect(TEMP dst, KILL cr);
 8166   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8167             "subl    $dst, $src\n\t"
 8168             "cmovll  $dst, $src" %}
 8169   ins_encode %{
 8170     __ xorl($dst$$Register, $dst$$Register);
 8171     __ subl($dst$$Register, $src$$Register);
 8172     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8173   %}
 8174 
 8175   ins_pipe(ialu_reg_reg);
 8176 %}
 8177 
 8178 // Long Absolute Instructions
 8179 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8180 %{
 8181   match(Set dst (AbsL src));
 8182   effect(TEMP dst, KILL cr);
 8183   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8184             "subq    $dst, $src\n\t"
 8185             "cmovlq  $dst, $src" %}
 8186   ins_encode %{
 8187     __ xorl($dst$$Register, $dst$$Register);
 8188     __ subq($dst$$Register, $src$$Register);
 8189     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8190   %}
 8191 
 8192   ins_pipe(ialu_reg_reg);
 8193 %}
 8194 
 8195 //----------Subtraction Instructions-------------------------------------------
 8196 
 8197 // Integer Subtraction Instructions
 8198 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8199 %{
 8200   predicate(!UseAPX);
 8201   match(Set dst (SubI dst src));
 8202   effect(KILL cr);
 8203   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8204 
 8205   format %{ "subl    $dst, $src\t# int" %}
 8206   ins_encode %{
 8207     __ subl($dst$$Register, $src$$Register);
 8208   %}
 8209   ins_pipe(ialu_reg_reg);
 8210 %}
 8211 
 8212 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8213 %{
 8214   predicate(UseAPX);
 8215   match(Set dst (SubI src1 src2));
 8216   effect(KILL cr);
 8217   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8218 
 8219   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8220   ins_encode %{
 8221     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8222   %}
 8223   ins_pipe(ialu_reg_reg);
 8224 %}
 8225 
 8226 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8227 %{
 8228   predicate(UseAPX);
 8229   match(Set dst (SubI src1 src2));
 8230   effect(KILL cr);
 8231   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8232 
 8233   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8234   ins_encode %{
 8235     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8236   %}
 8237   ins_pipe(ialu_reg_reg);
 8238 %}
 8239 
 8240 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8241 %{
 8242   predicate(UseAPX);
 8243   match(Set dst (SubI (LoadI src1) src2));
 8244   effect(KILL cr);
 8245   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8246 
 8247   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8248   ins_encode %{
 8249     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8250   %}
 8251   ins_pipe(ialu_reg_reg);
 8252 %}
 8253 
 8254 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8255 %{
 8256   predicate(!UseAPX);
 8257   match(Set dst (SubI dst (LoadI src)));
 8258   effect(KILL cr);
 8259   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8260 
 8261   ins_cost(150);
 8262   format %{ "subl    $dst, $src\t# int" %}
 8263   ins_encode %{
 8264     __ subl($dst$$Register, $src$$Address);
 8265   %}
 8266   ins_pipe(ialu_reg_mem);
 8267 %}
 8268 
 8269 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8270 %{
 8271   predicate(UseAPX);
 8272   match(Set dst (SubI src1 (LoadI src2)));
 8273   effect(KILL cr);
 8274   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8275 
 8276   ins_cost(150);
 8277   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8278   ins_encode %{
 8279     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8280   %}
 8281   ins_pipe(ialu_reg_mem);
 8282 %}
 8283 
 8284 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8285 %{
 8286   predicate(UseAPX);
 8287   match(Set dst (SubI (LoadI src1) src2));
 8288   effect(KILL cr);
 8289   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8290 
 8291   ins_cost(150);
 8292   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8293   ins_encode %{
 8294     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8295   %}
 8296   ins_pipe(ialu_reg_mem);
 8297 %}
 8298 
 8299 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8300 %{
 8301   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8302   effect(KILL cr);
 8303   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8304 
 8305   ins_cost(150);
 8306   format %{ "subl    $dst, $src\t# int" %}
 8307   ins_encode %{
 8308     __ subl($dst$$Address, $src$$Register);
 8309   %}
 8310   ins_pipe(ialu_mem_reg);
 8311 %}
 8312 
 8313 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8314 %{
 8315   predicate(!UseAPX);
 8316   match(Set dst (SubL dst src));
 8317   effect(KILL cr);
 8318   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8319 
 8320   format %{ "subq    $dst, $src\t# long" %}
 8321   ins_encode %{
 8322     __ subq($dst$$Register, $src$$Register);
 8323   %}
 8324   ins_pipe(ialu_reg_reg);
 8325 %}
 8326 
 8327 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8328 %{
 8329   predicate(UseAPX);
 8330   match(Set dst (SubL src1 src2));
 8331   effect(KILL cr);
 8332   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8333 
 8334   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8335   ins_encode %{
 8336     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8337   %}
 8338   ins_pipe(ialu_reg_reg);
 8339 %}
 8340 
 8341 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8342 %{
 8343   predicate(UseAPX);
 8344   match(Set dst (SubL src1 src2));
 8345   effect(KILL cr);
 8346   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8347 
 8348   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8349   ins_encode %{
 8350     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8351   %}
 8352   ins_pipe(ialu_reg_reg);
 8353 %}
 8354 
 8355 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8356 %{
 8357   predicate(UseAPX);
 8358   match(Set dst (SubL (LoadL src1) src2));
 8359   effect(KILL cr);
 8360   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8361 
 8362   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8363   ins_encode %{
 8364     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8365   %}
 8366   ins_pipe(ialu_reg_reg);
 8367 %}
 8368 
 8369 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8370 %{
 8371   predicate(!UseAPX);
 8372   match(Set dst (SubL dst (LoadL src)));
 8373   effect(KILL cr);
 8374   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8375 
 8376   ins_cost(150);
 8377   format %{ "subq    $dst, $src\t# long" %}
 8378   ins_encode %{
 8379     __ subq($dst$$Register, $src$$Address);
 8380   %}
 8381   ins_pipe(ialu_reg_mem);
 8382 %}
 8383 
 8384 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8385 %{
 8386   predicate(UseAPX);
 8387   match(Set dst (SubL src1 (LoadL src2)));
 8388   effect(KILL cr);
 8389   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8390 
 8391   ins_cost(150);
 8392   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8393   ins_encode %{
 8394     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8395   %}
 8396   ins_pipe(ialu_reg_mem);
 8397 %}
 8398 
 8399 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8400 %{
 8401   predicate(UseAPX);
 8402   match(Set dst (SubL (LoadL src1) src2));
 8403   effect(KILL cr);
 8404   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8405 
 8406   ins_cost(150);
 8407   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8408   ins_encode %{
 8409     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8410   %}
 8411   ins_pipe(ialu_reg_mem);
 8412 %}
 8413 
 8414 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8415 %{
 8416   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8417   effect(KILL cr);
 8418   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8419 
 8420   ins_cost(150);
 8421   format %{ "subq    $dst, $src\t# long" %}
 8422   ins_encode %{
 8423     __ subq($dst$$Address, $src$$Register);
 8424   %}
 8425   ins_pipe(ialu_mem_reg);
 8426 %}
 8427 
 8428 // Subtract from a pointer
 8429 // XXX hmpf???
 8430 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8431 %{
 8432   match(Set dst (AddP dst (SubI zero src)));
 8433   effect(KILL cr);
 8434 
 8435   format %{ "subq    $dst, $src\t# ptr - int" %}
 8436   ins_encode %{
 8437     __ subq($dst$$Register, $src$$Register);
 8438   %}
 8439   ins_pipe(ialu_reg_reg);
 8440 %}
 8441 
 8442 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8443 %{
 8444   predicate(!UseAPX);
 8445   match(Set dst (SubI zero dst));
 8446   effect(KILL cr);
 8447   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8448 
 8449   format %{ "negl    $dst\t# int" %}
 8450   ins_encode %{
 8451     __ negl($dst$$Register);
 8452   %}
 8453   ins_pipe(ialu_reg);
 8454 %}
 8455 
 8456 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8457 %{
 8458   predicate(UseAPX);
 8459   match(Set dst (SubI zero src));
 8460   effect(KILL cr);
 8461   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8462 
 8463   format %{ "enegl    $dst, $src\t# int ndd" %}
 8464   ins_encode %{
 8465     __ enegl($dst$$Register, $src$$Register, false);
 8466   %}
 8467   ins_pipe(ialu_reg);
 8468 %}
 8469 
 8470 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8471 %{
 8472   predicate(!UseAPX);
 8473   match(Set dst (NegI dst));
 8474   effect(KILL cr);
 8475   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8476 
 8477   format %{ "negl    $dst\t# int" %}
 8478   ins_encode %{
 8479     __ negl($dst$$Register);
 8480   %}
 8481   ins_pipe(ialu_reg);
 8482 %}
 8483 
 8484 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8485 %{
 8486   predicate(UseAPX);
 8487   match(Set dst (NegI src));
 8488   effect(KILL cr);
 8489   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8490 
 8491   format %{ "enegl    $dst, $src\t# int ndd" %}
 8492   ins_encode %{
 8493     __ enegl($dst$$Register, $src$$Register, false);
 8494   %}
 8495   ins_pipe(ialu_reg);
 8496 %}
 8497 
 8498 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8499 %{
 8500   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8501   effect(KILL cr);
 8502   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8503 
 8504   format %{ "negl    $dst\t# int" %}
 8505   ins_encode %{
 8506     __ negl($dst$$Address);
 8507   %}
 8508   ins_pipe(ialu_reg);
 8509 %}
 8510 
 8511 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8512 %{
 8513   predicate(!UseAPX);
 8514   match(Set dst (SubL zero dst));
 8515   effect(KILL cr);
 8516   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8517 
 8518   format %{ "negq    $dst\t# long" %}
 8519   ins_encode %{
 8520     __ negq($dst$$Register);
 8521   %}
 8522   ins_pipe(ialu_reg);
 8523 %}
 8524 
 8525 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8526 %{
 8527   predicate(UseAPX);
 8528   match(Set dst (SubL zero src));
 8529   effect(KILL cr);
 8530   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8531 
 8532   format %{ "enegq    $dst, $src\t# long ndd" %}
 8533   ins_encode %{
 8534     __ enegq($dst$$Register, $src$$Register, false);
 8535   %}
 8536   ins_pipe(ialu_reg);
 8537 %}
 8538 
 8539 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8540 %{
 8541   predicate(!UseAPX);
 8542   match(Set dst (NegL dst));
 8543   effect(KILL cr);
 8544   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8545 
 8546   format %{ "negq    $dst\t# int" %}
 8547   ins_encode %{
 8548     __ negq($dst$$Register);
 8549   %}
 8550   ins_pipe(ialu_reg);
 8551 %}
 8552 
 8553 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8554 %{
 8555   predicate(UseAPX);
 8556   match(Set dst (NegL src));
 8557   effect(KILL cr);
 8558   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8559 
 8560   format %{ "enegq    $dst, $src\t# long ndd" %}
 8561   ins_encode %{
 8562     __ enegq($dst$$Register, $src$$Register, false);
 8563   %}
 8564   ins_pipe(ialu_reg);
 8565 %}
 8566 
 8567 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8568 %{
 8569   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8570   effect(KILL cr);
 8571   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8572 
 8573   format %{ "negq    $dst\t# long" %}
 8574   ins_encode %{
 8575     __ negq($dst$$Address);
 8576   %}
 8577   ins_pipe(ialu_reg);
 8578 %}
 8579 
 8580 //----------Multiplication/Division Instructions-------------------------------
 8581 // Integer Multiplication Instructions
 8582 // Multiply Register
 8583 
 8584 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8585 %{
 8586   predicate(!UseAPX);
 8587   match(Set dst (MulI dst src));
 8588   effect(KILL cr);
 8589 
 8590   ins_cost(300);
 8591   format %{ "imull   $dst, $src\t# int" %}
 8592   ins_encode %{
 8593     __ imull($dst$$Register, $src$$Register);
 8594   %}
 8595   ins_pipe(ialu_reg_reg_alu0);
 8596 %}
 8597 
 8598 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8599 %{
 8600   predicate(UseAPX);
 8601   match(Set dst (MulI src1 src2));
 8602   effect(KILL cr);
 8603 
 8604   ins_cost(300);
 8605   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8606   ins_encode %{
 8607     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8608   %}
 8609   ins_pipe(ialu_reg_reg_alu0);
 8610 %}
 8611 
 8612 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8613 %{
 8614   match(Set dst (MulI src imm));
 8615   effect(KILL cr);
 8616 
 8617   ins_cost(300);
 8618   format %{ "imull   $dst, $src, $imm\t# int" %}
 8619   ins_encode %{
 8620     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8621   %}
 8622   ins_pipe(ialu_reg_reg_alu0);
 8623 %}
 8624 
 8625 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8626 %{
 8627   predicate(!UseAPX);
 8628   match(Set dst (MulI dst (LoadI src)));
 8629   effect(KILL cr);
 8630 
 8631   ins_cost(350);
 8632   format %{ "imull   $dst, $src\t# int" %}
 8633   ins_encode %{
 8634     __ imull($dst$$Register, $src$$Address);
 8635   %}
 8636   ins_pipe(ialu_reg_mem_alu0);
 8637 %}
 8638 
 8639 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8640 %{
 8641   predicate(UseAPX);
 8642   match(Set dst (MulI src1 (LoadI src2)));
 8643   effect(KILL cr);
 8644 
 8645   ins_cost(350);
 8646   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8647   ins_encode %{
 8648     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8649   %}
 8650   ins_pipe(ialu_reg_mem_alu0);
 8651 %}
 8652 
 8653 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8654 %{
 8655   match(Set dst (MulI (LoadI src) imm));
 8656   effect(KILL cr);
 8657 
 8658   ins_cost(300);
 8659   format %{ "imull   $dst, $src, $imm\t# int" %}
 8660   ins_encode %{
 8661     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8662   %}
 8663   ins_pipe(ialu_reg_mem_alu0);
 8664 %}
 8665 
 8666 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8667 %{
 8668   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8669   effect(KILL cr, KILL src2);
 8670 
 8671   expand %{ mulI_rReg(dst, src1, cr);
 8672            mulI_rReg(src2, src3, cr);
 8673            addI_rReg(dst, src2, cr); %}
 8674 %}
 8675 
 8676 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8677 %{
 8678   predicate(!UseAPX);
 8679   match(Set dst (MulL dst src));
 8680   effect(KILL cr);
 8681 
 8682   ins_cost(300);
 8683   format %{ "imulq   $dst, $src\t# long" %}
 8684   ins_encode %{
 8685     __ imulq($dst$$Register, $src$$Register);
 8686   %}
 8687   ins_pipe(ialu_reg_reg_alu0);
 8688 %}
 8689 
 8690 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8691 %{
 8692   predicate(UseAPX);
 8693   match(Set dst (MulL src1 src2));
 8694   effect(KILL cr);
 8695 
 8696   ins_cost(300);
 8697   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8698   ins_encode %{
 8699     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8700   %}
 8701   ins_pipe(ialu_reg_reg_alu0);
 8702 %}
 8703 
 8704 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8705 %{
 8706   match(Set dst (MulL src imm));
 8707   effect(KILL cr);
 8708 
 8709   ins_cost(300);
 8710   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8711   ins_encode %{
 8712     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8713   %}
 8714   ins_pipe(ialu_reg_reg_alu0);
 8715 %}
 8716 
 8717 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8718 %{
 8719   predicate(!UseAPX);
 8720   match(Set dst (MulL dst (LoadL src)));
 8721   effect(KILL cr);
 8722 
 8723   ins_cost(350);
 8724   format %{ "imulq   $dst, $src\t# long" %}
 8725   ins_encode %{
 8726     __ imulq($dst$$Register, $src$$Address);
 8727   %}
 8728   ins_pipe(ialu_reg_mem_alu0);
 8729 %}
 8730 
 8731 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8732 %{
 8733   predicate(UseAPX);
 8734   match(Set dst (MulL src1 (LoadL src2)));
 8735   effect(KILL cr);
 8736 
 8737   ins_cost(350);
 8738   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8739   ins_encode %{
 8740     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8741   %}
 8742   ins_pipe(ialu_reg_mem_alu0);
 8743 %}
 8744 
 8745 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8746 %{
 8747   match(Set dst (MulL (LoadL src) imm));
 8748   effect(KILL cr);
 8749 
 8750   ins_cost(300);
 8751   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8752   ins_encode %{
 8753     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8754   %}
 8755   ins_pipe(ialu_reg_mem_alu0);
 8756 %}
 8757 
 8758 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8759 %{
 8760   match(Set dst (MulHiL src rax));
 8761   effect(USE_KILL rax, KILL cr);
 8762 
 8763   ins_cost(300);
 8764   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8765   ins_encode %{
 8766     __ imulq($src$$Register);
 8767   %}
 8768   ins_pipe(ialu_reg_reg_alu0);
 8769 %}
 8770 
 8771 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8772 %{
 8773   match(Set dst (UMulHiL src rax));
 8774   effect(USE_KILL rax, KILL cr);
 8775 
 8776   ins_cost(300);
 8777   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8778   ins_encode %{
 8779     __ mulq($src$$Register);
 8780   %}
 8781   ins_pipe(ialu_reg_reg_alu0);
 8782 %}
 8783 
 8784 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8785                    rFlagsReg cr)
 8786 %{
 8787   match(Set rax (DivI rax div));
 8788   effect(KILL rdx, KILL cr);
 8789 
 8790   ins_cost(30*100+10*100); // XXX
 8791   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8792             "jne,s   normal\n\t"
 8793             "xorl    rdx, rdx\n\t"
 8794             "cmpl    $div, -1\n\t"
 8795             "je,s    done\n"
 8796     "normal: cdql\n\t"
 8797             "idivl   $div\n"
 8798     "done:"        %}
 8799   ins_encode(cdql_enc(div));
 8800   ins_pipe(ialu_reg_reg_alu0);
 8801 %}
 8802 
 8803 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8804                    rFlagsReg cr)
 8805 %{
 8806   match(Set rax (DivL rax div));
 8807   effect(KILL rdx, KILL cr);
 8808 
 8809   ins_cost(30*100+10*100); // XXX
 8810   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8811             "cmpq    rax, rdx\n\t"
 8812             "jne,s   normal\n\t"
 8813             "xorl    rdx, rdx\n\t"
 8814             "cmpq    $div, -1\n\t"
 8815             "je,s    done\n"
 8816     "normal: cdqq\n\t"
 8817             "idivq   $div\n"
 8818     "done:"        %}
 8819   ins_encode(cdqq_enc(div));
 8820   ins_pipe(ialu_reg_reg_alu0);
 8821 %}
 8822 
 8823 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8824 %{
 8825   match(Set rax (UDivI rax div));
 8826   effect(KILL rdx, KILL cr);
 8827 
 8828   ins_cost(300);
 8829   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8830   ins_encode %{
 8831     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8832   %}
 8833   ins_pipe(ialu_reg_reg_alu0);
 8834 %}
 8835 
 8836 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8837 %{
 8838   match(Set rax (UDivL rax div));
 8839   effect(KILL rdx, KILL cr);
 8840 
 8841   ins_cost(300);
 8842   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8843   ins_encode %{
 8844      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8845   %}
 8846   ins_pipe(ialu_reg_reg_alu0);
 8847 %}
 8848 
 8849 // Integer DIVMOD with Register, both quotient and mod results
 8850 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8851                              rFlagsReg cr)
 8852 %{
 8853   match(DivModI rax div);
 8854   effect(KILL cr);
 8855 
 8856   ins_cost(30*100+10*100); // XXX
 8857   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8858             "jne,s   normal\n\t"
 8859             "xorl    rdx, rdx\n\t"
 8860             "cmpl    $div, -1\n\t"
 8861             "je,s    done\n"
 8862     "normal: cdql\n\t"
 8863             "idivl   $div\n"
 8864     "done:"        %}
 8865   ins_encode(cdql_enc(div));
 8866   ins_pipe(pipe_slow);
 8867 %}
 8868 
 8869 // Long DIVMOD with Register, both quotient and mod results
 8870 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8871                              rFlagsReg cr)
 8872 %{
 8873   match(DivModL rax div);
 8874   effect(KILL cr);
 8875 
 8876   ins_cost(30*100+10*100); // XXX
 8877   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8878             "cmpq    rax, rdx\n\t"
 8879             "jne,s   normal\n\t"
 8880             "xorl    rdx, rdx\n\t"
 8881             "cmpq    $div, -1\n\t"
 8882             "je,s    done\n"
 8883     "normal: cdqq\n\t"
 8884             "idivq   $div\n"
 8885     "done:"        %}
 8886   ins_encode(cdqq_enc(div));
 8887   ins_pipe(pipe_slow);
 8888 %}
 8889 
 8890 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8891 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8892                               no_rax_rdx_RegI div, rFlagsReg cr)
 8893 %{
 8894   match(UDivModI rax div);
 8895   effect(TEMP tmp, KILL cr);
 8896 
 8897   ins_cost(300);
 8898   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8899             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8900           %}
 8901   ins_encode %{
 8902     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8903   %}
 8904   ins_pipe(pipe_slow);
 8905 %}
 8906 
 8907 // Unsigned long DIVMOD with Register, both quotient and mod results
 8908 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8909                               no_rax_rdx_RegL div, rFlagsReg cr)
 8910 %{
 8911   match(UDivModL rax div);
 8912   effect(TEMP tmp, KILL cr);
 8913 
 8914   ins_cost(300);
 8915   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8916             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8917           %}
 8918   ins_encode %{
 8919     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8920   %}
 8921   ins_pipe(pipe_slow);
 8922 %}
 8923 
 8924 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8925                    rFlagsReg cr)
 8926 %{
 8927   match(Set rdx (ModI rax div));
 8928   effect(KILL rax, KILL cr);
 8929 
 8930   ins_cost(300); // XXX
 8931   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8932             "jne,s   normal\n\t"
 8933             "xorl    rdx, rdx\n\t"
 8934             "cmpl    $div, -1\n\t"
 8935             "je,s    done\n"
 8936     "normal: cdql\n\t"
 8937             "idivl   $div\n"
 8938     "done:"        %}
 8939   ins_encode(cdql_enc(div));
 8940   ins_pipe(ialu_reg_reg_alu0);
 8941 %}
 8942 
 8943 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8944                    rFlagsReg cr)
 8945 %{
 8946   match(Set rdx (ModL rax div));
 8947   effect(KILL rax, KILL cr);
 8948 
 8949   ins_cost(300); // XXX
 8950   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8951             "cmpq    rax, rdx\n\t"
 8952             "jne,s   normal\n\t"
 8953             "xorl    rdx, rdx\n\t"
 8954             "cmpq    $div, -1\n\t"
 8955             "je,s    done\n"
 8956     "normal: cdqq\n\t"
 8957             "idivq   $div\n"
 8958     "done:"        %}
 8959   ins_encode(cdqq_enc(div));
 8960   ins_pipe(ialu_reg_reg_alu0);
 8961 %}
 8962 
 8963 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8964 %{
 8965   match(Set rdx (UModI rax div));
 8966   effect(KILL rax, KILL cr);
 8967 
 8968   ins_cost(300);
 8969   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8970   ins_encode %{
 8971     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8972   %}
 8973   ins_pipe(ialu_reg_reg_alu0);
 8974 %}
 8975 
 8976 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8977 %{
 8978   match(Set rdx (UModL rax div));
 8979   effect(KILL rax, KILL cr);
 8980 
 8981   ins_cost(300);
 8982   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8983   ins_encode %{
 8984     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8985   %}
 8986   ins_pipe(ialu_reg_reg_alu0);
 8987 %}
 8988 
 8989 // Integer Shift Instructions
 8990 // Shift Left by one, two, three
 8991 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8992 %{
 8993   predicate(!UseAPX);
 8994   match(Set dst (LShiftI dst shift));
 8995   effect(KILL cr);
 8996 
 8997   format %{ "sall    $dst, $shift" %}
 8998   ins_encode %{
 8999     __ sall($dst$$Register, $shift$$constant);
 9000   %}
 9001   ins_pipe(ialu_reg);
 9002 %}
 9003 
 9004 // Shift Left by one, two, three
 9005 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9006 %{
 9007   predicate(UseAPX);
 9008   match(Set dst (LShiftI src shift));
 9009   effect(KILL cr);
 9010 
 9011   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9012   ins_encode %{
 9013     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9014   %}
 9015   ins_pipe(ialu_reg);
 9016 %}
 9017 
 9018 // Shift Left by 8-bit immediate
 9019 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9020 %{
 9021   predicate(!UseAPX);
 9022   match(Set dst (LShiftI dst shift));
 9023   effect(KILL cr);
 9024 
 9025   format %{ "sall    $dst, $shift" %}
 9026   ins_encode %{
 9027     __ sall($dst$$Register, $shift$$constant);
 9028   %}
 9029   ins_pipe(ialu_reg);
 9030 %}
 9031 
 9032 // Shift Left by 8-bit immediate
 9033 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9034 %{
 9035   predicate(UseAPX);
 9036   match(Set dst (LShiftI src shift));
 9037   effect(KILL cr);
 9038 
 9039   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9040   ins_encode %{
 9041     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9042   %}
 9043   ins_pipe(ialu_reg);
 9044 %}
 9045 
 9046 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9047 %{
 9048   predicate(UseAPX);
 9049   match(Set dst (LShiftI (LoadI src) shift));
 9050   effect(KILL cr);
 9051 
 9052   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9053   ins_encode %{
 9054     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9055   %}
 9056   ins_pipe(ialu_reg);
 9057 %}
 9058 
 9059 // Shift Left by 8-bit immediate
 9060 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9061 %{
 9062   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9063   effect(KILL cr);
 9064 
 9065   format %{ "sall    $dst, $shift" %}
 9066   ins_encode %{
 9067     __ sall($dst$$Address, $shift$$constant);
 9068   %}
 9069   ins_pipe(ialu_mem_imm);
 9070 %}
 9071 
 9072 // Shift Left by variable
 9073 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9074 %{
 9075   predicate(!VM_Version::supports_bmi2());
 9076   match(Set dst (LShiftI dst shift));
 9077   effect(KILL cr);
 9078 
 9079   format %{ "sall    $dst, $shift" %}
 9080   ins_encode %{
 9081     __ sall($dst$$Register);
 9082   %}
 9083   ins_pipe(ialu_reg_reg);
 9084 %}
 9085 
 9086 // Shift Left by variable
 9087 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9088 %{
 9089   predicate(!VM_Version::supports_bmi2());
 9090   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9091   effect(KILL cr);
 9092 
 9093   format %{ "sall    $dst, $shift" %}
 9094   ins_encode %{
 9095     __ sall($dst$$Address);
 9096   %}
 9097   ins_pipe(ialu_mem_reg);
 9098 %}
 9099 
 9100 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9101 %{
 9102   predicate(VM_Version::supports_bmi2());
 9103   match(Set dst (LShiftI src shift));
 9104 
 9105   format %{ "shlxl   $dst, $src, $shift" %}
 9106   ins_encode %{
 9107     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9108   %}
 9109   ins_pipe(ialu_reg_reg);
 9110 %}
 9111 
 9112 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9113 %{
 9114   predicate(VM_Version::supports_bmi2());
 9115   match(Set dst (LShiftI (LoadI src) shift));
 9116   ins_cost(175);
 9117   format %{ "shlxl   $dst, $src, $shift" %}
 9118   ins_encode %{
 9119     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9120   %}
 9121   ins_pipe(ialu_reg_mem);
 9122 %}
 9123 
 9124 // Arithmetic Shift Right by 8-bit immediate
 9125 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9126 %{
 9127   predicate(!UseAPX);
 9128   match(Set dst (RShiftI dst shift));
 9129   effect(KILL cr);
 9130 
 9131   format %{ "sarl    $dst, $shift" %}
 9132   ins_encode %{
 9133     __ sarl($dst$$Register, $shift$$constant);
 9134   %}
 9135   ins_pipe(ialu_mem_imm);
 9136 %}
 9137 
 9138 // Arithmetic Shift Right by 8-bit immediate
 9139 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9140 %{
 9141   predicate(UseAPX);
 9142   match(Set dst (RShiftI src shift));
 9143   effect(KILL cr);
 9144 
 9145   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9146   ins_encode %{
 9147     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9148   %}
 9149   ins_pipe(ialu_mem_imm);
 9150 %}
 9151 
 9152 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9153 %{
 9154   predicate(UseAPX);
 9155   match(Set dst (RShiftI (LoadI src) shift));
 9156   effect(KILL cr);
 9157 
 9158   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9159   ins_encode %{
 9160     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9161   %}
 9162   ins_pipe(ialu_mem_imm);
 9163 %}
 9164 
 9165 // Arithmetic Shift Right by 8-bit immediate
 9166 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9167 %{
 9168   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9169   effect(KILL cr);
 9170 
 9171   format %{ "sarl    $dst, $shift" %}
 9172   ins_encode %{
 9173     __ sarl($dst$$Address, $shift$$constant);
 9174   %}
 9175   ins_pipe(ialu_mem_imm);
 9176 %}
 9177 
 9178 // Arithmetic Shift Right by variable
 9179 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9180 %{
 9181   predicate(!VM_Version::supports_bmi2());
 9182   match(Set dst (RShiftI dst shift));
 9183   effect(KILL cr);
 9184 
 9185   format %{ "sarl    $dst, $shift" %}
 9186   ins_encode %{
 9187     __ sarl($dst$$Register);
 9188   %}
 9189   ins_pipe(ialu_reg_reg);
 9190 %}
 9191 
 9192 // Arithmetic Shift Right by variable
 9193 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9194 %{
 9195   predicate(!VM_Version::supports_bmi2());
 9196   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9197   effect(KILL cr);
 9198 
 9199   format %{ "sarl    $dst, $shift" %}
 9200   ins_encode %{
 9201     __ sarl($dst$$Address);
 9202   %}
 9203   ins_pipe(ialu_mem_reg);
 9204 %}
 9205 
 9206 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9207 %{
 9208   predicate(VM_Version::supports_bmi2());
 9209   match(Set dst (RShiftI src shift));
 9210 
 9211   format %{ "sarxl   $dst, $src, $shift" %}
 9212   ins_encode %{
 9213     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9214   %}
 9215   ins_pipe(ialu_reg_reg);
 9216 %}
 9217 
 9218 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9219 %{
 9220   predicate(VM_Version::supports_bmi2());
 9221   match(Set dst (RShiftI (LoadI src) shift));
 9222   ins_cost(175);
 9223   format %{ "sarxl   $dst, $src, $shift" %}
 9224   ins_encode %{
 9225     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9226   %}
 9227   ins_pipe(ialu_reg_mem);
 9228 %}
 9229 
 9230 // Logical Shift Right by 8-bit immediate
 9231 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9232 %{
 9233   predicate(!UseAPX);
 9234   match(Set dst (URShiftI dst shift));
 9235   effect(KILL cr);
 9236 
 9237   format %{ "shrl    $dst, $shift" %}
 9238   ins_encode %{
 9239     __ shrl($dst$$Register, $shift$$constant);
 9240   %}
 9241   ins_pipe(ialu_reg);
 9242 %}
 9243 
 9244 // Logical Shift Right by 8-bit immediate
 9245 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9246 %{
 9247   predicate(UseAPX);
 9248   match(Set dst (URShiftI src shift));
 9249   effect(KILL cr);
 9250 
 9251   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9252   ins_encode %{
 9253     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9254   %}
 9255   ins_pipe(ialu_reg);
 9256 %}
 9257 
 9258 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9259 %{
 9260   predicate(UseAPX);
 9261   match(Set dst (URShiftI (LoadI src) shift));
 9262   effect(KILL cr);
 9263 
 9264   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9265   ins_encode %{
 9266     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9267   %}
 9268   ins_pipe(ialu_reg);
 9269 %}
 9270 
 9271 // Logical Shift Right by 8-bit immediate
 9272 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9273 %{
 9274   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9275   effect(KILL cr);
 9276 
 9277   format %{ "shrl    $dst, $shift" %}
 9278   ins_encode %{
 9279     __ shrl($dst$$Address, $shift$$constant);
 9280   %}
 9281   ins_pipe(ialu_mem_imm);
 9282 %}
 9283 
 9284 // Logical Shift Right by variable
 9285 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9286 %{
 9287   predicate(!VM_Version::supports_bmi2());
 9288   match(Set dst (URShiftI dst shift));
 9289   effect(KILL cr);
 9290 
 9291   format %{ "shrl    $dst, $shift" %}
 9292   ins_encode %{
 9293     __ shrl($dst$$Register);
 9294   %}
 9295   ins_pipe(ialu_reg_reg);
 9296 %}
 9297 
 9298 // Logical Shift Right by variable
 9299 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9300 %{
 9301   predicate(!VM_Version::supports_bmi2());
 9302   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9303   effect(KILL cr);
 9304 
 9305   format %{ "shrl    $dst, $shift" %}
 9306   ins_encode %{
 9307     __ shrl($dst$$Address);
 9308   %}
 9309   ins_pipe(ialu_mem_reg);
 9310 %}
 9311 
 9312 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9313 %{
 9314   predicate(VM_Version::supports_bmi2());
 9315   match(Set dst (URShiftI src shift));
 9316 
 9317   format %{ "shrxl   $dst, $src, $shift" %}
 9318   ins_encode %{
 9319     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9320   %}
 9321   ins_pipe(ialu_reg_reg);
 9322 %}
 9323 
 9324 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9325 %{
 9326   predicate(VM_Version::supports_bmi2());
 9327   match(Set dst (URShiftI (LoadI src) shift));
 9328   ins_cost(175);
 9329   format %{ "shrxl   $dst, $src, $shift" %}
 9330   ins_encode %{
 9331     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9332   %}
 9333   ins_pipe(ialu_reg_mem);
 9334 %}
 9335 
 9336 // Long Shift Instructions
 9337 // Shift Left by one, two, three
 9338 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9339 %{
 9340   predicate(!UseAPX);
 9341   match(Set dst (LShiftL dst shift));
 9342   effect(KILL cr);
 9343 
 9344   format %{ "salq    $dst, $shift" %}
 9345   ins_encode %{
 9346     __ salq($dst$$Register, $shift$$constant);
 9347   %}
 9348   ins_pipe(ialu_reg);
 9349 %}
 9350 
 9351 // Shift Left by one, two, three
 9352 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9353 %{
 9354   predicate(UseAPX);
 9355   match(Set dst (LShiftL src shift));
 9356   effect(KILL cr);
 9357 
 9358   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9359   ins_encode %{
 9360     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9361   %}
 9362   ins_pipe(ialu_reg);
 9363 %}
 9364 
 9365 // Shift Left by 8-bit immediate
 9366 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9367 %{
 9368   predicate(!UseAPX);
 9369   match(Set dst (LShiftL dst shift));
 9370   effect(KILL cr);
 9371 
 9372   format %{ "salq    $dst, $shift" %}
 9373   ins_encode %{
 9374     __ salq($dst$$Register, $shift$$constant);
 9375   %}
 9376   ins_pipe(ialu_reg);
 9377 %}
 9378 
 9379 // Shift Left by 8-bit immediate
 9380 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9381 %{
 9382   predicate(UseAPX);
 9383   match(Set dst (LShiftL src shift));
 9384   effect(KILL cr);
 9385 
 9386   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9387   ins_encode %{
 9388     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9389   %}
 9390   ins_pipe(ialu_reg);
 9391 %}
 9392 
 9393 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9394 %{
 9395   predicate(UseAPX);
 9396   match(Set dst (LShiftL (LoadL src) shift));
 9397   effect(KILL cr);
 9398 
 9399   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9400   ins_encode %{
 9401     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9402   %}
 9403   ins_pipe(ialu_reg);
 9404 %}
 9405 
 9406 // Shift Left by 8-bit immediate
 9407 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9408 %{
 9409   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9410   effect(KILL cr);
 9411 
 9412   format %{ "salq    $dst, $shift" %}
 9413   ins_encode %{
 9414     __ salq($dst$$Address, $shift$$constant);
 9415   %}
 9416   ins_pipe(ialu_mem_imm);
 9417 %}
 9418 
 9419 // Shift Left by variable
 9420 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9421 %{
 9422   predicate(!VM_Version::supports_bmi2());
 9423   match(Set dst (LShiftL dst shift));
 9424   effect(KILL cr);
 9425 
 9426   format %{ "salq    $dst, $shift" %}
 9427   ins_encode %{
 9428     __ salq($dst$$Register);
 9429   %}
 9430   ins_pipe(ialu_reg_reg);
 9431 %}
 9432 
 9433 // Shift Left by variable
 9434 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9435 %{
 9436   predicate(!VM_Version::supports_bmi2());
 9437   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9438   effect(KILL cr);
 9439 
 9440   format %{ "salq    $dst, $shift" %}
 9441   ins_encode %{
 9442     __ salq($dst$$Address);
 9443   %}
 9444   ins_pipe(ialu_mem_reg);
 9445 %}
 9446 
 9447 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9448 %{
 9449   predicate(VM_Version::supports_bmi2());
 9450   match(Set dst (LShiftL src shift));
 9451 
 9452   format %{ "shlxq   $dst, $src, $shift" %}
 9453   ins_encode %{
 9454     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9455   %}
 9456   ins_pipe(ialu_reg_reg);
 9457 %}
 9458 
 9459 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9460 %{
 9461   predicate(VM_Version::supports_bmi2());
 9462   match(Set dst (LShiftL (LoadL src) shift));
 9463   ins_cost(175);
 9464   format %{ "shlxq   $dst, $src, $shift" %}
 9465   ins_encode %{
 9466     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9467   %}
 9468   ins_pipe(ialu_reg_mem);
 9469 %}
 9470 
 9471 // Arithmetic Shift Right by 8-bit immediate
 9472 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9473 %{
 9474   predicate(!UseAPX);
 9475   match(Set dst (RShiftL dst shift));
 9476   effect(KILL cr);
 9477 
 9478   format %{ "sarq    $dst, $shift" %}
 9479   ins_encode %{
 9480     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9481   %}
 9482   ins_pipe(ialu_mem_imm);
 9483 %}
 9484 
 9485 // Arithmetic Shift Right by 8-bit immediate
 9486 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9487 %{
 9488   predicate(UseAPX);
 9489   match(Set dst (RShiftL src shift));
 9490   effect(KILL cr);
 9491 
 9492   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9493   ins_encode %{
 9494     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9495   %}
 9496   ins_pipe(ialu_mem_imm);
 9497 %}
 9498 
 9499 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9500 %{
 9501   predicate(UseAPX);
 9502   match(Set dst (RShiftL (LoadL src) shift));
 9503   effect(KILL cr);
 9504 
 9505   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9506   ins_encode %{
 9507     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9508   %}
 9509   ins_pipe(ialu_mem_imm);
 9510 %}
 9511 
 9512 // Arithmetic Shift Right by 8-bit immediate
 9513 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9514 %{
 9515   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9516   effect(KILL cr);
 9517 
 9518   format %{ "sarq    $dst, $shift" %}
 9519   ins_encode %{
 9520     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9521   %}
 9522   ins_pipe(ialu_mem_imm);
 9523 %}
 9524 
 9525 // Arithmetic Shift Right by variable
 9526 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9527 %{
 9528   predicate(!VM_Version::supports_bmi2());
 9529   match(Set dst (RShiftL dst shift));
 9530   effect(KILL cr);
 9531 
 9532   format %{ "sarq    $dst, $shift" %}
 9533   ins_encode %{
 9534     __ sarq($dst$$Register);
 9535   %}
 9536   ins_pipe(ialu_reg_reg);
 9537 %}
 9538 
 9539 // Arithmetic Shift Right by variable
 9540 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9541 %{
 9542   predicate(!VM_Version::supports_bmi2());
 9543   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9544   effect(KILL cr);
 9545 
 9546   format %{ "sarq    $dst, $shift" %}
 9547   ins_encode %{
 9548     __ sarq($dst$$Address);
 9549   %}
 9550   ins_pipe(ialu_mem_reg);
 9551 %}
 9552 
 9553 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9554 %{
 9555   predicate(VM_Version::supports_bmi2());
 9556   match(Set dst (RShiftL src shift));
 9557 
 9558   format %{ "sarxq   $dst, $src, $shift" %}
 9559   ins_encode %{
 9560     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9561   %}
 9562   ins_pipe(ialu_reg_reg);
 9563 %}
 9564 
 9565 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9566 %{
 9567   predicate(VM_Version::supports_bmi2());
 9568   match(Set dst (RShiftL (LoadL src) shift));
 9569   ins_cost(175);
 9570   format %{ "sarxq   $dst, $src, $shift" %}
 9571   ins_encode %{
 9572     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9573   %}
 9574   ins_pipe(ialu_reg_mem);
 9575 %}
 9576 
 9577 // Logical Shift Right by 8-bit immediate
 9578 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9579 %{
 9580   predicate(!UseAPX);
 9581   match(Set dst (URShiftL dst shift));
 9582   effect(KILL cr);
 9583 
 9584   format %{ "shrq    $dst, $shift" %}
 9585   ins_encode %{
 9586     __ shrq($dst$$Register, $shift$$constant);
 9587   %}
 9588   ins_pipe(ialu_reg);
 9589 %}
 9590 
 9591 // Logical Shift Right by 8-bit immediate
 9592 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9593 %{
 9594   predicate(UseAPX);
 9595   match(Set dst (URShiftL src shift));
 9596   effect(KILL cr);
 9597 
 9598   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9599   ins_encode %{
 9600     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9601   %}
 9602   ins_pipe(ialu_reg);
 9603 %}
 9604 
 9605 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9606 %{
 9607   predicate(UseAPX);
 9608   match(Set dst (URShiftL (LoadL src) shift));
 9609   effect(KILL cr);
 9610 
 9611   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9612   ins_encode %{
 9613     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9614   %}
 9615   ins_pipe(ialu_reg);
 9616 %}
 9617 
 9618 // Logical Shift Right by 8-bit immediate
 9619 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9620 %{
 9621   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9622   effect(KILL cr);
 9623 
 9624   format %{ "shrq    $dst, $shift" %}
 9625   ins_encode %{
 9626     __ shrq($dst$$Address, $shift$$constant);
 9627   %}
 9628   ins_pipe(ialu_mem_imm);
 9629 %}
 9630 
 9631 // Logical Shift Right by variable
 9632 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9633 %{
 9634   predicate(!VM_Version::supports_bmi2());
 9635   match(Set dst (URShiftL dst shift));
 9636   effect(KILL cr);
 9637 
 9638   format %{ "shrq    $dst, $shift" %}
 9639   ins_encode %{
 9640     __ shrq($dst$$Register);
 9641   %}
 9642   ins_pipe(ialu_reg_reg);
 9643 %}
 9644 
 9645 // Logical Shift Right by variable
 9646 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9647 %{
 9648   predicate(!VM_Version::supports_bmi2());
 9649   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9650   effect(KILL cr);
 9651 
 9652   format %{ "shrq    $dst, $shift" %}
 9653   ins_encode %{
 9654     __ shrq($dst$$Address);
 9655   %}
 9656   ins_pipe(ialu_mem_reg);
 9657 %}
 9658 
 9659 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9660 %{
 9661   predicate(VM_Version::supports_bmi2());
 9662   match(Set dst (URShiftL src shift));
 9663 
 9664   format %{ "shrxq   $dst, $src, $shift" %}
 9665   ins_encode %{
 9666     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9667   %}
 9668   ins_pipe(ialu_reg_reg);
 9669 %}
 9670 
 9671 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9672 %{
 9673   predicate(VM_Version::supports_bmi2());
 9674   match(Set dst (URShiftL (LoadL src) shift));
 9675   ins_cost(175);
 9676   format %{ "shrxq   $dst, $src, $shift" %}
 9677   ins_encode %{
 9678     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9679   %}
 9680   ins_pipe(ialu_reg_mem);
 9681 %}
 9682 
 9683 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9684 // This idiom is used by the compiler for the i2b bytecode.
 9685 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9686 %{
 9687   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9688 
 9689   format %{ "movsbl  $dst, $src\t# i2b" %}
 9690   ins_encode %{
 9691     __ movsbl($dst$$Register, $src$$Register);
 9692   %}
 9693   ins_pipe(ialu_reg_reg);
 9694 %}
 9695 
 9696 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9697 // This idiom is used by the compiler the i2s bytecode.
 9698 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9699 %{
 9700   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9701 
 9702   format %{ "movswl  $dst, $src\t# i2s" %}
 9703   ins_encode %{
 9704     __ movswl($dst$$Register, $src$$Register);
 9705   %}
 9706   ins_pipe(ialu_reg_reg);
 9707 %}
 9708 
 9709 // ROL/ROR instructions
 9710 
 9711 // Rotate left by constant.
 9712 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9713 %{
 9714   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9715   match(Set dst (RotateLeft dst shift));
 9716   effect(KILL cr);
 9717   format %{ "roll    $dst, $shift" %}
 9718   ins_encode %{
 9719     __ roll($dst$$Register, $shift$$constant);
 9720   %}
 9721   ins_pipe(ialu_reg);
 9722 %}
 9723 
 9724 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9725 %{
 9726   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9727   match(Set dst (RotateLeft src shift));
 9728   format %{ "rolxl   $dst, $src, $shift" %}
 9729   ins_encode %{
 9730     int shift = 32 - ($shift$$constant & 31);
 9731     __ rorxl($dst$$Register, $src$$Register, shift);
 9732   %}
 9733   ins_pipe(ialu_reg_reg);
 9734 %}
 9735 
 9736 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9737 %{
 9738   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9739   match(Set dst (RotateLeft (LoadI src) shift));
 9740   ins_cost(175);
 9741   format %{ "rolxl   $dst, $src, $shift" %}
 9742   ins_encode %{
 9743     int shift = 32 - ($shift$$constant & 31);
 9744     __ rorxl($dst$$Register, $src$$Address, shift);
 9745   %}
 9746   ins_pipe(ialu_reg_mem);
 9747 %}
 9748 
 9749 // Rotate Left by variable
 9750 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9751 %{
 9752   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9753   match(Set dst (RotateLeft dst shift));
 9754   effect(KILL cr);
 9755   format %{ "roll    $dst, $shift" %}
 9756   ins_encode %{
 9757     __ roll($dst$$Register);
 9758   %}
 9759   ins_pipe(ialu_reg_reg);
 9760 %}
 9761 
 9762 // Rotate Left by variable
 9763 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9764 %{
 9765   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9766   match(Set dst (RotateLeft src shift));
 9767   effect(KILL cr);
 9768 
 9769   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9770   ins_encode %{
 9771     __ eroll($dst$$Register, $src$$Register, false);
 9772   %}
 9773   ins_pipe(ialu_reg_reg);
 9774 %}
 9775 
 9776 // Rotate Right by constant.
 9777 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9778 %{
 9779   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9780   match(Set dst (RotateRight dst shift));
 9781   effect(KILL cr);
 9782   format %{ "rorl    $dst, $shift" %}
 9783   ins_encode %{
 9784     __ rorl($dst$$Register, $shift$$constant);
 9785   %}
 9786   ins_pipe(ialu_reg);
 9787 %}
 9788 
 9789 // Rotate Right by constant.
 9790 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9791 %{
 9792   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9793   match(Set dst (RotateRight src shift));
 9794   format %{ "rorxl   $dst, $src, $shift" %}
 9795   ins_encode %{
 9796     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9797   %}
 9798   ins_pipe(ialu_reg_reg);
 9799 %}
 9800 
 9801 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9802 %{
 9803   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9804   match(Set dst (RotateRight (LoadI src) shift));
 9805   ins_cost(175);
 9806   format %{ "rorxl   $dst, $src, $shift" %}
 9807   ins_encode %{
 9808     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9809   %}
 9810   ins_pipe(ialu_reg_mem);
 9811 %}
 9812 
 9813 // Rotate Right by variable
 9814 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9815 %{
 9816   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9817   match(Set dst (RotateRight dst shift));
 9818   effect(KILL cr);
 9819   format %{ "rorl    $dst, $shift" %}
 9820   ins_encode %{
 9821     __ rorl($dst$$Register);
 9822   %}
 9823   ins_pipe(ialu_reg_reg);
 9824 %}
 9825 
 9826 // Rotate Right by variable
 9827 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9828 %{
 9829   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9830   match(Set dst (RotateRight src shift));
 9831   effect(KILL cr);
 9832 
 9833   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9834   ins_encode %{
 9835     __ erorl($dst$$Register, $src$$Register, false);
 9836   %}
 9837   ins_pipe(ialu_reg_reg);
 9838 %}
 9839 
 9840 // Rotate Left by constant.
 9841 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9842 %{
 9843   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9844   match(Set dst (RotateLeft dst shift));
 9845   effect(KILL cr);
 9846   format %{ "rolq    $dst, $shift" %}
 9847   ins_encode %{
 9848     __ rolq($dst$$Register, $shift$$constant);
 9849   %}
 9850   ins_pipe(ialu_reg);
 9851 %}
 9852 
 9853 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9854 %{
 9855   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9856   match(Set dst (RotateLeft src shift));
 9857   format %{ "rolxq   $dst, $src, $shift" %}
 9858   ins_encode %{
 9859     int shift = 64 - ($shift$$constant & 63);
 9860     __ rorxq($dst$$Register, $src$$Register, shift);
 9861   %}
 9862   ins_pipe(ialu_reg_reg);
 9863 %}
 9864 
 9865 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9866 %{
 9867   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9868   match(Set dst (RotateLeft (LoadL src) shift));
 9869   ins_cost(175);
 9870   format %{ "rolxq   $dst, $src, $shift" %}
 9871   ins_encode %{
 9872     int shift = 64 - ($shift$$constant & 63);
 9873     __ rorxq($dst$$Register, $src$$Address, shift);
 9874   %}
 9875   ins_pipe(ialu_reg_mem);
 9876 %}
 9877 
 9878 // Rotate Left by variable
 9879 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9880 %{
 9881   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9882   match(Set dst (RotateLeft dst shift));
 9883   effect(KILL cr);
 9884   format %{ "rolq    $dst, $shift" %}
 9885   ins_encode %{
 9886     __ rolq($dst$$Register);
 9887   %}
 9888   ins_pipe(ialu_reg_reg);
 9889 %}
 9890 
 9891 // Rotate Left by variable
 9892 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9893 %{
 9894   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9895   match(Set dst (RotateLeft src shift));
 9896   effect(KILL cr);
 9897 
 9898   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9899   ins_encode %{
 9900     __ erolq($dst$$Register, $src$$Register, false);
 9901   %}
 9902   ins_pipe(ialu_reg_reg);
 9903 %}
 9904 
 9905 // Rotate Right by constant.
 9906 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9907 %{
 9908   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9909   match(Set dst (RotateRight dst shift));
 9910   effect(KILL cr);
 9911   format %{ "rorq    $dst, $shift" %}
 9912   ins_encode %{
 9913     __ rorq($dst$$Register, $shift$$constant);
 9914   %}
 9915   ins_pipe(ialu_reg);
 9916 %}
 9917 
 9918 // Rotate Right by constant
 9919 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9920 %{
 9921   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9922   match(Set dst (RotateRight src shift));
 9923   format %{ "rorxq   $dst, $src, $shift" %}
 9924   ins_encode %{
 9925     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9926   %}
 9927   ins_pipe(ialu_reg_reg);
 9928 %}
 9929 
 9930 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9931 %{
 9932   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9933   match(Set dst (RotateRight (LoadL src) shift));
 9934   ins_cost(175);
 9935   format %{ "rorxq   $dst, $src, $shift" %}
 9936   ins_encode %{
 9937     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9938   %}
 9939   ins_pipe(ialu_reg_mem);
 9940 %}
 9941 
 9942 // Rotate Right by variable
 9943 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9944 %{
 9945   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9946   match(Set dst (RotateRight dst shift));
 9947   effect(KILL cr);
 9948   format %{ "rorq    $dst, $shift" %}
 9949   ins_encode %{
 9950     __ rorq($dst$$Register);
 9951   %}
 9952   ins_pipe(ialu_reg_reg);
 9953 %}
 9954 
 9955 // Rotate Right by variable
 9956 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9957 %{
 9958   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9959   match(Set dst (RotateRight src shift));
 9960   effect(KILL cr);
 9961 
 9962   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
 9963   ins_encode %{
 9964     __ erorq($dst$$Register, $src$$Register, false);
 9965   %}
 9966   ins_pipe(ialu_reg_reg);
 9967 %}
 9968 
 9969 //----------------------------- CompressBits/ExpandBits ------------------------
 9970 
 9971 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9972   predicate(n->bottom_type()->isa_long());
 9973   match(Set dst (CompressBits src mask));
 9974   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9975   ins_encode %{
 9976     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9977   %}
 9978   ins_pipe( pipe_slow );
 9979 %}
 9980 
 9981 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9982   predicate(n->bottom_type()->isa_long());
 9983   match(Set dst (ExpandBits src mask));
 9984   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9985   ins_encode %{
 9986     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9987   %}
 9988   ins_pipe( pipe_slow );
 9989 %}
 9990 
 9991 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9992   predicate(n->bottom_type()->isa_long());
 9993   match(Set dst (CompressBits src (LoadL mask)));
 9994   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9995   ins_encode %{
 9996     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
 9997   %}
 9998   ins_pipe( pipe_slow );
 9999 %}
10000 
10001 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10002   predicate(n->bottom_type()->isa_long());
10003   match(Set dst (ExpandBits src (LoadL mask)));
10004   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10005   ins_encode %{
10006     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10007   %}
10008   ins_pipe( pipe_slow );
10009 %}
10010 
10011 
10012 // Logical Instructions
10013 
10014 // Integer Logical Instructions
10015 
10016 // And Instructions
10017 // And Register with Register
10018 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10019 %{
10020   predicate(!UseAPX);
10021   match(Set dst (AndI dst src));
10022   effect(KILL cr);
10023   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10024 
10025   format %{ "andl    $dst, $src\t# int" %}
10026   ins_encode %{
10027     __ andl($dst$$Register, $src$$Register);
10028   %}
10029   ins_pipe(ialu_reg_reg);
10030 %}
10031 
10032 // And Register with Register using New Data Destination (NDD)
10033 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10034 %{
10035   predicate(UseAPX);
10036   match(Set dst (AndI src1 src2));
10037   effect(KILL cr);
10038   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10039 
10040   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10041   ins_encode %{
10042     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10043 
10044   %}
10045   ins_pipe(ialu_reg_reg);
10046 %}
10047 
10048 // And Register with Immediate 255
10049 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10050 %{
10051   match(Set dst (AndI src mask));
10052 
10053   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10054   ins_encode %{
10055     __ movzbl($dst$$Register, $src$$Register);
10056   %}
10057   ins_pipe(ialu_reg);
10058 %}
10059 
10060 // And Register with Immediate 255 and promote to long
10061 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10062 %{
10063   match(Set dst (ConvI2L (AndI src mask)));
10064 
10065   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10066   ins_encode %{
10067     __ movzbl($dst$$Register, $src$$Register);
10068   %}
10069   ins_pipe(ialu_reg);
10070 %}
10071 
10072 // And Register with Immediate 65535
10073 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10074 %{
10075   match(Set dst (AndI src mask));
10076 
10077   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10078   ins_encode %{
10079     __ movzwl($dst$$Register, $src$$Register);
10080   %}
10081   ins_pipe(ialu_reg);
10082 %}
10083 
10084 // And Register with Immediate 65535 and promote to long
10085 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10086 %{
10087   match(Set dst (ConvI2L (AndI src mask)));
10088 
10089   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10090   ins_encode %{
10091     __ movzwl($dst$$Register, $src$$Register);
10092   %}
10093   ins_pipe(ialu_reg);
10094 %}
10095 
10096 // Can skip int2long conversions after AND with small bitmask
10097 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10098 %{
10099   predicate(VM_Version::supports_bmi2());
10100   ins_cost(125);
10101   effect(TEMP tmp, KILL cr);
10102   match(Set dst (ConvI2L (AndI src mask)));
10103   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10104   ins_encode %{
10105     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10106     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10107   %}
10108   ins_pipe(ialu_reg_reg);
10109 %}
10110 
10111 // And Register with Immediate
10112 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10113 %{
10114   predicate(!UseAPX);
10115   match(Set dst (AndI dst src));
10116   effect(KILL cr);
10117   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10118 
10119   format %{ "andl    $dst, $src\t# int" %}
10120   ins_encode %{
10121     __ andl($dst$$Register, $src$$constant);
10122   %}
10123   ins_pipe(ialu_reg);
10124 %}
10125 
10126 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10127 %{
10128   predicate(UseAPX);
10129   match(Set dst (AndI src1 src2));
10130   effect(KILL cr);
10131   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10132 
10133   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10134   ins_encode %{
10135     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10136   %}
10137   ins_pipe(ialu_reg);
10138 %}
10139 
10140 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10141 %{
10142   predicate(UseAPX);
10143   match(Set dst (AndI (LoadI src1) src2));
10144   effect(KILL cr);
10145   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10146 
10147   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10148   ins_encode %{
10149     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10150   %}
10151   ins_pipe(ialu_reg);
10152 %}
10153 
10154 // And Register with Memory
10155 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10156 %{
10157   predicate(!UseAPX);
10158   match(Set dst (AndI dst (LoadI src)));
10159   effect(KILL cr);
10160   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10161 
10162   ins_cost(150);
10163   format %{ "andl    $dst, $src\t# int" %}
10164   ins_encode %{
10165     __ andl($dst$$Register, $src$$Address);
10166   %}
10167   ins_pipe(ialu_reg_mem);
10168 %}
10169 
10170 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10171 %{
10172   predicate(UseAPX);
10173   match(Set dst (AndI src1 (LoadI src2)));
10174   effect(KILL cr);
10175   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10176 
10177   ins_cost(150);
10178   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10179   ins_encode %{
10180     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10181   %}
10182   ins_pipe(ialu_reg_mem);
10183 %}
10184 
10185 // And Memory with Register
10186 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10187 %{
10188   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10189   effect(KILL cr);
10190   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10191 
10192   ins_cost(150);
10193   format %{ "andb    $dst, $src\t# byte" %}
10194   ins_encode %{
10195     __ andb($dst$$Address, $src$$Register);
10196   %}
10197   ins_pipe(ialu_mem_reg);
10198 %}
10199 
10200 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10201 %{
10202   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10203   effect(KILL cr);
10204   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10205 
10206   ins_cost(150);
10207   format %{ "andl    $dst, $src\t# int" %}
10208   ins_encode %{
10209     __ andl($dst$$Address, $src$$Register);
10210   %}
10211   ins_pipe(ialu_mem_reg);
10212 %}
10213 
10214 // And Memory with Immediate
10215 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10216 %{
10217   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10218   effect(KILL cr);
10219   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10220 
10221   ins_cost(125);
10222   format %{ "andl    $dst, $src\t# int" %}
10223   ins_encode %{
10224     __ andl($dst$$Address, $src$$constant);
10225   %}
10226   ins_pipe(ialu_mem_imm);
10227 %}
10228 
10229 // BMI1 instructions
10230 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10231   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10232   predicate(UseBMI1Instructions);
10233   effect(KILL cr);
10234   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10235 
10236   ins_cost(125);
10237   format %{ "andnl  $dst, $src1, $src2" %}
10238 
10239   ins_encode %{
10240     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10241   %}
10242   ins_pipe(ialu_reg_mem);
10243 %}
10244 
10245 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10246   match(Set dst (AndI (XorI src1 minus_1) src2));
10247   predicate(UseBMI1Instructions);
10248   effect(KILL cr);
10249   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10250 
10251   format %{ "andnl  $dst, $src1, $src2" %}
10252 
10253   ins_encode %{
10254     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10255   %}
10256   ins_pipe(ialu_reg);
10257 %}
10258 
10259 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10260   match(Set dst (AndI (SubI imm_zero src) src));
10261   predicate(UseBMI1Instructions);
10262   effect(KILL cr);
10263   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10264 
10265   format %{ "blsil  $dst, $src" %}
10266 
10267   ins_encode %{
10268     __ blsil($dst$$Register, $src$$Register);
10269   %}
10270   ins_pipe(ialu_reg);
10271 %}
10272 
10273 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10274   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10275   predicate(UseBMI1Instructions);
10276   effect(KILL cr);
10277   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10278 
10279   ins_cost(125);
10280   format %{ "blsil  $dst, $src" %}
10281 
10282   ins_encode %{
10283     __ blsil($dst$$Register, $src$$Address);
10284   %}
10285   ins_pipe(ialu_reg_mem);
10286 %}
10287 
10288 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10289 %{
10290   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10291   predicate(UseBMI1Instructions);
10292   effect(KILL cr);
10293   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10294 
10295   ins_cost(125);
10296   format %{ "blsmskl $dst, $src" %}
10297 
10298   ins_encode %{
10299     __ blsmskl($dst$$Register, $src$$Address);
10300   %}
10301   ins_pipe(ialu_reg_mem);
10302 %}
10303 
10304 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10305 %{
10306   match(Set dst (XorI (AddI src minus_1) src));
10307   predicate(UseBMI1Instructions);
10308   effect(KILL cr);
10309   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10310 
10311   format %{ "blsmskl $dst, $src" %}
10312 
10313   ins_encode %{
10314     __ blsmskl($dst$$Register, $src$$Register);
10315   %}
10316 
10317   ins_pipe(ialu_reg);
10318 %}
10319 
10320 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10321 %{
10322   match(Set dst (AndI (AddI src minus_1) src) );
10323   predicate(UseBMI1Instructions);
10324   effect(KILL cr);
10325   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10326 
10327   format %{ "blsrl  $dst, $src" %}
10328 
10329   ins_encode %{
10330     __ blsrl($dst$$Register, $src$$Register);
10331   %}
10332 
10333   ins_pipe(ialu_reg_mem);
10334 %}
10335 
10336 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10337 %{
10338   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10339   predicate(UseBMI1Instructions);
10340   effect(KILL cr);
10341   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10342 
10343   ins_cost(125);
10344   format %{ "blsrl  $dst, $src" %}
10345 
10346   ins_encode %{
10347     __ blsrl($dst$$Register, $src$$Address);
10348   %}
10349 
10350   ins_pipe(ialu_reg);
10351 %}
10352 
10353 // Or Instructions
10354 // Or Register with Register
10355 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10356 %{
10357   predicate(!UseAPX);
10358   match(Set dst (OrI dst src));
10359   effect(KILL cr);
10360   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10361 
10362   format %{ "orl     $dst, $src\t# int" %}
10363   ins_encode %{
10364     __ orl($dst$$Register, $src$$Register);
10365   %}
10366   ins_pipe(ialu_reg_reg);
10367 %}
10368 
10369 // Or Register with Register using New Data Destination (NDD)
10370 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10371 %{
10372   predicate(UseAPX);
10373   match(Set dst (OrI src1 src2));
10374   effect(KILL cr);
10375   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10376 
10377   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10378   ins_encode %{
10379     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10380   %}
10381   ins_pipe(ialu_reg_reg);
10382 %}
10383 
10384 // Or Register with Immediate
10385 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10386 %{
10387   predicate(!UseAPX);
10388   match(Set dst (OrI dst src));
10389   effect(KILL cr);
10390   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10391 
10392   format %{ "orl     $dst, $src\t# int" %}
10393   ins_encode %{
10394     __ orl($dst$$Register, $src$$constant);
10395   %}
10396   ins_pipe(ialu_reg);
10397 %}
10398 
10399 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10400 %{
10401   predicate(UseAPX);
10402   match(Set dst (OrI src1 src2));
10403   effect(KILL cr);
10404   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10405 
10406   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10407   ins_encode %{
10408     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10409   %}
10410   ins_pipe(ialu_reg);
10411 %}
10412 
10413 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10414 %{
10415   predicate(UseAPX);
10416   match(Set dst (OrI src1 src2));
10417   effect(KILL cr);
10418   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10419 
10420   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10421   ins_encode %{
10422     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10423   %}
10424   ins_pipe(ialu_reg);
10425 %}
10426 
10427 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10428 %{
10429   predicate(UseAPX);
10430   match(Set dst (OrI (LoadI src1) src2));
10431   effect(KILL cr);
10432   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10433 
10434   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10435   ins_encode %{
10436     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10437   %}
10438   ins_pipe(ialu_reg);
10439 %}
10440 
10441 // Or Register with Memory
10442 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10443 %{
10444   predicate(!UseAPX);
10445   match(Set dst (OrI dst (LoadI src)));
10446   effect(KILL cr);
10447   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10448 
10449   ins_cost(150);
10450   format %{ "orl     $dst, $src\t# int" %}
10451   ins_encode %{
10452     __ orl($dst$$Register, $src$$Address);
10453   %}
10454   ins_pipe(ialu_reg_mem);
10455 %}
10456 
10457 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10458 %{
10459   predicate(UseAPX);
10460   match(Set dst (OrI src1 (LoadI src2)));
10461   effect(KILL cr);
10462   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10463 
10464   ins_cost(150);
10465   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10466   ins_encode %{
10467     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10468   %}
10469   ins_pipe(ialu_reg_mem);
10470 %}
10471 
10472 // Or Memory with Register
10473 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10474 %{
10475   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10476   effect(KILL cr);
10477   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10478 
10479   ins_cost(150);
10480   format %{ "orb    $dst, $src\t# byte" %}
10481   ins_encode %{
10482     __ orb($dst$$Address, $src$$Register);
10483   %}
10484   ins_pipe(ialu_mem_reg);
10485 %}
10486 
10487 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10488 %{
10489   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10490   effect(KILL cr);
10491   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10492 
10493   ins_cost(150);
10494   format %{ "orl     $dst, $src\t# int" %}
10495   ins_encode %{
10496     __ orl($dst$$Address, $src$$Register);
10497   %}
10498   ins_pipe(ialu_mem_reg);
10499 %}
10500 
10501 // Or Memory with Immediate
10502 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10503 %{
10504   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10505   effect(KILL cr);
10506   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10507 
10508   ins_cost(125);
10509   format %{ "orl     $dst, $src\t# int" %}
10510   ins_encode %{
10511     __ orl($dst$$Address, $src$$constant);
10512   %}
10513   ins_pipe(ialu_mem_imm);
10514 %}
10515 
10516 // Xor Instructions
10517 // Xor Register with Register
10518 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10519 %{
10520   predicate(!UseAPX);
10521   match(Set dst (XorI dst src));
10522   effect(KILL cr);
10523   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10524 
10525   format %{ "xorl    $dst, $src\t# int" %}
10526   ins_encode %{
10527     __ xorl($dst$$Register, $src$$Register);
10528   %}
10529   ins_pipe(ialu_reg_reg);
10530 %}
10531 
10532 // Xor Register with Register using New Data Destination (NDD)
10533 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10534 %{
10535   predicate(UseAPX);
10536   match(Set dst (XorI src1 src2));
10537   effect(KILL cr);
10538   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10539 
10540   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10541   ins_encode %{
10542     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10543   %}
10544   ins_pipe(ialu_reg_reg);
10545 %}
10546 
10547 // Xor Register with Immediate -1
10548 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10549 %{
10550   predicate(!UseAPX);
10551   match(Set dst (XorI dst imm));
10552 
10553   format %{ "notl    $dst" %}
10554   ins_encode %{
10555      __ notl($dst$$Register);
10556   %}
10557   ins_pipe(ialu_reg);
10558 %}
10559 
10560 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10561 %{
10562   match(Set dst (XorI src imm));
10563   predicate(UseAPX);
10564 
10565   format %{ "enotl    $dst, $src" %}
10566   ins_encode %{
10567      __ enotl($dst$$Register, $src$$Register);
10568   %}
10569   ins_pipe(ialu_reg);
10570 %}
10571 
10572 // Xor Register with Immediate
10573 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10574 %{
10575   // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
10576   predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10577   match(Set dst (XorI dst src));
10578   effect(KILL cr);
10579   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10580 
10581   format %{ "xorl    $dst, $src\t# int" %}
10582   ins_encode %{
10583     __ xorl($dst$$Register, $src$$constant);
10584   %}
10585   ins_pipe(ialu_reg);
10586 %}
10587 
10588 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10589 %{
10590   // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
10591   predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10592   match(Set dst (XorI src1 src2));
10593   effect(KILL cr);
10594   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10595 
10596   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10597   ins_encode %{
10598     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10599   %}
10600   ins_pipe(ialu_reg);
10601 %}
10602 
10603 // Xor Memory with Immediate
10604 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10605 %{
10606   predicate(UseAPX);
10607   match(Set dst (XorI (LoadI src1) src2));
10608   effect(KILL cr);
10609   ins_cost(150);
10610   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10611 
10612   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10613   ins_encode %{
10614     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10615   %}
10616   ins_pipe(ialu_reg);
10617 %}
10618 
10619 // Xor Register with Memory
10620 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10621 %{
10622   predicate(!UseAPX);
10623   match(Set dst (XorI dst (LoadI src)));
10624   effect(KILL cr);
10625   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10626 
10627   ins_cost(150);
10628   format %{ "xorl    $dst, $src\t# int" %}
10629   ins_encode %{
10630     __ xorl($dst$$Register, $src$$Address);
10631   %}
10632   ins_pipe(ialu_reg_mem);
10633 %}
10634 
10635 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10636 %{
10637   predicate(UseAPX);
10638   match(Set dst (XorI src1 (LoadI src2)));
10639   effect(KILL cr);
10640   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10641 
10642   ins_cost(150);
10643   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10644   ins_encode %{
10645     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10646   %}
10647   ins_pipe(ialu_reg_mem);
10648 %}
10649 
10650 // Xor Memory with Register
10651 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10652 %{
10653   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10654   effect(KILL cr);
10655   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10656 
10657   ins_cost(150);
10658   format %{ "xorb    $dst, $src\t# byte" %}
10659   ins_encode %{
10660     __ xorb($dst$$Address, $src$$Register);
10661   %}
10662   ins_pipe(ialu_mem_reg);
10663 %}
10664 
10665 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10666 %{
10667   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10668   effect(KILL cr);
10669   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10670 
10671   ins_cost(150);
10672   format %{ "xorl    $dst, $src\t# int" %}
10673   ins_encode %{
10674     __ xorl($dst$$Address, $src$$Register);
10675   %}
10676   ins_pipe(ialu_mem_reg);
10677 %}
10678 
10679 // Xor Memory with Immediate
10680 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10681 %{
10682   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10683   effect(KILL cr);
10684   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10685 
10686   ins_cost(125);
10687   format %{ "xorl    $dst, $src\t# int" %}
10688   ins_encode %{
10689     __ xorl($dst$$Address, $src$$constant);
10690   %}
10691   ins_pipe(ialu_mem_imm);
10692 %}
10693 
10694 
10695 // Long Logical Instructions
10696 
10697 // And Instructions
10698 // And Register with Register
10699 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10700 %{
10701   predicate(!UseAPX);
10702   match(Set dst (AndL dst src));
10703   effect(KILL cr);
10704   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10705 
10706   format %{ "andq    $dst, $src\t# long" %}
10707   ins_encode %{
10708     __ andq($dst$$Register, $src$$Register);
10709   %}
10710   ins_pipe(ialu_reg_reg);
10711 %}
10712 
10713 // And Register with Register using New Data Destination (NDD)
10714 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10715 %{
10716   predicate(UseAPX);
10717   match(Set dst (AndL src1 src2));
10718   effect(KILL cr);
10719   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10720 
10721   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10722   ins_encode %{
10723     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10724 
10725   %}
10726   ins_pipe(ialu_reg_reg);
10727 %}
10728 
10729 // And Register with Immediate 255
10730 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10731 %{
10732   match(Set dst (AndL src mask));
10733 
10734   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10735   ins_encode %{
10736     // movzbl zeroes out the upper 32-bit and does not need REX.W
10737     __ movzbl($dst$$Register, $src$$Register);
10738   %}
10739   ins_pipe(ialu_reg);
10740 %}
10741 
10742 // And Register with Immediate 65535
10743 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10744 %{
10745   match(Set dst (AndL src mask));
10746 
10747   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10748   ins_encode %{
10749     // movzwl zeroes out the upper 32-bit and does not need REX.W
10750     __ movzwl($dst$$Register, $src$$Register);
10751   %}
10752   ins_pipe(ialu_reg);
10753 %}
10754 
10755 // And Register with Immediate
10756 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10757 %{
10758   predicate(!UseAPX);
10759   match(Set dst (AndL dst src));
10760   effect(KILL cr);
10761   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10762 
10763   format %{ "andq    $dst, $src\t# long" %}
10764   ins_encode %{
10765     __ andq($dst$$Register, $src$$constant);
10766   %}
10767   ins_pipe(ialu_reg);
10768 %}
10769 
10770 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10771 %{
10772   predicate(UseAPX);
10773   match(Set dst (AndL src1 src2));
10774   effect(KILL cr);
10775   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10776 
10777   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10778   ins_encode %{
10779     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10780   %}
10781   ins_pipe(ialu_reg);
10782 %}
10783 
10784 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10785 %{
10786   predicate(UseAPX);
10787   match(Set dst (AndL (LoadL src1) src2));
10788   effect(KILL cr);
10789   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10790 
10791   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10792   ins_encode %{
10793     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10794   %}
10795   ins_pipe(ialu_reg);
10796 %}
10797 
10798 // And Register with Memory
10799 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10800 %{
10801   predicate(!UseAPX);
10802   match(Set dst (AndL dst (LoadL src)));
10803   effect(KILL cr);
10804   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10805 
10806   ins_cost(150);
10807   format %{ "andq    $dst, $src\t# long" %}
10808   ins_encode %{
10809     __ andq($dst$$Register, $src$$Address);
10810   %}
10811   ins_pipe(ialu_reg_mem);
10812 %}
10813 
10814 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10815 %{
10816   predicate(UseAPX);
10817   match(Set dst (AndL src1 (LoadL src2)));
10818   effect(KILL cr);
10819   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10820 
10821   ins_cost(150);
10822   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10823   ins_encode %{
10824     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10825   %}
10826   ins_pipe(ialu_reg_mem);
10827 %}
10828 
10829 // And Memory with Register
10830 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10831 %{
10832   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10833   effect(KILL cr);
10834   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10835 
10836   ins_cost(150);
10837   format %{ "andq    $dst, $src\t# long" %}
10838   ins_encode %{
10839     __ andq($dst$$Address, $src$$Register);
10840   %}
10841   ins_pipe(ialu_mem_reg);
10842 %}
10843 
10844 // And Memory with Immediate
10845 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10846 %{
10847   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10848   effect(KILL cr);
10849   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10850 
10851   ins_cost(125);
10852   format %{ "andq    $dst, $src\t# long" %}
10853   ins_encode %{
10854     __ andq($dst$$Address, $src$$constant);
10855   %}
10856   ins_pipe(ialu_mem_imm);
10857 %}
10858 
10859 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10860 %{
10861   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10862   // because AND/OR works well enough for 8/32-bit values.
10863   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10864 
10865   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10866   effect(KILL cr);
10867 
10868   ins_cost(125);
10869   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10870   ins_encode %{
10871     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10872   %}
10873   ins_pipe(ialu_mem_imm);
10874 %}
10875 
10876 // BMI1 instructions
10877 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10878   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10879   predicate(UseBMI1Instructions);
10880   effect(KILL cr);
10881   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10882 
10883   ins_cost(125);
10884   format %{ "andnq  $dst, $src1, $src2" %}
10885 
10886   ins_encode %{
10887     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10888   %}
10889   ins_pipe(ialu_reg_mem);
10890 %}
10891 
10892 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10893   match(Set dst (AndL (XorL src1 minus_1) src2));
10894   predicate(UseBMI1Instructions);
10895   effect(KILL cr);
10896   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10897 
10898   format %{ "andnq  $dst, $src1, $src2" %}
10899 
10900   ins_encode %{
10901   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10902   %}
10903   ins_pipe(ialu_reg_mem);
10904 %}
10905 
10906 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10907   match(Set dst (AndL (SubL imm_zero src) src));
10908   predicate(UseBMI1Instructions);
10909   effect(KILL cr);
10910   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10911 
10912   format %{ "blsiq  $dst, $src" %}
10913 
10914   ins_encode %{
10915     __ blsiq($dst$$Register, $src$$Register);
10916   %}
10917   ins_pipe(ialu_reg);
10918 %}
10919 
10920 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10921   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10922   predicate(UseBMI1Instructions);
10923   effect(KILL cr);
10924   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10925 
10926   ins_cost(125);
10927   format %{ "blsiq  $dst, $src" %}
10928 
10929   ins_encode %{
10930     __ blsiq($dst$$Register, $src$$Address);
10931   %}
10932   ins_pipe(ialu_reg_mem);
10933 %}
10934 
10935 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10936 %{
10937   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10938   predicate(UseBMI1Instructions);
10939   effect(KILL cr);
10940   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10941 
10942   ins_cost(125);
10943   format %{ "blsmskq $dst, $src" %}
10944 
10945   ins_encode %{
10946     __ blsmskq($dst$$Register, $src$$Address);
10947   %}
10948   ins_pipe(ialu_reg_mem);
10949 %}
10950 
10951 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10952 %{
10953   match(Set dst (XorL (AddL src minus_1) src));
10954   predicate(UseBMI1Instructions);
10955   effect(KILL cr);
10956   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10957 
10958   format %{ "blsmskq $dst, $src" %}
10959 
10960   ins_encode %{
10961     __ blsmskq($dst$$Register, $src$$Register);
10962   %}
10963 
10964   ins_pipe(ialu_reg);
10965 %}
10966 
10967 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10968 %{
10969   match(Set dst (AndL (AddL src minus_1) src) );
10970   predicate(UseBMI1Instructions);
10971   effect(KILL cr);
10972   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10973 
10974   format %{ "blsrq  $dst, $src" %}
10975 
10976   ins_encode %{
10977     __ blsrq($dst$$Register, $src$$Register);
10978   %}
10979 
10980   ins_pipe(ialu_reg);
10981 %}
10982 
10983 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10984 %{
10985   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10986   predicate(UseBMI1Instructions);
10987   effect(KILL cr);
10988   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10989 
10990   ins_cost(125);
10991   format %{ "blsrq  $dst, $src" %}
10992 
10993   ins_encode %{
10994     __ blsrq($dst$$Register, $src$$Address);
10995   %}
10996 
10997   ins_pipe(ialu_reg);
10998 %}
10999 
11000 // Or Instructions
11001 // Or Register with Register
11002 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11003 %{
11004   predicate(!UseAPX);
11005   match(Set dst (OrL dst src));
11006   effect(KILL cr);
11007   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11008 
11009   format %{ "orq     $dst, $src\t# long" %}
11010   ins_encode %{
11011     __ orq($dst$$Register, $src$$Register);
11012   %}
11013   ins_pipe(ialu_reg_reg);
11014 %}
11015 
11016 // Or Register with Register using New Data Destination (NDD)
11017 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11018 %{
11019   predicate(UseAPX);
11020   match(Set dst (OrL src1 src2));
11021   effect(KILL cr);
11022   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11023 
11024   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11025   ins_encode %{
11026     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11027 
11028   %}
11029   ins_pipe(ialu_reg_reg);
11030 %}
11031 
11032 // Use any_RegP to match R15 (TLS register) without spilling.
11033 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11034   match(Set dst (OrL dst (CastP2X src)));
11035   effect(KILL cr);
11036   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11037 
11038   format %{ "orq     $dst, $src\t# long" %}
11039   ins_encode %{
11040     __ orq($dst$$Register, $src$$Register);
11041   %}
11042   ins_pipe(ialu_reg_reg);
11043 %}
11044 
11045 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11046   match(Set dst (OrL src1 (CastP2X src2)));
11047   effect(KILL cr);
11048   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11049 
11050   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11051   ins_encode %{
11052     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11053   %}
11054   ins_pipe(ialu_reg_reg);
11055 %}
11056 
11057 // Or Register with Immediate
11058 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11059 %{
11060   predicate(!UseAPX);
11061   match(Set dst (OrL dst src));
11062   effect(KILL cr);
11063   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11064 
11065   format %{ "orq     $dst, $src\t# long" %}
11066   ins_encode %{
11067     __ orq($dst$$Register, $src$$constant);
11068   %}
11069   ins_pipe(ialu_reg);
11070 %}
11071 
11072 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11073 %{
11074   predicate(UseAPX);
11075   match(Set dst (OrL src1 src2));
11076   effect(KILL cr);
11077   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11078 
11079   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11080   ins_encode %{
11081     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11082   %}
11083   ins_pipe(ialu_reg);
11084 %}
11085 
11086 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11087 %{
11088   predicate(UseAPX);
11089   match(Set dst (OrL src1 src2));
11090   effect(KILL cr);
11091   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11092 
11093   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11094   ins_encode %{
11095     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11096   %}
11097   ins_pipe(ialu_reg);
11098 %}
11099 
11100 // Or Memory with Immediate
11101 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11102 %{
11103   predicate(UseAPX);
11104   match(Set dst (OrL (LoadL src1) src2));
11105   effect(KILL cr);
11106   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11107 
11108   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11109   ins_encode %{
11110     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11111   %}
11112   ins_pipe(ialu_reg);
11113 %}
11114 
11115 // Or Register with Memory
11116 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11117 %{
11118   predicate(!UseAPX);
11119   match(Set dst (OrL dst (LoadL src)));
11120   effect(KILL cr);
11121   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11122 
11123   ins_cost(150);
11124   format %{ "orq     $dst, $src\t# long" %}
11125   ins_encode %{
11126     __ orq($dst$$Register, $src$$Address);
11127   %}
11128   ins_pipe(ialu_reg_mem);
11129 %}
11130 
11131 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11132 %{
11133   predicate(UseAPX);
11134   match(Set dst (OrL src1 (LoadL src2)));
11135   effect(KILL cr);
11136   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11137 
11138   ins_cost(150);
11139   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11140   ins_encode %{
11141     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11142   %}
11143   ins_pipe(ialu_reg_mem);
11144 %}
11145 
11146 // Or Memory with Register
11147 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11148 %{
11149   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11150   effect(KILL cr);
11151   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11152 
11153   ins_cost(150);
11154   format %{ "orq     $dst, $src\t# long" %}
11155   ins_encode %{
11156     __ orq($dst$$Address, $src$$Register);
11157   %}
11158   ins_pipe(ialu_mem_reg);
11159 %}
11160 
11161 // Or Memory with Immediate
11162 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11163 %{
11164   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11165   effect(KILL cr);
11166   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11167 
11168   ins_cost(125);
11169   format %{ "orq     $dst, $src\t# long" %}
11170   ins_encode %{
11171     __ orq($dst$$Address, $src$$constant);
11172   %}
11173   ins_pipe(ialu_mem_imm);
11174 %}
11175 
11176 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11177 %{
11178   // con should be a pure 64-bit power of 2 immediate
11179   // because AND/OR works well enough for 8/32-bit values.
11180   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11181 
11182   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11183   effect(KILL cr);
11184 
11185   ins_cost(125);
11186   format %{ "btsq    $dst, log2($con)\t# long" %}
11187   ins_encode %{
11188     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11189   %}
11190   ins_pipe(ialu_mem_imm);
11191 %}
11192 
11193 // Xor Instructions
11194 // Xor Register with Register
11195 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11196 %{
11197   predicate(!UseAPX);
11198   match(Set dst (XorL dst src));
11199   effect(KILL cr);
11200   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11201 
11202   format %{ "xorq    $dst, $src\t# long" %}
11203   ins_encode %{
11204     __ xorq($dst$$Register, $src$$Register);
11205   %}
11206   ins_pipe(ialu_reg_reg);
11207 %}
11208 
11209 // Xor Register with Register using New Data Destination (NDD)
11210 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11211 %{
11212   predicate(UseAPX);
11213   match(Set dst (XorL src1 src2));
11214   effect(KILL cr);
11215   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11216 
11217   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11218   ins_encode %{
11219     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11220   %}
11221   ins_pipe(ialu_reg_reg);
11222 %}
11223 
11224 // Xor Register with Immediate -1
11225 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11226 %{
11227   predicate(!UseAPX);
11228   match(Set dst (XorL dst imm));
11229 
11230   format %{ "notq   $dst" %}
11231   ins_encode %{
11232      __ notq($dst$$Register);
11233   %}
11234   ins_pipe(ialu_reg);
11235 %}
11236 
11237 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11238 %{
11239   predicate(UseAPX);
11240   match(Set dst (XorL src imm));
11241 
11242   format %{ "enotq   $dst, $src" %}
11243   ins_encode %{
11244     __ enotq($dst$$Register, $src$$Register);
11245   %}
11246   ins_pipe(ialu_reg);
11247 %}
11248 
11249 // Xor Register with Immediate
11250 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11251 %{
11252   // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
11253   predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11254   match(Set dst (XorL dst src));
11255   effect(KILL cr);
11256   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11257 
11258   format %{ "xorq    $dst, $src\t# long" %}
11259   ins_encode %{
11260     __ xorq($dst$$Register, $src$$constant);
11261   %}
11262   ins_pipe(ialu_reg);
11263 %}
11264 
11265 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11266 %{
11267   // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
11268   predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11269   match(Set dst (XorL src1 src2));
11270   effect(KILL cr);
11271   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11272 
11273   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11274   ins_encode %{
11275     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11276   %}
11277   ins_pipe(ialu_reg);
11278 %}
11279 
11280 // Xor Memory with Immediate
11281 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11282 %{
11283   predicate(UseAPX);
11284   match(Set dst (XorL (LoadL src1) src2));
11285   effect(KILL cr);
11286   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11287   ins_cost(150);
11288 
11289   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11290   ins_encode %{
11291     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11292   %}
11293   ins_pipe(ialu_reg);
11294 %}
11295 
11296 // Xor Register with Memory
11297 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11298 %{
11299   predicate(!UseAPX);
11300   match(Set dst (XorL dst (LoadL src)));
11301   effect(KILL cr);
11302   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11303 
11304   ins_cost(150);
11305   format %{ "xorq    $dst, $src\t# long" %}
11306   ins_encode %{
11307     __ xorq($dst$$Register, $src$$Address);
11308   %}
11309   ins_pipe(ialu_reg_mem);
11310 %}
11311 
11312 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11313 %{
11314   predicate(UseAPX);
11315   match(Set dst (XorL src1 (LoadL src2)));
11316   effect(KILL cr);
11317   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11318 
11319   ins_cost(150);
11320   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11321   ins_encode %{
11322     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11323   %}
11324   ins_pipe(ialu_reg_mem);
11325 %}
11326 
11327 // Xor Memory with Register
11328 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11329 %{
11330   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11331   effect(KILL cr);
11332   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11333 
11334   ins_cost(150);
11335   format %{ "xorq    $dst, $src\t# long" %}
11336   ins_encode %{
11337     __ xorq($dst$$Address, $src$$Register);
11338   %}
11339   ins_pipe(ialu_mem_reg);
11340 %}
11341 
11342 // Xor Memory with Immediate
11343 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11344 %{
11345   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11346   effect(KILL cr);
11347   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11348 
11349   ins_cost(125);
11350   format %{ "xorq    $dst, $src\t# long" %}
11351   ins_encode %{
11352     __ xorq($dst$$Address, $src$$constant);
11353   %}
11354   ins_pipe(ialu_mem_imm);
11355 %}
11356 
11357 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11358 %{
11359   match(Set dst (CmpLTMask p q));
11360   effect(KILL cr);
11361 
11362   ins_cost(400);
11363   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11364             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11365             "negl    $dst" %}
11366   ins_encode %{
11367     __ cmpl($p$$Register, $q$$Register);
11368     __ setcc(Assembler::less, $dst$$Register);
11369     __ negl($dst$$Register);
11370   %}
11371   ins_pipe(pipe_slow);
11372 %}
11373 
11374 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11375 %{
11376   match(Set dst (CmpLTMask dst zero));
11377   effect(KILL cr);
11378 
11379   ins_cost(100);
11380   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11381   ins_encode %{
11382     __ sarl($dst$$Register, 31);
11383   %}
11384   ins_pipe(ialu_reg);
11385 %}
11386 
11387 /* Better to save a register than avoid a branch */
11388 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11389 %{
11390   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11391   effect(KILL cr);
11392   ins_cost(300);
11393   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11394             "jge     done\n\t"
11395             "addl    $p,$y\n"
11396             "done:   " %}
11397   ins_encode %{
11398     Register Rp = $p$$Register;
11399     Register Rq = $q$$Register;
11400     Register Ry = $y$$Register;
11401     Label done;
11402     __ subl(Rp, Rq);
11403     __ jccb(Assembler::greaterEqual, done);
11404     __ addl(Rp, Ry);
11405     __ bind(done);
11406   %}
11407   ins_pipe(pipe_cmplt);
11408 %}
11409 
11410 /* Better to save a register than avoid a branch */
11411 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11412 %{
11413   match(Set y (AndI (CmpLTMask p q) y));
11414   effect(KILL cr);
11415 
11416   ins_cost(300);
11417 
11418   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11419             "jlt     done\n\t"
11420             "xorl    $y, $y\n"
11421             "done:   " %}
11422   ins_encode %{
11423     Register Rp = $p$$Register;
11424     Register Rq = $q$$Register;
11425     Register Ry = $y$$Register;
11426     Label done;
11427     __ cmpl(Rp, Rq);
11428     __ jccb(Assembler::less, done);
11429     __ xorl(Ry, Ry);
11430     __ bind(done);
11431   %}
11432   ins_pipe(pipe_cmplt);
11433 %}
11434 
11435 
11436 //---------- FP Instructions------------------------------------------------
11437 
11438 // Really expensive, avoid
11439 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11440 %{
11441   match(Set cr (CmpF src1 src2));
11442 
11443   ins_cost(500);
11444   format %{ "ucomiss $src1, $src2\n\t"
11445             "jnp,s   exit\n\t"
11446             "pushfq\t# saw NaN, set CF\n\t"
11447             "andq    [rsp], #0xffffff2b\n\t"
11448             "popfq\n"
11449     "exit:" %}
11450   ins_encode %{
11451     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11452     emit_cmpfp_fixup(masm);
11453   %}
11454   ins_pipe(pipe_slow);
11455 %}
11456 
11457 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11458   match(Set cr (CmpF src1 src2));
11459 
11460   ins_cost(100);
11461   format %{ "ucomiss $src1, $src2" %}
11462   ins_encode %{
11463     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11464   %}
11465   ins_pipe(pipe_slow);
11466 %}
11467 
11468 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11469   match(Set cr (CmpF src1 (LoadF src2)));
11470 
11471   ins_cost(100);
11472   format %{ "ucomiss $src1, $src2" %}
11473   ins_encode %{
11474     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11475   %}
11476   ins_pipe(pipe_slow);
11477 %}
11478 
11479 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11480   match(Set cr (CmpF src con));
11481   ins_cost(100);
11482   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11483   ins_encode %{
11484     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11485   %}
11486   ins_pipe(pipe_slow);
11487 %}
11488 
11489 // Really expensive, avoid
11490 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11491 %{
11492   match(Set cr (CmpD src1 src2));
11493 
11494   ins_cost(500);
11495   format %{ "ucomisd $src1, $src2\n\t"
11496             "jnp,s   exit\n\t"
11497             "pushfq\t# saw NaN, set CF\n\t"
11498             "andq    [rsp], #0xffffff2b\n\t"
11499             "popfq\n"
11500     "exit:" %}
11501   ins_encode %{
11502     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11503     emit_cmpfp_fixup(masm);
11504   %}
11505   ins_pipe(pipe_slow);
11506 %}
11507 
11508 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11509   match(Set cr (CmpD src1 src2));
11510 
11511   ins_cost(100);
11512   format %{ "ucomisd $src1, $src2 test" %}
11513   ins_encode %{
11514     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11515   %}
11516   ins_pipe(pipe_slow);
11517 %}
11518 
11519 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11520   match(Set cr (CmpD src1 (LoadD src2)));
11521 
11522   ins_cost(100);
11523   format %{ "ucomisd $src1, $src2" %}
11524   ins_encode %{
11525     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11526   %}
11527   ins_pipe(pipe_slow);
11528 %}
11529 
11530 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11531   match(Set cr (CmpD src con));
11532   ins_cost(100);
11533   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11534   ins_encode %{
11535     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11536   %}
11537   ins_pipe(pipe_slow);
11538 %}
11539 
11540 // Compare into -1,0,1
11541 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11542 %{
11543   match(Set dst (CmpF3 src1 src2));
11544   effect(KILL cr);
11545 
11546   ins_cost(275);
11547   format %{ "ucomiss $src1, $src2\n\t"
11548             "movl    $dst, #-1\n\t"
11549             "jp,s    done\n\t"
11550             "jb,s    done\n\t"
11551             "setne   $dst\n\t"
11552             "movzbl  $dst, $dst\n"
11553     "done:" %}
11554   ins_encode %{
11555     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11556     emit_cmpfp3(masm, $dst$$Register);
11557   %}
11558   ins_pipe(pipe_slow);
11559 %}
11560 
11561 // Compare into -1,0,1
11562 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11563 %{
11564   match(Set dst (CmpF3 src1 (LoadF src2)));
11565   effect(KILL cr);
11566 
11567   ins_cost(275);
11568   format %{ "ucomiss $src1, $src2\n\t"
11569             "movl    $dst, #-1\n\t"
11570             "jp,s    done\n\t"
11571             "jb,s    done\n\t"
11572             "setne   $dst\n\t"
11573             "movzbl  $dst, $dst\n"
11574     "done:" %}
11575   ins_encode %{
11576     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11577     emit_cmpfp3(masm, $dst$$Register);
11578   %}
11579   ins_pipe(pipe_slow);
11580 %}
11581 
11582 // Compare into -1,0,1
11583 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11584   match(Set dst (CmpF3 src con));
11585   effect(KILL cr);
11586 
11587   ins_cost(275);
11588   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11589             "movl    $dst, #-1\n\t"
11590             "jp,s    done\n\t"
11591             "jb,s    done\n\t"
11592             "setne   $dst\n\t"
11593             "movzbl  $dst, $dst\n"
11594     "done:" %}
11595   ins_encode %{
11596     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11597     emit_cmpfp3(masm, $dst$$Register);
11598   %}
11599   ins_pipe(pipe_slow);
11600 %}
11601 
11602 // Compare into -1,0,1
11603 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11604 %{
11605   match(Set dst (CmpD3 src1 src2));
11606   effect(KILL cr);
11607 
11608   ins_cost(275);
11609   format %{ "ucomisd $src1, $src2\n\t"
11610             "movl    $dst, #-1\n\t"
11611             "jp,s    done\n\t"
11612             "jb,s    done\n\t"
11613             "setne   $dst\n\t"
11614             "movzbl  $dst, $dst\n"
11615     "done:" %}
11616   ins_encode %{
11617     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11618     emit_cmpfp3(masm, $dst$$Register);
11619   %}
11620   ins_pipe(pipe_slow);
11621 %}
11622 
11623 // Compare into -1,0,1
11624 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11625 %{
11626   match(Set dst (CmpD3 src1 (LoadD src2)));
11627   effect(KILL cr);
11628 
11629   ins_cost(275);
11630   format %{ "ucomisd $src1, $src2\n\t"
11631             "movl    $dst, #-1\n\t"
11632             "jp,s    done\n\t"
11633             "jb,s    done\n\t"
11634             "setne   $dst\n\t"
11635             "movzbl  $dst, $dst\n"
11636     "done:" %}
11637   ins_encode %{
11638     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11639     emit_cmpfp3(masm, $dst$$Register);
11640   %}
11641   ins_pipe(pipe_slow);
11642 %}
11643 
11644 // Compare into -1,0,1
11645 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11646   match(Set dst (CmpD3 src con));
11647   effect(KILL cr);
11648 
11649   ins_cost(275);
11650   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11651             "movl    $dst, #-1\n\t"
11652             "jp,s    done\n\t"
11653             "jb,s    done\n\t"
11654             "setne   $dst\n\t"
11655             "movzbl  $dst, $dst\n"
11656     "done:" %}
11657   ins_encode %{
11658     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11659     emit_cmpfp3(masm, $dst$$Register);
11660   %}
11661   ins_pipe(pipe_slow);
11662 %}
11663 
11664 //----------Arithmetic Conversion Instructions---------------------------------
11665 
11666 instruct convF2D_reg_reg(regD dst, regF src)
11667 %{
11668   match(Set dst (ConvF2D src));
11669 
11670   format %{ "cvtss2sd $dst, $src" %}
11671   ins_encode %{
11672     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11673   %}
11674   ins_pipe(pipe_slow); // XXX
11675 %}
11676 
11677 instruct convF2D_reg_mem(regD dst, memory src)
11678 %{
11679   predicate(UseAVX == 0);
11680   match(Set dst (ConvF2D (LoadF src)));
11681 
11682   format %{ "cvtss2sd $dst, $src" %}
11683   ins_encode %{
11684     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11685   %}
11686   ins_pipe(pipe_slow); // XXX
11687 %}
11688 
11689 instruct convD2F_reg_reg(regF dst, regD src)
11690 %{
11691   match(Set dst (ConvD2F src));
11692 
11693   format %{ "cvtsd2ss $dst, $src" %}
11694   ins_encode %{
11695     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11696   %}
11697   ins_pipe(pipe_slow); // XXX
11698 %}
11699 
11700 instruct convD2F_reg_mem(regF dst, memory src)
11701 %{
11702   predicate(UseAVX == 0);
11703   match(Set dst (ConvD2F (LoadD src)));
11704 
11705   format %{ "cvtsd2ss $dst, $src" %}
11706   ins_encode %{
11707     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11708   %}
11709   ins_pipe(pipe_slow); // XXX
11710 %}
11711 
11712 // XXX do mem variants
11713 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11714 %{
11715   match(Set dst (ConvF2I src));
11716   effect(KILL cr);
11717   format %{ "convert_f2i $dst, $src" %}
11718   ins_encode %{
11719     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11720   %}
11721   ins_pipe(pipe_slow);
11722 %}
11723 
11724 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11725 %{
11726   match(Set dst (ConvF2L src));
11727   effect(KILL cr);
11728   format %{ "convert_f2l $dst, $src"%}
11729   ins_encode %{
11730     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11731   %}
11732   ins_pipe(pipe_slow);
11733 %}
11734 
11735 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11736 %{
11737   match(Set dst (ConvD2I src));
11738   effect(KILL cr);
11739   format %{ "convert_d2i $dst, $src"%}
11740   ins_encode %{
11741     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11742   %}
11743   ins_pipe(pipe_slow);
11744 %}
11745 
11746 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11747 %{
11748   match(Set dst (ConvD2L src));
11749   effect(KILL cr);
11750   format %{ "convert_d2l $dst, $src"%}
11751   ins_encode %{
11752     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11753   %}
11754   ins_pipe(pipe_slow);
11755 %}
11756 
11757 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11758 %{
11759   match(Set dst (RoundD src));
11760   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11761   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11762   ins_encode %{
11763     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11764   %}
11765   ins_pipe(pipe_slow);
11766 %}
11767 
11768 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11769 %{
11770   match(Set dst (RoundF src));
11771   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11772   format %{ "round_float $dst,$src" %}
11773   ins_encode %{
11774     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11775   %}
11776   ins_pipe(pipe_slow);
11777 %}
11778 
11779 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11780 %{
11781   predicate(!UseXmmI2F);
11782   match(Set dst (ConvI2F src));
11783 
11784   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11785   ins_encode %{
11786     if (UseAVX > 0) {
11787       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11788     }
11789     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11790   %}
11791   ins_pipe(pipe_slow); // XXX
11792 %}
11793 
11794 instruct convI2F_reg_mem(regF dst, memory src)
11795 %{
11796   predicate(UseAVX == 0);
11797   match(Set dst (ConvI2F (LoadI src)));
11798 
11799   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11800   ins_encode %{
11801     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11802   %}
11803   ins_pipe(pipe_slow); // XXX
11804 %}
11805 
11806 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11807 %{
11808   predicate(!UseXmmI2D);
11809   match(Set dst (ConvI2D src));
11810 
11811   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11812   ins_encode %{
11813     if (UseAVX > 0) {
11814       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11815     }
11816     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11817   %}
11818   ins_pipe(pipe_slow); // XXX
11819 %}
11820 
11821 instruct convI2D_reg_mem(regD dst, memory src)
11822 %{
11823   predicate(UseAVX == 0);
11824   match(Set dst (ConvI2D (LoadI src)));
11825 
11826   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11827   ins_encode %{
11828     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11829   %}
11830   ins_pipe(pipe_slow); // XXX
11831 %}
11832 
11833 instruct convXI2F_reg(regF dst, rRegI src)
11834 %{
11835   predicate(UseXmmI2F);
11836   match(Set dst (ConvI2F src));
11837 
11838   format %{ "movdl $dst, $src\n\t"
11839             "cvtdq2psl $dst, $dst\t# i2f" %}
11840   ins_encode %{
11841     __ movdl($dst$$XMMRegister, $src$$Register);
11842     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11843   %}
11844   ins_pipe(pipe_slow); // XXX
11845 %}
11846 
11847 instruct convXI2D_reg(regD dst, rRegI src)
11848 %{
11849   predicate(UseXmmI2D);
11850   match(Set dst (ConvI2D src));
11851 
11852   format %{ "movdl $dst, $src\n\t"
11853             "cvtdq2pdl $dst, $dst\t# i2d" %}
11854   ins_encode %{
11855     __ movdl($dst$$XMMRegister, $src$$Register);
11856     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11857   %}
11858   ins_pipe(pipe_slow); // XXX
11859 %}
11860 
11861 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11862 %{
11863   match(Set dst (ConvL2F src));
11864 
11865   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11866   ins_encode %{
11867     if (UseAVX > 0) {
11868       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11869     }
11870     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11871   %}
11872   ins_pipe(pipe_slow); // XXX
11873 %}
11874 
11875 instruct convL2F_reg_mem(regF dst, memory src)
11876 %{
11877   predicate(UseAVX == 0);
11878   match(Set dst (ConvL2F (LoadL src)));
11879 
11880   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11881   ins_encode %{
11882     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11883   %}
11884   ins_pipe(pipe_slow); // XXX
11885 %}
11886 
11887 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11888 %{
11889   match(Set dst (ConvL2D src));
11890 
11891   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11892   ins_encode %{
11893     if (UseAVX > 0) {
11894       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11895     }
11896     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11897   %}
11898   ins_pipe(pipe_slow); // XXX
11899 %}
11900 
11901 instruct convL2D_reg_mem(regD dst, memory src)
11902 %{
11903   predicate(UseAVX == 0);
11904   match(Set dst (ConvL2D (LoadL src)));
11905 
11906   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11907   ins_encode %{
11908     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11909   %}
11910   ins_pipe(pipe_slow); // XXX
11911 %}
11912 
11913 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11914 %{
11915   match(Set dst (ConvI2L src));
11916 
11917   ins_cost(125);
11918   format %{ "movslq  $dst, $src\t# i2l" %}
11919   ins_encode %{
11920     __ movslq($dst$$Register, $src$$Register);
11921   %}
11922   ins_pipe(ialu_reg_reg);
11923 %}
11924 
11925 // Zero-extend convert int to long
11926 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11927 %{
11928   match(Set dst (AndL (ConvI2L src) mask));
11929 
11930   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11931   ins_encode %{
11932     if ($dst$$reg != $src$$reg) {
11933       __ movl($dst$$Register, $src$$Register);
11934     }
11935   %}
11936   ins_pipe(ialu_reg_reg);
11937 %}
11938 
11939 // Zero-extend convert int to long
11940 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11941 %{
11942   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11943 
11944   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11945   ins_encode %{
11946     __ movl($dst$$Register, $src$$Address);
11947   %}
11948   ins_pipe(ialu_reg_mem);
11949 %}
11950 
11951 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11952 %{
11953   match(Set dst (AndL src mask));
11954 
11955   format %{ "movl    $dst, $src\t# zero-extend long" %}
11956   ins_encode %{
11957     __ movl($dst$$Register, $src$$Register);
11958   %}
11959   ins_pipe(ialu_reg_reg);
11960 %}
11961 
11962 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11963 %{
11964   match(Set dst (ConvL2I src));
11965 
11966   format %{ "movl    $dst, $src\t# l2i" %}
11967   ins_encode %{
11968     __ movl($dst$$Register, $src$$Register);
11969   %}
11970   ins_pipe(ialu_reg_reg);
11971 %}
11972 
11973 
11974 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11975   match(Set dst (MoveF2I src));
11976   effect(DEF dst, USE src);
11977 
11978   ins_cost(125);
11979   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11980   ins_encode %{
11981     __ movl($dst$$Register, Address(rsp, $src$$disp));
11982   %}
11983   ins_pipe(ialu_reg_mem);
11984 %}
11985 
11986 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11987   match(Set dst (MoveI2F src));
11988   effect(DEF dst, USE src);
11989 
11990   ins_cost(125);
11991   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11992   ins_encode %{
11993     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11994   %}
11995   ins_pipe(pipe_slow);
11996 %}
11997 
11998 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11999   match(Set dst (MoveD2L src));
12000   effect(DEF dst, USE src);
12001 
12002   ins_cost(125);
12003   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12004   ins_encode %{
12005     __ movq($dst$$Register, Address(rsp, $src$$disp));
12006   %}
12007   ins_pipe(ialu_reg_mem);
12008 %}
12009 
12010 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12011   predicate(!UseXmmLoadAndClearUpper);
12012   match(Set dst (MoveL2D src));
12013   effect(DEF dst, USE src);
12014 
12015   ins_cost(125);
12016   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12017   ins_encode %{
12018     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12019   %}
12020   ins_pipe(pipe_slow);
12021 %}
12022 
12023 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12024   predicate(UseXmmLoadAndClearUpper);
12025   match(Set dst (MoveL2D src));
12026   effect(DEF dst, USE src);
12027 
12028   ins_cost(125);
12029   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12030   ins_encode %{
12031     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12032   %}
12033   ins_pipe(pipe_slow);
12034 %}
12035 
12036 
12037 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12038   match(Set dst (MoveF2I src));
12039   effect(DEF dst, USE src);
12040 
12041   ins_cost(95); // XXX
12042   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12043   ins_encode %{
12044     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12045   %}
12046   ins_pipe(pipe_slow);
12047 %}
12048 
12049 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12050   match(Set dst (MoveI2F src));
12051   effect(DEF dst, USE src);
12052 
12053   ins_cost(100);
12054   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12055   ins_encode %{
12056     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12057   %}
12058   ins_pipe( ialu_mem_reg );
12059 %}
12060 
12061 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12062   match(Set dst (MoveD2L src));
12063   effect(DEF dst, USE src);
12064 
12065   ins_cost(95); // XXX
12066   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12067   ins_encode %{
12068     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12069   %}
12070   ins_pipe(pipe_slow);
12071 %}
12072 
12073 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12074   match(Set dst (MoveL2D src));
12075   effect(DEF dst, USE src);
12076 
12077   ins_cost(100);
12078   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12079   ins_encode %{
12080     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12081   %}
12082   ins_pipe(ialu_mem_reg);
12083 %}
12084 
12085 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12086   match(Set dst (MoveF2I src));
12087   effect(DEF dst, USE src);
12088   ins_cost(85);
12089   format %{ "movd    $dst,$src\t# MoveF2I" %}
12090   ins_encode %{
12091     __ movdl($dst$$Register, $src$$XMMRegister);
12092   %}
12093   ins_pipe( pipe_slow );
12094 %}
12095 
12096 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12097   match(Set dst (MoveD2L src));
12098   effect(DEF dst, USE src);
12099   ins_cost(85);
12100   format %{ "movd    $dst,$src\t# MoveD2L" %}
12101   ins_encode %{
12102     __ movdq($dst$$Register, $src$$XMMRegister);
12103   %}
12104   ins_pipe( pipe_slow );
12105 %}
12106 
12107 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12108   match(Set dst (MoveI2F src));
12109   effect(DEF dst, USE src);
12110   ins_cost(100);
12111   format %{ "movd    $dst,$src\t# MoveI2F" %}
12112   ins_encode %{
12113     __ movdl($dst$$XMMRegister, $src$$Register);
12114   %}
12115   ins_pipe( pipe_slow );
12116 %}
12117 
12118 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12119   match(Set dst (MoveL2D src));
12120   effect(DEF dst, USE src);
12121   ins_cost(100);
12122   format %{ "movd    $dst,$src\t# MoveL2D" %}
12123   ins_encode %{
12124      __ movdq($dst$$XMMRegister, $src$$Register);
12125   %}
12126   ins_pipe( pipe_slow );
12127 %}
12128 
12129 // Fast clearing of an array
12130 // Small non-constant lenght ClearArray for non-AVX512 targets.
12131 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12132                   Universe dummy, rFlagsReg cr)
12133 %{
12134   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12135   match(Set dummy (ClearArray cnt base));
12136   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12137 
12138   format %{ $$template
12139     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12140     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12141     $$emit$$"jg      LARGE\n\t"
12142     $$emit$$"dec     rcx\n\t"
12143     $$emit$$"js      DONE\t# Zero length\n\t"
12144     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12145     $$emit$$"dec     rcx\n\t"
12146     $$emit$$"jge     LOOP\n\t"
12147     $$emit$$"jmp     DONE\n\t"
12148     $$emit$$"# LARGE:\n\t"
12149     if (UseFastStosb) {
12150        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12151        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12152     } else if (UseXMMForObjInit) {
12153        $$emit$$"mov     rdi,rax\n\t"
12154        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12155        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12156        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12157        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12158        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12159        $$emit$$"add     0x40,rax\n\t"
12160        $$emit$$"# L_zero_64_bytes:\n\t"
12161        $$emit$$"sub     0x8,rcx\n\t"
12162        $$emit$$"jge     L_loop\n\t"
12163        $$emit$$"add     0x4,rcx\n\t"
12164        $$emit$$"jl      L_tail\n\t"
12165        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12166        $$emit$$"add     0x20,rax\n\t"
12167        $$emit$$"sub     0x4,rcx\n\t"
12168        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12169        $$emit$$"add     0x4,rcx\n\t"
12170        $$emit$$"jle     L_end\n\t"
12171        $$emit$$"dec     rcx\n\t"
12172        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12173        $$emit$$"vmovq   xmm0,(rax)\n\t"
12174        $$emit$$"add     0x8,rax\n\t"
12175        $$emit$$"dec     rcx\n\t"
12176        $$emit$$"jge     L_sloop\n\t"
12177        $$emit$$"# L_end:\n\t"
12178     } else {
12179        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12180     }
12181     $$emit$$"# DONE"
12182   %}
12183   ins_encode %{
12184     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12185                  $tmp$$XMMRegister, false, knoreg);
12186   %}
12187   ins_pipe(pipe_slow);
12188 %}
12189 
12190 // Small non-constant length ClearArray for AVX512 targets.
12191 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12192                        Universe dummy, rFlagsReg cr)
12193 %{
12194   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12195   match(Set dummy (ClearArray cnt base));
12196   ins_cost(125);
12197   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12198 
12199   format %{ $$template
12200     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12201     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12202     $$emit$$"jg      LARGE\n\t"
12203     $$emit$$"dec     rcx\n\t"
12204     $$emit$$"js      DONE\t# Zero length\n\t"
12205     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12206     $$emit$$"dec     rcx\n\t"
12207     $$emit$$"jge     LOOP\n\t"
12208     $$emit$$"jmp     DONE\n\t"
12209     $$emit$$"# LARGE:\n\t"
12210     if (UseFastStosb) {
12211        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12212        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12213     } else if (UseXMMForObjInit) {
12214        $$emit$$"mov     rdi,rax\n\t"
12215        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12216        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12217        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12218        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12219        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12220        $$emit$$"add     0x40,rax\n\t"
12221        $$emit$$"# L_zero_64_bytes:\n\t"
12222        $$emit$$"sub     0x8,rcx\n\t"
12223        $$emit$$"jge     L_loop\n\t"
12224        $$emit$$"add     0x4,rcx\n\t"
12225        $$emit$$"jl      L_tail\n\t"
12226        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12227        $$emit$$"add     0x20,rax\n\t"
12228        $$emit$$"sub     0x4,rcx\n\t"
12229        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12230        $$emit$$"add     0x4,rcx\n\t"
12231        $$emit$$"jle     L_end\n\t"
12232        $$emit$$"dec     rcx\n\t"
12233        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12234        $$emit$$"vmovq   xmm0,(rax)\n\t"
12235        $$emit$$"add     0x8,rax\n\t"
12236        $$emit$$"dec     rcx\n\t"
12237        $$emit$$"jge     L_sloop\n\t"
12238        $$emit$$"# L_end:\n\t"
12239     } else {
12240        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12241     }
12242     $$emit$$"# DONE"
12243   %}
12244   ins_encode %{
12245     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12246                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12247   %}
12248   ins_pipe(pipe_slow);
12249 %}
12250 
12251 // Large non-constant length ClearArray for non-AVX512 targets.
12252 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12253                         Universe dummy, rFlagsReg cr)
12254 %{
12255   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12256   match(Set dummy (ClearArray cnt base));
12257   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12258 
12259   format %{ $$template
12260     if (UseFastStosb) {
12261        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12262        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12263        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12264     } else if (UseXMMForObjInit) {
12265        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12266        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12267        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12268        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12269        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12270        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12271        $$emit$$"add     0x40,rax\n\t"
12272        $$emit$$"# L_zero_64_bytes:\n\t"
12273        $$emit$$"sub     0x8,rcx\n\t"
12274        $$emit$$"jge     L_loop\n\t"
12275        $$emit$$"add     0x4,rcx\n\t"
12276        $$emit$$"jl      L_tail\n\t"
12277        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12278        $$emit$$"add     0x20,rax\n\t"
12279        $$emit$$"sub     0x4,rcx\n\t"
12280        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12281        $$emit$$"add     0x4,rcx\n\t"
12282        $$emit$$"jle     L_end\n\t"
12283        $$emit$$"dec     rcx\n\t"
12284        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12285        $$emit$$"vmovq   xmm0,(rax)\n\t"
12286        $$emit$$"add     0x8,rax\n\t"
12287        $$emit$$"dec     rcx\n\t"
12288        $$emit$$"jge     L_sloop\n\t"
12289        $$emit$$"# L_end:\n\t"
12290     } else {
12291        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12292        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12293     }
12294   %}
12295   ins_encode %{
12296     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12297                  $tmp$$XMMRegister, true, knoreg);
12298   %}
12299   ins_pipe(pipe_slow);
12300 %}
12301 
12302 // Large non-constant length ClearArray for AVX512 targets.
12303 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12304                              Universe dummy, rFlagsReg cr)
12305 %{
12306   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12307   match(Set dummy (ClearArray cnt base));
12308   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12309 
12310   format %{ $$template
12311     if (UseFastStosb) {
12312        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12313        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12314        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12315     } else if (UseXMMForObjInit) {
12316        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12317        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12318        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12319        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12320        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12321        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12322        $$emit$$"add     0x40,rax\n\t"
12323        $$emit$$"# L_zero_64_bytes:\n\t"
12324        $$emit$$"sub     0x8,rcx\n\t"
12325        $$emit$$"jge     L_loop\n\t"
12326        $$emit$$"add     0x4,rcx\n\t"
12327        $$emit$$"jl      L_tail\n\t"
12328        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12329        $$emit$$"add     0x20,rax\n\t"
12330        $$emit$$"sub     0x4,rcx\n\t"
12331        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12332        $$emit$$"add     0x4,rcx\n\t"
12333        $$emit$$"jle     L_end\n\t"
12334        $$emit$$"dec     rcx\n\t"
12335        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12336        $$emit$$"vmovq   xmm0,(rax)\n\t"
12337        $$emit$$"add     0x8,rax\n\t"
12338        $$emit$$"dec     rcx\n\t"
12339        $$emit$$"jge     L_sloop\n\t"
12340        $$emit$$"# L_end:\n\t"
12341     } else {
12342        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12343        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12344     }
12345   %}
12346   ins_encode %{
12347     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12348                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12349   %}
12350   ins_pipe(pipe_slow);
12351 %}
12352 
12353 // Small constant length ClearArray for AVX512 targets.
12354 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12355 %{
12356   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12357   match(Set dummy (ClearArray cnt base));
12358   ins_cost(100);
12359   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12360   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12361   ins_encode %{
12362    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12363   %}
12364   ins_pipe(pipe_slow);
12365 %}
12366 
12367 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12368                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12369 %{
12370   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12371   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12372   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12373 
12374   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12375   ins_encode %{
12376     __ string_compare($str1$$Register, $str2$$Register,
12377                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12378                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12379   %}
12380   ins_pipe( pipe_slow );
12381 %}
12382 
12383 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12384                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12385 %{
12386   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12387   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12388   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12389 
12390   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12391   ins_encode %{
12392     __ string_compare($str1$$Register, $str2$$Register,
12393                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12394                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12395   %}
12396   ins_pipe( pipe_slow );
12397 %}
12398 
12399 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12400                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12401 %{
12402   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12403   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12404   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12405 
12406   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12407   ins_encode %{
12408     __ string_compare($str1$$Register, $str2$$Register,
12409                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12410                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12411   %}
12412   ins_pipe( pipe_slow );
12413 %}
12414 
12415 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12416                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12417 %{
12418   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12419   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12420   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12421 
12422   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12423   ins_encode %{
12424     __ string_compare($str1$$Register, $str2$$Register,
12425                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12426                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12427   %}
12428   ins_pipe( pipe_slow );
12429 %}
12430 
12431 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12432                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12433 %{
12434   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12435   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12436   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12437 
12438   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12439   ins_encode %{
12440     __ string_compare($str1$$Register, $str2$$Register,
12441                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12442                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12443   %}
12444   ins_pipe( pipe_slow );
12445 %}
12446 
12447 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12448                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12449 %{
12450   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12451   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12452   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12453 
12454   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12455   ins_encode %{
12456     __ string_compare($str1$$Register, $str2$$Register,
12457                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12458                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12459   %}
12460   ins_pipe( pipe_slow );
12461 %}
12462 
12463 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12464                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12465 %{
12466   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12467   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12468   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12469 
12470   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12471   ins_encode %{
12472     __ string_compare($str2$$Register, $str1$$Register,
12473                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12474                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12475   %}
12476   ins_pipe( pipe_slow );
12477 %}
12478 
12479 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12480                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12481 %{
12482   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12483   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12484   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12485 
12486   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12487   ins_encode %{
12488     __ string_compare($str2$$Register, $str1$$Register,
12489                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12490                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12491   %}
12492   ins_pipe( pipe_slow );
12493 %}
12494 
12495 // fast search of substring with known size.
12496 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12497                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12498 %{
12499   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12500   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12501   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12502 
12503   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12504   ins_encode %{
12505     int icnt2 = (int)$int_cnt2$$constant;
12506     if (icnt2 >= 16) {
12507       // IndexOf for constant substrings with size >= 16 elements
12508       // which don't need to be loaded through stack.
12509       __ string_indexofC8($str1$$Register, $str2$$Register,
12510                           $cnt1$$Register, $cnt2$$Register,
12511                           icnt2, $result$$Register,
12512                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12513     } else {
12514       // Small strings are loaded through stack if they cross page boundary.
12515       __ string_indexof($str1$$Register, $str2$$Register,
12516                         $cnt1$$Register, $cnt2$$Register,
12517                         icnt2, $result$$Register,
12518                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12519     }
12520   %}
12521   ins_pipe( pipe_slow );
12522 %}
12523 
12524 // fast search of substring with known size.
12525 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12526                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12527 %{
12528   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12529   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12530   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12531 
12532   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12533   ins_encode %{
12534     int icnt2 = (int)$int_cnt2$$constant;
12535     if (icnt2 >= 8) {
12536       // IndexOf for constant substrings with size >= 8 elements
12537       // which don't need to be loaded through stack.
12538       __ string_indexofC8($str1$$Register, $str2$$Register,
12539                           $cnt1$$Register, $cnt2$$Register,
12540                           icnt2, $result$$Register,
12541                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12542     } else {
12543       // Small strings are loaded through stack if they cross page boundary.
12544       __ string_indexof($str1$$Register, $str2$$Register,
12545                         $cnt1$$Register, $cnt2$$Register,
12546                         icnt2, $result$$Register,
12547                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12548     }
12549   %}
12550   ins_pipe( pipe_slow );
12551 %}
12552 
12553 // fast search of substring with known size.
12554 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12555                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12556 %{
12557   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12558   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12559   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12560 
12561   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12562   ins_encode %{
12563     int icnt2 = (int)$int_cnt2$$constant;
12564     if (icnt2 >= 8) {
12565       // IndexOf for constant substrings with size >= 8 elements
12566       // which don't need to be loaded through stack.
12567       __ string_indexofC8($str1$$Register, $str2$$Register,
12568                           $cnt1$$Register, $cnt2$$Register,
12569                           icnt2, $result$$Register,
12570                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12571     } else {
12572       // Small strings are loaded through stack if they cross page boundary.
12573       __ string_indexof($str1$$Register, $str2$$Register,
12574                         $cnt1$$Register, $cnt2$$Register,
12575                         icnt2, $result$$Register,
12576                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12577     }
12578   %}
12579   ins_pipe( pipe_slow );
12580 %}
12581 
12582 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12583                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12584 %{
12585   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12586   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12587   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12588 
12589   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12590   ins_encode %{
12591     __ string_indexof($str1$$Register, $str2$$Register,
12592                       $cnt1$$Register, $cnt2$$Register,
12593                       (-1), $result$$Register,
12594                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12595   %}
12596   ins_pipe( pipe_slow );
12597 %}
12598 
12599 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12600                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12601 %{
12602   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12603   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12604   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12605 
12606   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12607   ins_encode %{
12608     __ string_indexof($str1$$Register, $str2$$Register,
12609                       $cnt1$$Register, $cnt2$$Register,
12610                       (-1), $result$$Register,
12611                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12612   %}
12613   ins_pipe( pipe_slow );
12614 %}
12615 
12616 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12617                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12618 %{
12619   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12620   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12621   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12622 
12623   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12624   ins_encode %{
12625     __ string_indexof($str1$$Register, $str2$$Register,
12626                       $cnt1$$Register, $cnt2$$Register,
12627                       (-1), $result$$Register,
12628                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12629   %}
12630   ins_pipe( pipe_slow );
12631 %}
12632 
12633 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12634                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12635 %{
12636   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12637   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12638   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12639   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12640   ins_encode %{
12641     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12642                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12643   %}
12644   ins_pipe( pipe_slow );
12645 %}
12646 
12647 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12648                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12649 %{
12650   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12651   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12652   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12653   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12654   ins_encode %{
12655     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12656                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12657   %}
12658   ins_pipe( pipe_slow );
12659 %}
12660 
12661 // fast string equals
12662 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12663                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12664 %{
12665   predicate(!VM_Version::supports_avx512vlbw());
12666   match(Set result (StrEquals (Binary str1 str2) cnt));
12667   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12668 
12669   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12670   ins_encode %{
12671     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12672                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12673                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12674   %}
12675   ins_pipe( pipe_slow );
12676 %}
12677 
12678 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12679                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12680 %{
12681   predicate(VM_Version::supports_avx512vlbw());
12682   match(Set result (StrEquals (Binary str1 str2) cnt));
12683   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12684 
12685   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12686   ins_encode %{
12687     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12688                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12689                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12690   %}
12691   ins_pipe( pipe_slow );
12692 %}
12693 
12694 // fast array equals
12695 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12696                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12697 %{
12698   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12699   match(Set result (AryEq ary1 ary2));
12700   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12701 
12702   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12703   ins_encode %{
12704     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12705                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12706                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12707   %}
12708   ins_pipe( pipe_slow );
12709 %}
12710 
12711 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12712                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12713 %{
12714   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12715   match(Set result (AryEq ary1 ary2));
12716   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12717 
12718   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12719   ins_encode %{
12720     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12721                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12722                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12723   %}
12724   ins_pipe( pipe_slow );
12725 %}
12726 
12727 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12728                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12729 %{
12730   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12731   match(Set result (AryEq ary1 ary2));
12732   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12733 
12734   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12735   ins_encode %{
12736     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12737                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12738                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12739   %}
12740   ins_pipe( pipe_slow );
12741 %}
12742 
12743 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12744                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12745 %{
12746   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12747   match(Set result (AryEq ary1 ary2));
12748   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12749 
12750   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12751   ins_encode %{
12752     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12753                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12754                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12755   %}
12756   ins_pipe( pipe_slow );
12757 %}
12758 
12759 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12760                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12761                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12762                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12763                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12764 %{
12765   predicate(UseAVX >= 2);
12766   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12767   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12768          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12769          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12770          USE basic_type, KILL cr);
12771 
12772   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12773   ins_encode %{
12774     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12775                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12776                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12777                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12778                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12779                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12780                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12781   %}
12782   ins_pipe( pipe_slow );
12783 %}
12784 
12785 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12786                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12787 %{
12788   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12789   match(Set result (CountPositives ary1 len));
12790   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12791 
12792   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12793   ins_encode %{
12794     __ count_positives($ary1$$Register, $len$$Register,
12795                        $result$$Register, $tmp3$$Register,
12796                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12797   %}
12798   ins_pipe( pipe_slow );
12799 %}
12800 
12801 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12802                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12803 %{
12804   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12805   match(Set result (CountPositives ary1 len));
12806   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12807 
12808   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12809   ins_encode %{
12810     __ count_positives($ary1$$Register, $len$$Register,
12811                        $result$$Register, $tmp3$$Register,
12812                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12813   %}
12814   ins_pipe( pipe_slow );
12815 %}
12816 
12817 // fast char[] to byte[] compression
12818 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12819                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12820   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12821   match(Set result (StrCompressedCopy src (Binary dst len)));
12822   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12823          USE_KILL len, KILL tmp5, KILL cr);
12824 
12825   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12826   ins_encode %{
12827     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12828                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12829                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12830                            knoreg, knoreg);
12831   %}
12832   ins_pipe( pipe_slow );
12833 %}
12834 
12835 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12836                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12837   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12838   match(Set result (StrCompressedCopy src (Binary dst len)));
12839   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12840          USE_KILL len, KILL tmp5, KILL cr);
12841 
12842   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12843   ins_encode %{
12844     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12845                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12846                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12847                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12848   %}
12849   ins_pipe( pipe_slow );
12850 %}
12851 // fast byte[] to char[] inflation
12852 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12853                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12854   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12855   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12856   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12857 
12858   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12859   ins_encode %{
12860     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12861                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12862   %}
12863   ins_pipe( pipe_slow );
12864 %}
12865 
12866 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12867                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12868   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12869   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12870   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12871 
12872   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12873   ins_encode %{
12874     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12875                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12876   %}
12877   ins_pipe( pipe_slow );
12878 %}
12879 
12880 // encode char[] to byte[] in ISO_8859_1
12881 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12882                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12883                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12884   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12885   match(Set result (EncodeISOArray src (Binary dst len)));
12886   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12887 
12888   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12889   ins_encode %{
12890     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12891                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12892                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12893   %}
12894   ins_pipe( pipe_slow );
12895 %}
12896 
12897 // encode char[] to byte[] in ASCII
12898 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12899                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12900                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12901   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12902   match(Set result (EncodeISOArray src (Binary dst len)));
12903   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12904 
12905   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12906   ins_encode %{
12907     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12908                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12909                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12910   %}
12911   ins_pipe( pipe_slow );
12912 %}
12913 
12914 //----------Overflow Math Instructions-----------------------------------------
12915 
12916 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12917 %{
12918   match(Set cr (OverflowAddI op1 op2));
12919   effect(DEF cr, USE_KILL op1, USE op2);
12920 
12921   format %{ "addl    $op1, $op2\t# overflow check int" %}
12922 
12923   ins_encode %{
12924     __ addl($op1$$Register, $op2$$Register);
12925   %}
12926   ins_pipe(ialu_reg_reg);
12927 %}
12928 
12929 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12930 %{
12931   match(Set cr (OverflowAddI op1 op2));
12932   effect(DEF cr, USE_KILL op1, USE op2);
12933 
12934   format %{ "addl    $op1, $op2\t# overflow check int" %}
12935 
12936   ins_encode %{
12937     __ addl($op1$$Register, $op2$$constant);
12938   %}
12939   ins_pipe(ialu_reg_reg);
12940 %}
12941 
12942 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12943 %{
12944   match(Set cr (OverflowAddL op1 op2));
12945   effect(DEF cr, USE_KILL op1, USE op2);
12946 
12947   format %{ "addq    $op1, $op2\t# overflow check long" %}
12948   ins_encode %{
12949     __ addq($op1$$Register, $op2$$Register);
12950   %}
12951   ins_pipe(ialu_reg_reg);
12952 %}
12953 
12954 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12955 %{
12956   match(Set cr (OverflowAddL op1 op2));
12957   effect(DEF cr, USE_KILL op1, USE op2);
12958 
12959   format %{ "addq    $op1, $op2\t# overflow check long" %}
12960   ins_encode %{
12961     __ addq($op1$$Register, $op2$$constant);
12962   %}
12963   ins_pipe(ialu_reg_reg);
12964 %}
12965 
12966 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12967 %{
12968   match(Set cr (OverflowSubI op1 op2));
12969 
12970   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12971   ins_encode %{
12972     __ cmpl($op1$$Register, $op2$$Register);
12973   %}
12974   ins_pipe(ialu_reg_reg);
12975 %}
12976 
12977 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12978 %{
12979   match(Set cr (OverflowSubI op1 op2));
12980 
12981   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12982   ins_encode %{
12983     __ cmpl($op1$$Register, $op2$$constant);
12984   %}
12985   ins_pipe(ialu_reg_reg);
12986 %}
12987 
12988 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12989 %{
12990   match(Set cr (OverflowSubL op1 op2));
12991 
12992   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12993   ins_encode %{
12994     __ cmpq($op1$$Register, $op2$$Register);
12995   %}
12996   ins_pipe(ialu_reg_reg);
12997 %}
12998 
12999 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13000 %{
13001   match(Set cr (OverflowSubL op1 op2));
13002 
13003   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13004   ins_encode %{
13005     __ cmpq($op1$$Register, $op2$$constant);
13006   %}
13007   ins_pipe(ialu_reg_reg);
13008 %}
13009 
13010 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13011 %{
13012   match(Set cr (OverflowSubI zero op2));
13013   effect(DEF cr, USE_KILL op2);
13014 
13015   format %{ "negl    $op2\t# overflow check int" %}
13016   ins_encode %{
13017     __ negl($op2$$Register);
13018   %}
13019   ins_pipe(ialu_reg_reg);
13020 %}
13021 
13022 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13023 %{
13024   match(Set cr (OverflowSubL zero op2));
13025   effect(DEF cr, USE_KILL op2);
13026 
13027   format %{ "negq    $op2\t# overflow check long" %}
13028   ins_encode %{
13029     __ negq($op2$$Register);
13030   %}
13031   ins_pipe(ialu_reg_reg);
13032 %}
13033 
13034 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13035 %{
13036   match(Set cr (OverflowMulI op1 op2));
13037   effect(DEF cr, USE_KILL op1, USE op2);
13038 
13039   format %{ "imull    $op1, $op2\t# overflow check int" %}
13040   ins_encode %{
13041     __ imull($op1$$Register, $op2$$Register);
13042   %}
13043   ins_pipe(ialu_reg_reg_alu0);
13044 %}
13045 
13046 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13047 %{
13048   match(Set cr (OverflowMulI op1 op2));
13049   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13050 
13051   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13052   ins_encode %{
13053     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13054   %}
13055   ins_pipe(ialu_reg_reg_alu0);
13056 %}
13057 
13058 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13059 %{
13060   match(Set cr (OverflowMulL op1 op2));
13061   effect(DEF cr, USE_KILL op1, USE op2);
13062 
13063   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13064   ins_encode %{
13065     __ imulq($op1$$Register, $op2$$Register);
13066   %}
13067   ins_pipe(ialu_reg_reg_alu0);
13068 %}
13069 
13070 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13071 %{
13072   match(Set cr (OverflowMulL op1 op2));
13073   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13074 
13075   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13076   ins_encode %{
13077     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13078   %}
13079   ins_pipe(ialu_reg_reg_alu0);
13080 %}
13081 
13082 
13083 //----------Control Flow Instructions------------------------------------------
13084 // Signed compare Instructions
13085 
13086 // XXX more variants!!
13087 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13088 %{
13089   match(Set cr (CmpI op1 op2));
13090   effect(DEF cr, USE op1, USE op2);
13091 
13092   format %{ "cmpl    $op1, $op2" %}
13093   ins_encode %{
13094     __ cmpl($op1$$Register, $op2$$Register);
13095   %}
13096   ins_pipe(ialu_cr_reg_reg);
13097 %}
13098 
13099 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13100 %{
13101   match(Set cr (CmpI op1 op2));
13102 
13103   format %{ "cmpl    $op1, $op2" %}
13104   ins_encode %{
13105     __ cmpl($op1$$Register, $op2$$constant);
13106   %}
13107   ins_pipe(ialu_cr_reg_imm);
13108 %}
13109 
13110 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13111 %{
13112   match(Set cr (CmpI op1 (LoadI op2)));
13113 
13114   ins_cost(500); // XXX
13115   format %{ "cmpl    $op1, $op2" %}
13116   ins_encode %{
13117     __ cmpl($op1$$Register, $op2$$Address);
13118   %}
13119   ins_pipe(ialu_cr_reg_mem);
13120 %}
13121 
13122 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13123 %{
13124   match(Set cr (CmpI src zero));
13125 
13126   format %{ "testl   $src, $src" %}
13127   ins_encode %{
13128     __ testl($src$$Register, $src$$Register);
13129   %}
13130   ins_pipe(ialu_cr_reg_imm);
13131 %}
13132 
13133 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13134 %{
13135   match(Set cr (CmpI (AndI src con) zero));
13136 
13137   format %{ "testl   $src, $con" %}
13138   ins_encode %{
13139     __ testl($src$$Register, $con$$constant);
13140   %}
13141   ins_pipe(ialu_cr_reg_imm);
13142 %}
13143 
13144 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13145 %{
13146   match(Set cr (CmpI (AndI src1 src2) zero));
13147 
13148   format %{ "testl   $src1, $src2" %}
13149   ins_encode %{
13150     __ testl($src1$$Register, $src2$$Register);
13151   %}
13152   ins_pipe(ialu_cr_reg_imm);
13153 %}
13154 
13155 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13156 %{
13157   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13158 
13159   format %{ "testl   $src, $mem" %}
13160   ins_encode %{
13161     __ testl($src$$Register, $mem$$Address);
13162   %}
13163   ins_pipe(ialu_cr_reg_mem);
13164 %}
13165 
13166 // Unsigned compare Instructions; really, same as signed except they
13167 // produce an rFlagsRegU instead of rFlagsReg.
13168 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13169 %{
13170   match(Set cr (CmpU op1 op2));
13171 
13172   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13173   ins_encode %{
13174     __ cmpl($op1$$Register, $op2$$Register);
13175   %}
13176   ins_pipe(ialu_cr_reg_reg);
13177 %}
13178 
13179 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13180 %{
13181   match(Set cr (CmpU op1 op2));
13182 
13183   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13184   ins_encode %{
13185     __ cmpl($op1$$Register, $op2$$constant);
13186   %}
13187   ins_pipe(ialu_cr_reg_imm);
13188 %}
13189 
13190 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13191 %{
13192   match(Set cr (CmpU op1 (LoadI op2)));
13193 
13194   ins_cost(500); // XXX
13195   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13196   ins_encode %{
13197     __ cmpl($op1$$Register, $op2$$Address);
13198   %}
13199   ins_pipe(ialu_cr_reg_mem);
13200 %}
13201 
13202 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13203 %{
13204   match(Set cr (CmpU src zero));
13205 
13206   format %{ "testl   $src, $src\t# unsigned" %}
13207   ins_encode %{
13208     __ testl($src$$Register, $src$$Register);
13209   %}
13210   ins_pipe(ialu_cr_reg_imm);
13211 %}
13212 
13213 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13214 %{
13215   match(Set cr (CmpP op1 op2));
13216 
13217   format %{ "cmpq    $op1, $op2\t# ptr" %}
13218   ins_encode %{
13219     __ cmpq($op1$$Register, $op2$$Register);
13220   %}
13221   ins_pipe(ialu_cr_reg_reg);
13222 %}
13223 
13224 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13225 %{
13226   match(Set cr (CmpP op1 (LoadP op2)));
13227   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13228 
13229   ins_cost(500); // XXX
13230   format %{ "cmpq    $op1, $op2\t# ptr" %}
13231   ins_encode %{
13232     __ cmpq($op1$$Register, $op2$$Address);
13233   %}
13234   ins_pipe(ialu_cr_reg_mem);
13235 %}
13236 
13237 // XXX this is generalized by compP_rReg_mem???
13238 // Compare raw pointer (used in out-of-heap check).
13239 // Only works because non-oop pointers must be raw pointers
13240 // and raw pointers have no anti-dependencies.
13241 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13242 %{
13243   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13244             n->in(2)->as_Load()->barrier_data() == 0);
13245   match(Set cr (CmpP op1 (LoadP op2)));
13246 
13247   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13248   ins_encode %{
13249     __ cmpq($op1$$Register, $op2$$Address);
13250   %}
13251   ins_pipe(ialu_cr_reg_mem);
13252 %}
13253 
13254 // This will generate a signed flags result. This should be OK since
13255 // any compare to a zero should be eq/neq.
13256 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13257 %{
13258   match(Set cr (CmpP src zero));
13259 
13260   format %{ "testq   $src, $src\t# ptr" %}
13261   ins_encode %{
13262     __ testq($src$$Register, $src$$Register);
13263   %}
13264   ins_pipe(ialu_cr_reg_imm);
13265 %}
13266 
13267 // This will generate a signed flags result. This should be OK since
13268 // any compare to a zero should be eq/neq.
13269 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13270 %{
13271   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13272             n->in(1)->as_Load()->barrier_data() == 0);
13273   match(Set cr (CmpP (LoadP op) zero));
13274 
13275   ins_cost(500); // XXX
13276   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13277   ins_encode %{
13278     __ testq($op$$Address, 0xFFFFFFFF);
13279   %}
13280   ins_pipe(ialu_cr_reg_imm);
13281 %}
13282 
13283 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13284 %{
13285   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13286             n->in(1)->as_Load()->barrier_data() == 0);
13287   match(Set cr (CmpP (LoadP mem) zero));
13288 
13289   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13290   ins_encode %{
13291     __ cmpq(r12, $mem$$Address);
13292   %}
13293   ins_pipe(ialu_cr_reg_mem);
13294 %}
13295 
13296 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13297 %{
13298   match(Set cr (CmpN op1 op2));
13299 
13300   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13301   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13302   ins_pipe(ialu_cr_reg_reg);
13303 %}
13304 
13305 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13306 %{
13307   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13308   match(Set cr (CmpN src (LoadN mem)));
13309 
13310   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13311   ins_encode %{
13312     __ cmpl($src$$Register, $mem$$Address);
13313   %}
13314   ins_pipe(ialu_cr_reg_mem);
13315 %}
13316 
13317 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13318   match(Set cr (CmpN op1 op2));
13319 
13320   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13321   ins_encode %{
13322     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13323   %}
13324   ins_pipe(ialu_cr_reg_imm);
13325 %}
13326 
13327 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13328 %{
13329   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13330   match(Set cr (CmpN src (LoadN mem)));
13331 
13332   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13333   ins_encode %{
13334     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13335   %}
13336   ins_pipe(ialu_cr_reg_mem);
13337 %}
13338 
13339 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13340   match(Set cr (CmpN op1 op2));
13341 
13342   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13343   ins_encode %{
13344     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13345   %}
13346   ins_pipe(ialu_cr_reg_imm);
13347 %}
13348 
13349 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13350 %{
13351   predicate(!UseCompactObjectHeaders);
13352   match(Set cr (CmpN src (LoadNKlass mem)));
13353 
13354   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13355   ins_encode %{
13356     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13357   %}
13358   ins_pipe(ialu_cr_reg_mem);
13359 %}
13360 
13361 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13362   match(Set cr (CmpN src zero));
13363 
13364   format %{ "testl   $src, $src\t# compressed ptr" %}
13365   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13366   ins_pipe(ialu_cr_reg_imm);
13367 %}
13368 
13369 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13370 %{
13371   predicate(CompressedOops::base() != nullptr &&
13372             n->in(1)->as_Load()->barrier_data() == 0);
13373   match(Set cr (CmpN (LoadN mem) zero));
13374 
13375   ins_cost(500); // XXX
13376   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13377   ins_encode %{
13378     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13379   %}
13380   ins_pipe(ialu_cr_reg_mem);
13381 %}
13382 
13383 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13384 %{
13385   predicate(CompressedOops::base() == nullptr &&
13386             n->in(1)->as_Load()->barrier_data() == 0);
13387   match(Set cr (CmpN (LoadN mem) zero));
13388 
13389   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13390   ins_encode %{
13391     __ cmpl(r12, $mem$$Address);
13392   %}
13393   ins_pipe(ialu_cr_reg_mem);
13394 %}
13395 
13396 // Yanked all unsigned pointer compare operations.
13397 // Pointer compares are done with CmpP which is already unsigned.
13398 
13399 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13400 %{
13401   match(Set cr (CmpL op1 op2));
13402 
13403   format %{ "cmpq    $op1, $op2" %}
13404   ins_encode %{
13405     __ cmpq($op1$$Register, $op2$$Register);
13406   %}
13407   ins_pipe(ialu_cr_reg_reg);
13408 %}
13409 
13410 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13411 %{
13412   match(Set cr (CmpL op1 op2));
13413 
13414   format %{ "cmpq    $op1, $op2" %}
13415   ins_encode %{
13416     __ cmpq($op1$$Register, $op2$$constant);
13417   %}
13418   ins_pipe(ialu_cr_reg_imm);
13419 %}
13420 
13421 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13422 %{
13423   match(Set cr (CmpL op1 (LoadL op2)));
13424 
13425   format %{ "cmpq    $op1, $op2" %}
13426   ins_encode %{
13427     __ cmpq($op1$$Register, $op2$$Address);
13428   %}
13429   ins_pipe(ialu_cr_reg_mem);
13430 %}
13431 
13432 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13433 %{
13434   match(Set cr (CmpL src zero));
13435 
13436   format %{ "testq   $src, $src" %}
13437   ins_encode %{
13438     __ testq($src$$Register, $src$$Register);
13439   %}
13440   ins_pipe(ialu_cr_reg_imm);
13441 %}
13442 
13443 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13444 %{
13445   match(Set cr (CmpL (AndL src con) zero));
13446 
13447   format %{ "testq   $src, $con\t# long" %}
13448   ins_encode %{
13449     __ testq($src$$Register, $con$$constant);
13450   %}
13451   ins_pipe(ialu_cr_reg_imm);
13452 %}
13453 
13454 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13455 %{
13456   match(Set cr (CmpL (AndL src1 src2) zero));
13457 
13458   format %{ "testq   $src1, $src2\t# long" %}
13459   ins_encode %{
13460     __ testq($src1$$Register, $src2$$Register);
13461   %}
13462   ins_pipe(ialu_cr_reg_imm);
13463 %}
13464 
13465 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13466 %{
13467   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13468 
13469   format %{ "testq   $src, $mem" %}
13470   ins_encode %{
13471     __ testq($src$$Register, $mem$$Address);
13472   %}
13473   ins_pipe(ialu_cr_reg_mem);
13474 %}
13475 
13476 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13477 %{
13478   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13479 
13480   format %{ "testq   $src, $mem" %}
13481   ins_encode %{
13482     __ testq($src$$Register, $mem$$Address);
13483   %}
13484   ins_pipe(ialu_cr_reg_mem);
13485 %}
13486 
13487 // Manifest a CmpU result in an integer register.  Very painful.
13488 // This is the test to avoid.
13489 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13490 %{
13491   match(Set dst (CmpU3 src1 src2));
13492   effect(KILL flags);
13493 
13494   ins_cost(275); // XXX
13495   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13496             "movl    $dst, -1\n\t"
13497             "jb,u    done\n\t"
13498             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13499     "done:" %}
13500   ins_encode %{
13501     Label done;
13502     __ cmpl($src1$$Register, $src2$$Register);
13503     __ movl($dst$$Register, -1);
13504     __ jccb(Assembler::below, done);
13505     __ setcc(Assembler::notZero, $dst$$Register);
13506     __ bind(done);
13507   %}
13508   ins_pipe(pipe_slow);
13509 %}
13510 
13511 // Manifest a CmpL result in an integer register.  Very painful.
13512 // This is the test to avoid.
13513 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13514 %{
13515   match(Set dst (CmpL3 src1 src2));
13516   effect(KILL flags);
13517 
13518   ins_cost(275); // XXX
13519   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13520             "movl    $dst, -1\n\t"
13521             "jl,s    done\n\t"
13522             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13523     "done:" %}
13524   ins_encode %{
13525     Label done;
13526     __ cmpq($src1$$Register, $src2$$Register);
13527     __ movl($dst$$Register, -1);
13528     __ jccb(Assembler::less, done);
13529     __ setcc(Assembler::notZero, $dst$$Register);
13530     __ bind(done);
13531   %}
13532   ins_pipe(pipe_slow);
13533 %}
13534 
13535 // Manifest a CmpUL result in an integer register.  Very painful.
13536 // This is the test to avoid.
13537 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13538 %{
13539   match(Set dst (CmpUL3 src1 src2));
13540   effect(KILL flags);
13541 
13542   ins_cost(275); // XXX
13543   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13544             "movl    $dst, -1\n\t"
13545             "jb,u    done\n\t"
13546             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13547     "done:" %}
13548   ins_encode %{
13549     Label done;
13550     __ cmpq($src1$$Register, $src2$$Register);
13551     __ movl($dst$$Register, -1);
13552     __ jccb(Assembler::below, done);
13553     __ setcc(Assembler::notZero, $dst$$Register);
13554     __ bind(done);
13555   %}
13556   ins_pipe(pipe_slow);
13557 %}
13558 
13559 // Unsigned long compare Instructions; really, same as signed long except they
13560 // produce an rFlagsRegU instead of rFlagsReg.
13561 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13562 %{
13563   match(Set cr (CmpUL op1 op2));
13564 
13565   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13566   ins_encode %{
13567     __ cmpq($op1$$Register, $op2$$Register);
13568   %}
13569   ins_pipe(ialu_cr_reg_reg);
13570 %}
13571 
13572 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13573 %{
13574   match(Set cr (CmpUL op1 op2));
13575 
13576   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13577   ins_encode %{
13578     __ cmpq($op1$$Register, $op2$$constant);
13579   %}
13580   ins_pipe(ialu_cr_reg_imm);
13581 %}
13582 
13583 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13584 %{
13585   match(Set cr (CmpUL op1 (LoadL op2)));
13586 
13587   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13588   ins_encode %{
13589     __ cmpq($op1$$Register, $op2$$Address);
13590   %}
13591   ins_pipe(ialu_cr_reg_mem);
13592 %}
13593 
13594 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13595 %{
13596   match(Set cr (CmpUL src zero));
13597 
13598   format %{ "testq   $src, $src\t# unsigned" %}
13599   ins_encode %{
13600     __ testq($src$$Register, $src$$Register);
13601   %}
13602   ins_pipe(ialu_cr_reg_imm);
13603 %}
13604 
13605 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13606 %{
13607   match(Set cr (CmpI (LoadB mem) imm));
13608 
13609   ins_cost(125);
13610   format %{ "cmpb    $mem, $imm" %}
13611   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13612   ins_pipe(ialu_cr_reg_mem);
13613 %}
13614 
13615 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13616 %{
13617   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13618 
13619   ins_cost(125);
13620   format %{ "testb   $mem, $imm\t# ubyte" %}
13621   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13622   ins_pipe(ialu_cr_reg_mem);
13623 %}
13624 
13625 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13626 %{
13627   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13628 
13629   ins_cost(125);
13630   format %{ "testb   $mem, $imm\t# byte" %}
13631   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13632   ins_pipe(ialu_cr_reg_mem);
13633 %}
13634 
13635 //----------Max and Min--------------------------------------------------------
13636 // Min Instructions
13637 
13638 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13639 %{
13640   predicate(!UseAPX);
13641   effect(USE_DEF dst, USE src, USE cr);
13642 
13643   format %{ "cmovlgt $dst, $src\t# min" %}
13644   ins_encode %{
13645     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13646   %}
13647   ins_pipe(pipe_cmov_reg);
13648 %}
13649 
13650 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13651 %{
13652   predicate(UseAPX);
13653   effect(DEF dst, USE src1, USE src2, USE cr);
13654 
13655   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13656   ins_encode %{
13657     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13658   %}
13659   ins_pipe(pipe_cmov_reg);
13660 %}
13661 
13662 instruct minI_rReg(rRegI dst, rRegI src)
13663 %{
13664   predicate(!UseAPX);
13665   match(Set dst (MinI dst src));
13666 
13667   ins_cost(200);
13668   expand %{
13669     rFlagsReg cr;
13670     compI_rReg(cr, dst, src);
13671     cmovI_reg_g(dst, src, cr);
13672   %}
13673 %}
13674 
13675 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13676 %{
13677   predicate(UseAPX);
13678   match(Set dst (MinI src1 src2));
13679   effect(DEF dst, USE src1, USE src2);
13680 
13681   ins_cost(200);
13682   expand %{
13683     rFlagsReg cr;
13684     compI_rReg(cr, src1, src2);
13685     cmovI_reg_g_ndd(dst, src1, src2, cr);
13686   %}
13687 %}
13688 
13689 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13690 %{
13691   predicate(!UseAPX);
13692   effect(USE_DEF dst, USE src, USE cr);
13693 
13694   format %{ "cmovllt $dst, $src\t# max" %}
13695   ins_encode %{
13696     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13697   %}
13698   ins_pipe(pipe_cmov_reg);
13699 %}
13700 
13701 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13702 %{
13703   predicate(UseAPX);
13704   effect(DEF dst, USE src1, USE src2, USE cr);
13705 
13706   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13707   ins_encode %{
13708     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13709   %}
13710   ins_pipe(pipe_cmov_reg);
13711 %}
13712 
13713 instruct maxI_rReg(rRegI dst, rRegI src)
13714 %{
13715   predicate(!UseAPX);
13716   match(Set dst (MaxI dst src));
13717 
13718   ins_cost(200);
13719   expand %{
13720     rFlagsReg cr;
13721     compI_rReg(cr, dst, src);
13722     cmovI_reg_l(dst, src, cr);
13723   %}
13724 %}
13725 
13726 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13727 %{
13728   predicate(UseAPX);
13729   match(Set dst (MaxI src1 src2));
13730   effect(DEF dst, USE src1, USE src2);
13731 
13732   ins_cost(200);
13733   expand %{
13734     rFlagsReg cr;
13735     compI_rReg(cr, src1, src2);
13736     cmovI_reg_l_ndd(dst, src1, src2, cr);
13737   %}
13738 %}
13739 
13740 // ============================================================================
13741 // Branch Instructions
13742 
13743 // Jump Direct - Label defines a relative address from JMP+1
13744 instruct jmpDir(label labl)
13745 %{
13746   match(Goto);
13747   effect(USE labl);
13748 
13749   ins_cost(300);
13750   format %{ "jmp     $labl" %}
13751   size(5);
13752   ins_encode %{
13753     Label* L = $labl$$label;
13754     __ jmp(*L, false); // Always long jump
13755   %}
13756   ins_pipe(pipe_jmp);
13757 %}
13758 
13759 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13760 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13761 %{
13762   match(If cop cr);
13763   effect(USE labl);
13764 
13765   ins_cost(300);
13766   format %{ "j$cop     $labl" %}
13767   size(6);
13768   ins_encode %{
13769     Label* L = $labl$$label;
13770     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13771   %}
13772   ins_pipe(pipe_jcc);
13773 %}
13774 
13775 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13776 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13777 %{
13778   match(CountedLoopEnd cop cr);
13779   effect(USE labl);
13780 
13781   ins_cost(300);
13782   format %{ "j$cop     $labl\t# loop end" %}
13783   size(6);
13784   ins_encode %{
13785     Label* L = $labl$$label;
13786     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13787   %}
13788   ins_pipe(pipe_jcc);
13789 %}
13790 
13791 // Jump Direct Conditional - using unsigned comparison
13792 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13793   match(If cop cmp);
13794   effect(USE labl);
13795 
13796   ins_cost(300);
13797   format %{ "j$cop,u   $labl" %}
13798   size(6);
13799   ins_encode %{
13800     Label* L = $labl$$label;
13801     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13802   %}
13803   ins_pipe(pipe_jcc);
13804 %}
13805 
13806 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13807   match(If cop cmp);
13808   effect(USE labl);
13809 
13810   ins_cost(200);
13811   format %{ "j$cop,u   $labl" %}
13812   size(6);
13813   ins_encode %{
13814     Label* L = $labl$$label;
13815     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13816   %}
13817   ins_pipe(pipe_jcc);
13818 %}
13819 
13820 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13821   match(If cop cmp);
13822   effect(USE labl);
13823 
13824   ins_cost(200);
13825   format %{ $$template
13826     if ($cop$$cmpcode == Assembler::notEqual) {
13827       $$emit$$"jp,u    $labl\n\t"
13828       $$emit$$"j$cop,u   $labl"
13829     } else {
13830       $$emit$$"jp,u    done\n\t"
13831       $$emit$$"j$cop,u   $labl\n\t"
13832       $$emit$$"done:"
13833     }
13834   %}
13835   ins_encode %{
13836     Label* l = $labl$$label;
13837     if ($cop$$cmpcode == Assembler::notEqual) {
13838       __ jcc(Assembler::parity, *l, false);
13839       __ jcc(Assembler::notEqual, *l, false);
13840     } else if ($cop$$cmpcode == Assembler::equal) {
13841       Label done;
13842       __ jccb(Assembler::parity, done);
13843       __ jcc(Assembler::equal, *l, false);
13844       __ bind(done);
13845     } else {
13846        ShouldNotReachHere();
13847     }
13848   %}
13849   ins_pipe(pipe_jcc);
13850 %}
13851 
13852 // ============================================================================
13853 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13854 // superklass array for an instance of the superklass.  Set a hidden
13855 // internal cache on a hit (cache is checked with exposed code in
13856 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13857 // encoding ALSO sets flags.
13858 
13859 instruct partialSubtypeCheck(rdi_RegP result,
13860                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13861                              rFlagsReg cr)
13862 %{
13863   match(Set result (PartialSubtypeCheck sub super));
13864   predicate(!UseSecondarySupersTable);
13865   effect(KILL rcx, KILL cr);
13866 
13867   ins_cost(1100);  // slightly larger than the next version
13868   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13869             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13870             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13871             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13872             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13873             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13874             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13875     "miss:\t" %}
13876 
13877   ins_encode %{
13878     Label miss;
13879     // NB: Callers may assume that, when $result is a valid register,
13880     // check_klass_subtype_slow_path_linear sets it to a nonzero
13881     // value.
13882     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13883                                             $rcx$$Register, $result$$Register,
13884                                             nullptr, &miss,
13885                                             /*set_cond_codes:*/ true);
13886     __ xorptr($result$$Register, $result$$Register);
13887     __ bind(miss);
13888   %}
13889 
13890   ins_pipe(pipe_slow);
13891 %}
13892 
13893 // ============================================================================
13894 // Two versions of hashtable-based partialSubtypeCheck, both used when
13895 // we need to search for a super class in the secondary supers array.
13896 // The first is used when we don't know _a priori_ the class being
13897 // searched for. The second, far more common, is used when we do know:
13898 // this is used for instanceof, checkcast, and any case where C2 can
13899 // determine it by constant propagation.
13900 
13901 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
13902                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13903                                        rFlagsReg cr)
13904 %{
13905   match(Set result (PartialSubtypeCheck sub super));
13906   predicate(UseSecondarySupersTable);
13907   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13908 
13909   ins_cost(1000);
13910   format %{ "partialSubtypeCheck $result, $sub, $super" %}
13911 
13912   ins_encode %{
13913     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
13914 					 $temp3$$Register, $temp4$$Register, $result$$Register);
13915   %}
13916 
13917   ins_pipe(pipe_slow);
13918 %}
13919 
13920 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
13921                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13922                                        rFlagsReg cr)
13923 %{
13924   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
13925   predicate(UseSecondarySupersTable);
13926   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13927 
13928   ins_cost(700);  // smaller than the next version
13929   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
13930 
13931   ins_encode %{
13932     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
13933     if (InlineSecondarySupersTest) {
13934       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
13935                                        $temp3$$Register, $temp4$$Register, $result$$Register,
13936                                        super_klass_slot);
13937     } else {
13938       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
13939     }
13940   %}
13941 
13942   ins_pipe(pipe_slow);
13943 %}
13944 
13945 // ============================================================================
13946 // Branch Instructions -- short offset versions
13947 //
13948 // These instructions are used to replace jumps of a long offset (the default
13949 // match) with jumps of a shorter offset.  These instructions are all tagged
13950 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13951 // match rules in general matching.  Instead, the ADLC generates a conversion
13952 // method in the MachNode which can be used to do in-place replacement of the
13953 // long variant with the shorter variant.  The compiler will determine if a
13954 // branch can be taken by the is_short_branch_offset() predicate in the machine
13955 // specific code section of the file.
13956 
13957 // Jump Direct - Label defines a relative address from JMP+1
13958 instruct jmpDir_short(label labl) %{
13959   match(Goto);
13960   effect(USE labl);
13961 
13962   ins_cost(300);
13963   format %{ "jmp,s   $labl" %}
13964   size(2);
13965   ins_encode %{
13966     Label* L = $labl$$label;
13967     __ jmpb(*L);
13968   %}
13969   ins_pipe(pipe_jmp);
13970   ins_short_branch(1);
13971 %}
13972 
13973 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13974 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13975   match(If cop cr);
13976   effect(USE labl);
13977 
13978   ins_cost(300);
13979   format %{ "j$cop,s   $labl" %}
13980   size(2);
13981   ins_encode %{
13982     Label* L = $labl$$label;
13983     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13984   %}
13985   ins_pipe(pipe_jcc);
13986   ins_short_branch(1);
13987 %}
13988 
13989 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13990 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13991   match(CountedLoopEnd cop cr);
13992   effect(USE labl);
13993 
13994   ins_cost(300);
13995   format %{ "j$cop,s   $labl\t# loop end" %}
13996   size(2);
13997   ins_encode %{
13998     Label* L = $labl$$label;
13999     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14000   %}
14001   ins_pipe(pipe_jcc);
14002   ins_short_branch(1);
14003 %}
14004 
14005 // Jump Direct Conditional - using unsigned comparison
14006 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14007   match(If cop cmp);
14008   effect(USE labl);
14009 
14010   ins_cost(300);
14011   format %{ "j$cop,us  $labl" %}
14012   size(2);
14013   ins_encode %{
14014     Label* L = $labl$$label;
14015     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14016   %}
14017   ins_pipe(pipe_jcc);
14018   ins_short_branch(1);
14019 %}
14020 
14021 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14022   match(If cop cmp);
14023   effect(USE labl);
14024 
14025   ins_cost(300);
14026   format %{ "j$cop,us  $labl" %}
14027   size(2);
14028   ins_encode %{
14029     Label* L = $labl$$label;
14030     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14031   %}
14032   ins_pipe(pipe_jcc);
14033   ins_short_branch(1);
14034 %}
14035 
14036 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14037   match(If cop cmp);
14038   effect(USE labl);
14039 
14040   ins_cost(300);
14041   format %{ $$template
14042     if ($cop$$cmpcode == Assembler::notEqual) {
14043       $$emit$$"jp,u,s  $labl\n\t"
14044       $$emit$$"j$cop,u,s  $labl"
14045     } else {
14046       $$emit$$"jp,u,s  done\n\t"
14047       $$emit$$"j$cop,u,s  $labl\n\t"
14048       $$emit$$"done:"
14049     }
14050   %}
14051   size(4);
14052   ins_encode %{
14053     Label* l = $labl$$label;
14054     if ($cop$$cmpcode == Assembler::notEqual) {
14055       __ jccb(Assembler::parity, *l);
14056       __ jccb(Assembler::notEqual, *l);
14057     } else if ($cop$$cmpcode == Assembler::equal) {
14058       Label done;
14059       __ jccb(Assembler::parity, done);
14060       __ jccb(Assembler::equal, *l);
14061       __ bind(done);
14062     } else {
14063        ShouldNotReachHere();
14064     }
14065   %}
14066   ins_pipe(pipe_jcc);
14067   ins_short_branch(1);
14068 %}
14069 
14070 // ============================================================================
14071 // inlined locking and unlocking
14072 
14073 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14074   match(Set cr (FastLock object box));
14075   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14076   ins_cost(300);
14077   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14078   ins_encode %{
14079     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14080   %}
14081   ins_pipe(pipe_slow);
14082 %}
14083 
14084 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14085   match(Set cr (FastUnlock object rax_reg));
14086   effect(TEMP tmp, USE_KILL rax_reg);
14087   ins_cost(300);
14088   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14089   ins_encode %{
14090     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14091   %}
14092   ins_pipe(pipe_slow);
14093 %}
14094 
14095 
14096 // ============================================================================
14097 // Safepoint Instructions
14098 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14099 %{
14100   match(SafePoint poll);
14101   effect(KILL cr, USE poll);
14102 
14103   format %{ "testl   rax, [$poll]\t"
14104             "# Safepoint: poll for GC" %}
14105   ins_cost(125);
14106   ins_encode %{
14107     __ relocate(relocInfo::poll_type);
14108     address pre_pc = __ pc();
14109     __ testl(rax, Address($poll$$Register, 0));
14110     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14111   %}
14112   ins_pipe(ialu_reg_mem);
14113 %}
14114 
14115 instruct mask_all_evexL(kReg dst, rRegL src) %{
14116   match(Set dst (MaskAll src));
14117   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14118   ins_encode %{
14119     int mask_len = Matcher::vector_length(this);
14120     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14121   %}
14122   ins_pipe( pipe_slow );
14123 %}
14124 
14125 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14126   predicate(Matcher::vector_length(n) > 32);
14127   match(Set dst (MaskAll src));
14128   effect(TEMP tmp);
14129   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14130   ins_encode %{
14131     int mask_len = Matcher::vector_length(this);
14132     __ movslq($tmp$$Register, $src$$Register);
14133     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14134   %}
14135   ins_pipe( pipe_slow );
14136 %}
14137 
14138 // ============================================================================
14139 // Procedure Call/Return Instructions
14140 // Call Java Static Instruction
14141 // Note: If this code changes, the corresponding ret_addr_offset() and
14142 //       compute_padding() functions will have to be adjusted.
14143 instruct CallStaticJavaDirect(method meth) %{
14144   match(CallStaticJava);
14145   effect(USE meth);
14146 
14147   ins_cost(300);
14148   format %{ "call,static " %}
14149   opcode(0xE8); /* E8 cd */
14150   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14151   ins_pipe(pipe_slow);
14152   ins_alignment(4);
14153 %}
14154 
14155 // Call Java Dynamic Instruction
14156 // Note: If this code changes, the corresponding ret_addr_offset() and
14157 //       compute_padding() functions will have to be adjusted.
14158 instruct CallDynamicJavaDirect(method meth)
14159 %{
14160   match(CallDynamicJava);
14161   effect(USE meth);
14162 
14163   ins_cost(300);
14164   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14165             "call,dynamic " %}
14166   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14167   ins_pipe(pipe_slow);
14168   ins_alignment(4);
14169 %}
14170 
14171 // Call Runtime Instruction
14172 instruct CallRuntimeDirect(method meth)
14173 %{
14174   match(CallRuntime);
14175   effect(USE meth);
14176 
14177   ins_cost(300);
14178   format %{ "call,runtime " %}
14179   ins_encode(clear_avx, Java_To_Runtime(meth));
14180   ins_pipe(pipe_slow);
14181 %}
14182 
14183 // Call runtime without safepoint
14184 instruct CallLeafDirect(method meth)
14185 %{
14186   match(CallLeaf);
14187   effect(USE meth);
14188 
14189   ins_cost(300);
14190   format %{ "call_leaf,runtime " %}
14191   ins_encode(clear_avx, Java_To_Runtime(meth));
14192   ins_pipe(pipe_slow);
14193 %}
14194 
14195 // Call runtime without safepoint and with vector arguments
14196 instruct CallLeafDirectVector(method meth)
14197 %{
14198   match(CallLeafVector);
14199   effect(USE meth);
14200 
14201   ins_cost(300);
14202   format %{ "call_leaf,vector " %}
14203   ins_encode(Java_To_Runtime(meth));
14204   ins_pipe(pipe_slow);
14205 %}
14206 
14207 // Call runtime without safepoint
14208 instruct CallLeafNoFPDirect(method meth)
14209 %{
14210   match(CallLeafNoFP);
14211   effect(USE meth);
14212 
14213   ins_cost(300);
14214   format %{ "call_leaf_nofp,runtime " %}
14215   ins_encode(clear_avx, Java_To_Runtime(meth));
14216   ins_pipe(pipe_slow);
14217 %}
14218 
14219 // Return Instruction
14220 // Remove the return address & jump to it.
14221 // Notice: We always emit a nop after a ret to make sure there is room
14222 // for safepoint patching
14223 instruct Ret()
14224 %{
14225   match(Return);
14226 
14227   format %{ "ret" %}
14228   ins_encode %{
14229     __ ret(0);
14230   %}
14231   ins_pipe(pipe_jmp);
14232 %}
14233 
14234 // Tail Call; Jump from runtime stub to Java code.
14235 // Also known as an 'interprocedural jump'.
14236 // Target of jump will eventually return to caller.
14237 // TailJump below removes the return address.
14238 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14239 // emitted just above the TailCall which has reset rbp to the caller state.
14240 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14241 %{
14242   match(TailCall jump_target method_ptr);
14243 
14244   ins_cost(300);
14245   format %{ "jmp     $jump_target\t# rbx holds method" %}
14246   ins_encode %{
14247     __ jmp($jump_target$$Register);
14248   %}
14249   ins_pipe(pipe_jmp);
14250 %}
14251 
14252 // Tail Jump; remove the return address; jump to target.
14253 // TailCall above leaves the return address around.
14254 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14255 %{
14256   match(TailJump jump_target ex_oop);
14257 
14258   ins_cost(300);
14259   format %{ "popq    rdx\t# pop return address\n\t"
14260             "jmp     $jump_target" %}
14261   ins_encode %{
14262     __ popq(as_Register(RDX_enc));
14263     __ jmp($jump_target$$Register);
14264   %}
14265   ins_pipe(pipe_jmp);
14266 %}
14267 
14268 // Forward exception.
14269 instruct ForwardExceptionjmp()
14270 %{
14271   match(ForwardException);
14272 
14273   format %{ "jmp     forward_exception_stub" %}
14274   ins_encode %{
14275     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14276   %}
14277   ins_pipe(pipe_jmp);
14278 %}
14279 
14280 // Create exception oop: created by stack-crawling runtime code.
14281 // Created exception is now available to this handler, and is setup
14282 // just prior to jumping to this handler.  No code emitted.
14283 instruct CreateException(rax_RegP ex_oop)
14284 %{
14285   match(Set ex_oop (CreateEx));
14286 
14287   size(0);
14288   // use the following format syntax
14289   format %{ "# exception oop is in rax; no code emitted" %}
14290   ins_encode();
14291   ins_pipe(empty);
14292 %}
14293 
14294 // Rethrow exception:
14295 // The exception oop will come in the first argument position.
14296 // Then JUMP (not call) to the rethrow stub code.
14297 instruct RethrowException()
14298 %{
14299   match(Rethrow);
14300 
14301   // use the following format syntax
14302   format %{ "jmp     rethrow_stub" %}
14303   ins_encode %{
14304     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14305   %}
14306   ins_pipe(pipe_jmp);
14307 %}
14308 
14309 // ============================================================================
14310 // This name is KNOWN by the ADLC and cannot be changed.
14311 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14312 // for this guy.
14313 instruct tlsLoadP(r15_RegP dst) %{
14314   match(Set dst (ThreadLocal));
14315   effect(DEF dst);
14316 
14317   size(0);
14318   format %{ "# TLS is in R15" %}
14319   ins_encode( /*empty encoding*/ );
14320   ins_pipe(ialu_reg_reg);
14321 %}
14322 
14323 
14324 //----------PEEPHOLE RULES-----------------------------------------------------
14325 // These must follow all instruction definitions as they use the names
14326 // defined in the instructions definitions.
14327 //
14328 // peeppredicate ( rule_predicate );
14329 // // the predicate unless which the peephole rule will be ignored
14330 //
14331 // peepmatch ( root_instr_name [preceding_instruction]* );
14332 //
14333 // peepprocedure ( procedure_name );
14334 // // provide a procedure name to perform the optimization, the procedure should
14335 // // reside in the architecture dependent peephole file, the method has the
14336 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14337 // // with the arguments being the basic block, the current node index inside the
14338 // // block, the register allocator, the functions upon invoked return a new node
14339 // // defined in peepreplace, and the rules of the nodes appearing in the
14340 // // corresponding peepmatch, the function return true if successful, else
14341 // // return false
14342 //
14343 // peepconstraint %{
14344 // (instruction_number.operand_name relational_op instruction_number.operand_name
14345 //  [, ...] );
14346 // // instruction numbers are zero-based using left to right order in peepmatch
14347 //
14348 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14349 // // provide an instruction_number.operand_name for each operand that appears
14350 // // in the replacement instruction's match rule
14351 //
14352 // ---------VM FLAGS---------------------------------------------------------
14353 //
14354 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14355 //
14356 // Each peephole rule is given an identifying number starting with zero and
14357 // increasing by one in the order seen by the parser.  An individual peephole
14358 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14359 // on the command-line.
14360 //
14361 // ---------CURRENT LIMITATIONS----------------------------------------------
14362 //
14363 // Only transformations inside a basic block (do we need more for peephole)
14364 //
14365 // ---------EXAMPLE----------------------------------------------------------
14366 //
14367 // // pertinent parts of existing instructions in architecture description
14368 // instruct movI(rRegI dst, rRegI src)
14369 // %{
14370 //   match(Set dst (CopyI src));
14371 // %}
14372 //
14373 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14374 // %{
14375 //   match(Set dst (AddI dst src));
14376 //   effect(KILL cr);
14377 // %}
14378 //
14379 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14380 // %{
14381 //   match(Set dst (AddI dst src));
14382 // %}
14383 //
14384 // 1. Simple replacement
14385 // - Only match adjacent instructions in same basic block
14386 // - Only equality constraints
14387 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14388 // - Only one replacement instruction
14389 //
14390 // // Change (inc mov) to lea
14391 // peephole %{
14392 //   // lea should only be emitted when beneficial
14393 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14394 //   // increment preceded by register-register move
14395 //   peepmatch ( incI_rReg movI );
14396 //   // require that the destination register of the increment
14397 //   // match the destination register of the move
14398 //   peepconstraint ( 0.dst == 1.dst );
14399 //   // construct a replacement instruction that sets
14400 //   // the destination to ( move's source register + one )
14401 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14402 // %}
14403 //
14404 // 2. Procedural replacement
14405 // - More flexible finding relevent nodes
14406 // - More flexible constraints
14407 // - More flexible transformations
14408 // - May utilise architecture-dependent API more effectively
14409 // - Currently only one replacement instruction due to adlc parsing capabilities
14410 //
14411 // // Change (inc mov) to lea
14412 // peephole %{
14413 //   // lea should only be emitted when beneficial
14414 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14415 //   // the rule numbers of these nodes inside are passed into the function below
14416 //   peepmatch ( incI_rReg movI );
14417 //   // the method that takes the responsibility of transformation
14418 //   peepprocedure ( inc_mov_to_lea );
14419 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14420 //   // node is passed into the function above
14421 //   peepreplace ( leaI_rReg_immI() );
14422 // %}
14423 
14424 // These instructions is not matched by the matcher but used by the peephole
14425 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14426 %{
14427   predicate(false);
14428   match(Set dst (AddI src1 src2));
14429   format %{ "leal    $dst, [$src1 + $src2]" %}
14430   ins_encode %{
14431     Register dst = $dst$$Register;
14432     Register src1 = $src1$$Register;
14433     Register src2 = $src2$$Register;
14434     if (src1 != rbp && src1 != r13) {
14435       __ leal(dst, Address(src1, src2, Address::times_1));
14436     } else {
14437       assert(src2 != rbp && src2 != r13, "");
14438       __ leal(dst, Address(src2, src1, Address::times_1));
14439     }
14440   %}
14441   ins_pipe(ialu_reg_reg);
14442 %}
14443 
14444 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14445 %{
14446   predicate(false);
14447   match(Set dst (AddI src1 src2));
14448   format %{ "leal    $dst, [$src1 + $src2]" %}
14449   ins_encode %{
14450     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14451   %}
14452   ins_pipe(ialu_reg_reg);
14453 %}
14454 
14455 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14456 %{
14457   predicate(false);
14458   match(Set dst (LShiftI src shift));
14459   format %{ "leal    $dst, [$src << $shift]" %}
14460   ins_encode %{
14461     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14462     Register src = $src$$Register;
14463     if (scale == Address::times_2 && src != rbp && src != r13) {
14464       __ leal($dst$$Register, Address(src, src, Address::times_1));
14465     } else {
14466       __ leal($dst$$Register, Address(noreg, src, scale));
14467     }
14468   %}
14469   ins_pipe(ialu_reg_reg);
14470 %}
14471 
14472 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14473 %{
14474   predicate(false);
14475   match(Set dst (AddL src1 src2));
14476   format %{ "leaq    $dst, [$src1 + $src2]" %}
14477   ins_encode %{
14478     Register dst = $dst$$Register;
14479     Register src1 = $src1$$Register;
14480     Register src2 = $src2$$Register;
14481     if (src1 != rbp && src1 != r13) {
14482       __ leaq(dst, Address(src1, src2, Address::times_1));
14483     } else {
14484       assert(src2 != rbp && src2 != r13, "");
14485       __ leaq(dst, Address(src2, src1, Address::times_1));
14486     }
14487   %}
14488   ins_pipe(ialu_reg_reg);
14489 %}
14490 
14491 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14492 %{
14493   predicate(false);
14494   match(Set dst (AddL src1 src2));
14495   format %{ "leaq    $dst, [$src1 + $src2]" %}
14496   ins_encode %{
14497     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14498   %}
14499   ins_pipe(ialu_reg_reg);
14500 %}
14501 
14502 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14503 %{
14504   predicate(false);
14505   match(Set dst (LShiftL src shift));
14506   format %{ "leaq    $dst, [$src << $shift]" %}
14507   ins_encode %{
14508     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14509     Register src = $src$$Register;
14510     if (scale == Address::times_2 && src != rbp && src != r13) {
14511       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14512     } else {
14513       __ leaq($dst$$Register, Address(noreg, src, scale));
14514     }
14515   %}
14516   ins_pipe(ialu_reg_reg);
14517 %}
14518 
14519 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14520 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14521 // processors with at least partial ALU support for lea
14522 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14523 // beneficial for processors with full ALU support
14524 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14525 
14526 peephole
14527 %{
14528   peeppredicate(VM_Version::supports_fast_2op_lea());
14529   peepmatch (addI_rReg);
14530   peepprocedure (lea_coalesce_reg);
14531   peepreplace (leaI_rReg_rReg_peep());
14532 %}
14533 
14534 peephole
14535 %{
14536   peeppredicate(VM_Version::supports_fast_2op_lea());
14537   peepmatch (addI_rReg_imm);
14538   peepprocedure (lea_coalesce_imm);
14539   peepreplace (leaI_rReg_immI_peep());
14540 %}
14541 
14542 peephole
14543 %{
14544   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14545                 VM_Version::is_intel_cascade_lake());
14546   peepmatch (incI_rReg);
14547   peepprocedure (lea_coalesce_imm);
14548   peepreplace (leaI_rReg_immI_peep());
14549 %}
14550 
14551 peephole
14552 %{
14553   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14554                 VM_Version::is_intel_cascade_lake());
14555   peepmatch (decI_rReg);
14556   peepprocedure (lea_coalesce_imm);
14557   peepreplace (leaI_rReg_immI_peep());
14558 %}
14559 
14560 peephole
14561 %{
14562   peeppredicate(VM_Version::supports_fast_2op_lea());
14563   peepmatch (salI_rReg_immI2);
14564   peepprocedure (lea_coalesce_imm);
14565   peepreplace (leaI_rReg_immI2_peep());
14566 %}
14567 
14568 peephole
14569 %{
14570   peeppredicate(VM_Version::supports_fast_2op_lea());
14571   peepmatch (addL_rReg);
14572   peepprocedure (lea_coalesce_reg);
14573   peepreplace (leaL_rReg_rReg_peep());
14574 %}
14575 
14576 peephole
14577 %{
14578   peeppredicate(VM_Version::supports_fast_2op_lea());
14579   peepmatch (addL_rReg_imm);
14580   peepprocedure (lea_coalesce_imm);
14581   peepreplace (leaL_rReg_immL32_peep());
14582 %}
14583 
14584 peephole
14585 %{
14586   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14587                 VM_Version::is_intel_cascade_lake());
14588   peepmatch (incL_rReg);
14589   peepprocedure (lea_coalesce_imm);
14590   peepreplace (leaL_rReg_immL32_peep());
14591 %}
14592 
14593 peephole
14594 %{
14595   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14596                 VM_Version::is_intel_cascade_lake());
14597   peepmatch (decL_rReg);
14598   peepprocedure (lea_coalesce_imm);
14599   peepreplace (leaL_rReg_immL32_peep());
14600 %}
14601 
14602 peephole
14603 %{
14604   peeppredicate(VM_Version::supports_fast_2op_lea());
14605   peepmatch (salL_rReg_immI2);
14606   peepprocedure (lea_coalesce_imm);
14607   peepreplace (leaL_rReg_immI2_peep());
14608 %}
14609 
14610 peephole
14611 %{
14612   peepmatch (leaPCompressedOopOffset);
14613   peepprocedure (lea_remove_redundant);
14614 %}
14615 
14616 peephole
14617 %{
14618   peepmatch (leaP8Narrow);
14619   peepprocedure (lea_remove_redundant);
14620 %}
14621 
14622 peephole
14623 %{
14624   peepmatch (leaP32Narrow);
14625   peepprocedure (lea_remove_redundant);
14626 %}
14627 
14628 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14629 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14630 
14631 //int variant
14632 peephole
14633 %{
14634   peepmatch (testI_reg);
14635   peepprocedure (test_may_remove);
14636 %}
14637 
14638 //long variant
14639 peephole
14640 %{
14641   peepmatch (testL_reg);
14642   peepprocedure (test_may_remove);
14643 %}
14644 
14645 
14646 //----------SMARTSPILL RULES---------------------------------------------------
14647 // These must follow all instruction definitions as they use the names
14648 // defined in the instructions definitions.