1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 
  442 extern RegMask _ANY_REG_mask;
  443 extern RegMask _PTR_REG_mask;
  444 extern RegMask _PTR_REG_NO_RBP_mask;
  445 extern RegMask _PTR_NO_RAX_REG_mask;
  446 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  447 extern RegMask _LONG_REG_mask;
  448 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  449 extern RegMask _LONG_NO_RCX_REG_mask;
  450 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  451 extern RegMask _INT_REG_mask;
  452 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  453 extern RegMask _INT_NO_RCX_REG_mask;
  454 extern RegMask _INT_NO_RBP_R13_REG_mask;
  455 extern RegMask _FLOAT_REG_mask;
  456 
  457 extern RegMask _STACK_OR_PTR_REG_mask;
  458 extern RegMask _STACK_OR_LONG_REG_mask;
  459 extern RegMask _STACK_OR_INT_REG_mask;
  460 
  461 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  462 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  463 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  464 
  465 %}
  466 
  467 source %{
  468 #define   RELOC_IMM64    Assembler::imm_operand
  469 #define   RELOC_DISP32   Assembler::disp32_operand
  470 
  471 #define __ masm->
  472 
  473 RegMask _ANY_REG_mask;
  474 RegMask _PTR_REG_mask;
  475 RegMask _PTR_REG_NO_RBP_mask;
  476 RegMask _PTR_NO_RAX_REG_mask;
  477 RegMask _PTR_NO_RAX_RBX_REG_mask;
  478 RegMask _LONG_REG_mask;
  479 RegMask _LONG_NO_RAX_RDX_REG_mask;
  480 RegMask _LONG_NO_RCX_REG_mask;
  481 RegMask _LONG_NO_RBP_R13_REG_mask;
  482 RegMask _INT_REG_mask;
  483 RegMask _INT_NO_RAX_RDX_REG_mask;
  484 RegMask _INT_NO_RCX_REG_mask;
  485 RegMask _INT_NO_RBP_R13_REG_mask;
  486 RegMask _FLOAT_REG_mask;
  487 RegMask _STACK_OR_PTR_REG_mask;
  488 RegMask _STACK_OR_LONG_REG_mask;
  489 RegMask _STACK_OR_INT_REG_mask;
  490 
  491 static bool need_r12_heapbase() {
  492   return UseCompressedOops;
  493 }
  494 
  495 void reg_mask_init() {
  496   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  497 
  498   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  499   // We derive a number of subsets from it.
  500   _ANY_REG_mask = _ALL_REG_mask;
  501 
  502   if (PreserveFramePointer) {
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  504     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  505   }
  506   if (need_r12_heapbase()) {
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  508     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  509   }
  510 
  511   _PTR_REG_mask = _ANY_REG_mask;
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  515   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  516   if (!UseAPX) {
  517     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  519       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  520     }
  521   }
  522 
  523   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  524   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  525 
  526   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  528   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  529 
  530   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  532   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  533 
  534   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  536   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  537 
  538 
  539   _LONG_REG_mask = _PTR_REG_mask;
  540   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  541   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  542 
  543   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  547   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  548 
  549   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  551   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  552 
  553   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  557   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  558 
  559   _INT_REG_mask = _ALL_INT_REG_mask;
  560   if (!UseAPX) {
  561     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  562       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  563     }
  564   }
  565 
  566   if (PreserveFramePointer) {
  567     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  568   }
  569   if (need_r12_heapbase()) {
  570     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  571   }
  572 
  573   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  574   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  575 
  576   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  578   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  579 
  580   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  581   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  582 
  583   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  585   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  586 
  587   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  588   // from the float_reg_legacy/float_reg_evex register class.
  589   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  590 }
  591 
  592 static bool generate_vzeroupper(Compile* C) {
  593   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  594 }
  595 
  596 static int clear_avx_size() {
  597   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  598 }
  599 
  600 // !!!!! Special hack to get all types of calls to specify the byte offset
  601 //       from the start of the call to the point where the return address
  602 //       will point.
  603 int MachCallStaticJavaNode::ret_addr_offset()
  604 {
  605   int offset = 5; // 5 bytes from start of call to where return address points
  606   offset += clear_avx_size();
  607   return offset;
  608 }
  609 
  610 int MachCallDynamicJavaNode::ret_addr_offset()
  611 {
  612   int offset = 15; // 15 bytes from start of call to where return address points
  613   offset += clear_avx_size();
  614   return offset;
  615 }
  616 
  617 int MachCallRuntimeNode::ret_addr_offset() {
  618   int offset = 13; // movq r10,#addr; callq (r10)
  619   if (this->ideal_Opcode() != Op_CallLeafVector) {
  620     offset += clear_avx_size();
  621   }
  622   return offset;
  623 }
  624 //
  625 // Compute padding required for nodes which need alignment
  626 //
  627 
  628 // The address of the call instruction needs to be 4-byte aligned to
  629 // ensure that it does not span a cache line so that it can be patched.
  630 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  631 {
  632   current_offset += clear_avx_size(); // skip vzeroupper
  633   current_offset += 1; // skip call opcode byte
  634   return align_up(current_offset, alignment_required()) - current_offset;
  635 }
  636 
  637 // The address of the call instruction needs to be 4-byte aligned to
  638 // ensure that it does not span a cache line so that it can be patched.
  639 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  640 {
  641   current_offset += clear_avx_size(); // skip vzeroupper
  642   current_offset += 11; // skip movq instruction + call opcode byte
  643   return align_up(current_offset, alignment_required()) - current_offset;
  644 }
  645 
  646 // This could be in MacroAssembler but it's fairly C2 specific
  647 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  648   Label exit;
  649   __ jccb(Assembler::noParity, exit);
  650   __ pushf();
  651   //
  652   // comiss/ucomiss instructions set ZF,PF,CF flags and
  653   // zero OF,AF,SF for NaN values.
  654   // Fixup flags by zeroing ZF,PF so that compare of NaN
  655   // values returns 'less than' result (CF is set).
  656   // Leave the rest of flags unchanged.
  657   //
  658   //    7 6 5 4 3 2 1 0
  659   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  660   //    0 0 1 0 1 0 1 1   (0x2B)
  661   //
  662   __ andq(Address(rsp, 0), 0xffffff2b);
  663   __ popf();
  664   __ bind(exit);
  665 }
  666 
  667 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  668   Label done;
  669   __ movl(dst, -1);
  670   __ jcc(Assembler::parity, done);
  671   __ jcc(Assembler::below, done);
  672   __ setcc(Assembler::notEqual, dst);
  673   __ bind(done);
  674 }
  675 
  676 // Math.min()    # Math.max()
  677 // --------------------------
  678 // ucomis[s/d]   #
  679 // ja   -> b     # a
  680 // jp   -> NaN   # NaN
  681 // jb   -> a     # b
  682 // je            #
  683 // |-jz -> a | b # a & b
  684 // |    -> a     #
  685 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  686                             XMMRegister a, XMMRegister b,
  687                             XMMRegister xmmt, Register rt,
  688                             bool min, bool single) {
  689 
  690   Label nan, zero, below, above, done;
  691 
  692   if (single)
  693     __ ucomiss(a, b);
  694   else
  695     __ ucomisd(a, b);
  696 
  697   if (dst->encoding() != (min ? b : a)->encoding())
  698     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  699   else
  700     __ jccb(Assembler::above, done);
  701 
  702   __ jccb(Assembler::parity, nan);  // PF=1
  703   __ jccb(Assembler::below, below); // CF=1
  704 
  705   // equal
  706   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  707   if (single) {
  708     __ ucomiss(a, xmmt);
  709     __ jccb(Assembler::equal, zero);
  710 
  711     __ movflt(dst, a);
  712     __ jmp(done);
  713   }
  714   else {
  715     __ ucomisd(a, xmmt);
  716     __ jccb(Assembler::equal, zero);
  717 
  718     __ movdbl(dst, a);
  719     __ jmp(done);
  720   }
  721 
  722   __ bind(zero);
  723   if (min)
  724     __ vpor(dst, a, b, Assembler::AVX_128bit);
  725   else
  726     __ vpand(dst, a, b, Assembler::AVX_128bit);
  727 
  728   __ jmp(done);
  729 
  730   __ bind(above);
  731   if (single)
  732     __ movflt(dst, min ? b : a);
  733   else
  734     __ movdbl(dst, min ? b : a);
  735 
  736   __ jmp(done);
  737 
  738   __ bind(nan);
  739   if (single) {
  740     __ movl(rt, 0x7fc00000); // Float.NaN
  741     __ movdl(dst, rt);
  742   }
  743   else {
  744     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  745     __ movdq(dst, rt);
  746   }
  747   __ jmp(done);
  748 
  749   __ bind(below);
  750   if (single)
  751     __ movflt(dst, min ? a : b);
  752   else
  753     __ movdbl(dst, min ? a : b);
  754 
  755   __ bind(done);
  756 }
  757 
  758 //=============================================================================
  759 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  760 
  761 int ConstantTable::calculate_table_base_offset() const {
  762   return 0;  // absolute addressing, no offset
  763 }
  764 
  765 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  766 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  767   ShouldNotReachHere();
  768 }
  769 
  770 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  771   // Empty encoding
  772 }
  773 
  774 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  775   return 0;
  776 }
  777 
  778 #ifndef PRODUCT
  779 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  780   st->print("# MachConstantBaseNode (empty encoding)");
  781 }
  782 #endif
  783 
  784 
  785 //=============================================================================
  786 #ifndef PRODUCT
  787 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  788   Compile* C = ra_->C;
  789 
  790   int framesize = C->output()->frame_size_in_bytes();
  791   int bangsize = C->output()->bang_size_in_bytes();
  792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  793   // Remove wordSize for return addr which is already pushed.
  794   framesize -= wordSize;
  795 
  796   if (C->output()->need_stack_bang(bangsize)) {
  797     framesize -= wordSize;
  798     st->print("# stack bang (%d bytes)", bangsize);
  799     st->print("\n\t");
  800     st->print("pushq   rbp\t# Save rbp");
  801     if (PreserveFramePointer) {
  802         st->print("\n\t");
  803         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  804     }
  805     if (framesize) {
  806       st->print("\n\t");
  807       st->print("subq    rsp, #%d\t# Create frame",framesize);
  808     }
  809   } else {
  810     st->print("subq    rsp, #%d\t# Create frame",framesize);
  811     st->print("\n\t");
  812     framesize -= wordSize;
  813     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  814     if (PreserveFramePointer) {
  815       st->print("\n\t");
  816       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  817       if (framesize > 0) {
  818         st->print("\n\t");
  819         st->print("addq    rbp, #%d", framesize);
  820       }
  821     }
  822   }
  823 
  824   if (VerifyStackAtCalls) {
  825     st->print("\n\t");
  826     framesize -= wordSize;
  827     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  828 #ifdef ASSERT
  829     st->print("\n\t");
  830     st->print("# stack alignment check");
  831 #endif
  832   }
  833   if (C->stub_function() != nullptr) {
  834     st->print("\n\t");
  835     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  836     st->print("\n\t");
  837     st->print("je      fast_entry\t");
  838     st->print("\n\t");
  839     st->print("call    #nmethod_entry_barrier_stub\t");
  840     st->print("\n\tfast_entry:");
  841   }
  842   st->cr();
  843 }
  844 #endif
  845 
  846 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  847   Compile* C = ra_->C;
  848 
  849   int framesize = C->output()->frame_size_in_bytes();
  850   int bangsize = C->output()->bang_size_in_bytes();
  851 
  852   if (C->clinit_barrier_on_entry()) {
  853     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  854     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
  855 
  856     Label L_skip_barrier;
  857     Register klass = rscratch1;
  858 
  859     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  860     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  861 
  862     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  863 
  864     __ bind(L_skip_barrier);
  865   }
  866 
  867   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  868 
  869   C->output()->set_frame_complete(__ offset());
  870 
  871   if (C->has_mach_constant_base_node()) {
  872     // NOTE: We set the table base offset here because users might be
  873     // emitted before MachConstantBaseNode.
  874     ConstantTable& constant_table = C->output()->constant_table();
  875     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  876   }
  877 }
  878 
  879 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  880 {
  881   return MachNode::size(ra_); // too many variables; just compute it
  882                               // the hard way
  883 }
  884 
  885 int MachPrologNode::reloc() const
  886 {
  887   return 0; // a large enough number
  888 }
  889 
  890 //=============================================================================
  891 #ifndef PRODUCT
  892 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  893 {
  894   Compile* C = ra_->C;
  895   if (generate_vzeroupper(C)) {
  896     st->print("vzeroupper");
  897     st->cr(); st->print("\t");
  898   }
  899 
  900   int framesize = C->output()->frame_size_in_bytes();
  901   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  902   // Remove word for return adr already pushed
  903   // and RBP
  904   framesize -= 2*wordSize;
  905 
  906   if (framesize) {
  907     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  908     st->print("\t");
  909   }
  910 
  911   st->print_cr("popq    rbp");
  912   if (do_polling() && C->is_method_compilation()) {
  913     st->print("\t");
  914     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  915                  "ja      #safepoint_stub\t"
  916                  "# Safepoint: poll for GC");
  917   }
  918 }
  919 #endif
  920 
  921 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  922 {
  923   Compile* C = ra_->C;
  924 
  925   if (generate_vzeroupper(C)) {
  926     // Clear upper bits of YMM registers when current compiled code uses
  927     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  928     __ vzeroupper();
  929   }
  930 
  931   int framesize = C->output()->frame_size_in_bytes();
  932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  933   // Remove word for return adr already pushed
  934   // and RBP
  935   framesize -= 2*wordSize;
  936 
  937   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  938 
  939   if (framesize) {
  940     __ addq(rsp, framesize);
  941   }
  942 
  943   __ popq(rbp);
  944 
  945   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  946     __ reserved_stack_check();
  947   }
  948 
  949   if (do_polling() && C->is_method_compilation()) {
  950     Label dummy_label;
  951     Label* code_stub = &dummy_label;
  952     if (!C->output()->in_scratch_emit_size()) {
  953       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  954       C->output()->add_stub(stub);
  955       code_stub = &stub->entry();
  956     }
  957     __ relocate(relocInfo::poll_return_type);
  958     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  959   }
  960 }
  961 
  962 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  963 {
  964   return MachNode::size(ra_); // too many variables; just compute it
  965                               // the hard way
  966 }
  967 
  968 int MachEpilogNode::reloc() const
  969 {
  970   return 2; // a large enough number
  971 }
  972 
  973 const Pipeline* MachEpilogNode::pipeline() const
  974 {
  975   return MachNode::pipeline_class();
  976 }
  977 
  978 //=============================================================================
  979 
  980 enum RC {
  981   rc_bad,
  982   rc_int,
  983   rc_kreg,
  984   rc_float,
  985   rc_stack
  986 };
  987 
  988 static enum RC rc_class(OptoReg::Name reg)
  989 {
  990   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  991 
  992   if (OptoReg::is_stack(reg)) return rc_stack;
  993 
  994   VMReg r = OptoReg::as_VMReg(reg);
  995 
  996   if (r->is_Register()) return rc_int;
  997 
  998   if (r->is_KRegister()) return rc_kreg;
  999 
 1000   assert(r->is_XMMRegister(), "must be");
 1001   return rc_float;
 1002 }
 1003 
 1004 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1005 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1006                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1007 
 1008 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1009                      int stack_offset, int reg, uint ireg, outputStream* st);
 1010 
 1011 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1012                                       int dst_offset, uint ireg, outputStream* st) {
 1013   if (masm) {
 1014     switch (ireg) {
 1015     case Op_VecS:
 1016       __ movq(Address(rsp, -8), rax);
 1017       __ movl(rax, Address(rsp, src_offset));
 1018       __ movl(Address(rsp, dst_offset), rax);
 1019       __ movq(rax, Address(rsp, -8));
 1020       break;
 1021     case Op_VecD:
 1022       __ pushq(Address(rsp, src_offset));
 1023       __ popq (Address(rsp, dst_offset));
 1024       break;
 1025     case Op_VecX:
 1026       __ pushq(Address(rsp, src_offset));
 1027       __ popq (Address(rsp, dst_offset));
 1028       __ pushq(Address(rsp, src_offset+8));
 1029       __ popq (Address(rsp, dst_offset+8));
 1030       break;
 1031     case Op_VecY:
 1032       __ vmovdqu(Address(rsp, -32), xmm0);
 1033       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1034       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1035       __ vmovdqu(xmm0, Address(rsp, -32));
 1036       break;
 1037     case Op_VecZ:
 1038       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1039       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1040       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1041       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1042       break;
 1043     default:
 1044       ShouldNotReachHere();
 1045     }
 1046 #ifndef PRODUCT
 1047   } else {
 1048     switch (ireg) {
 1049     case Op_VecS:
 1050       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1051                 "movl    rax, [rsp + #%d]\n\t"
 1052                 "movl    [rsp + #%d], rax\n\t"
 1053                 "movq    rax, [rsp - #8]",
 1054                 src_offset, dst_offset);
 1055       break;
 1056     case Op_VecD:
 1057       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1058                 "popq    [rsp + #%d]",
 1059                 src_offset, dst_offset);
 1060       break;
 1061      case Op_VecX:
 1062       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1063                 "popq    [rsp + #%d]\n\t"
 1064                 "pushq   [rsp + #%d]\n\t"
 1065                 "popq    [rsp + #%d]",
 1066                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1067       break;
 1068     case Op_VecY:
 1069       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1070                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1071                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1072                 "vmovdqu xmm0, [rsp - #32]",
 1073                 src_offset, dst_offset);
 1074       break;
 1075     case Op_VecZ:
 1076       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1077                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1078                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1079                 "vmovdqu xmm0, [rsp - #64]",
 1080                 src_offset, dst_offset);
 1081       break;
 1082     default:
 1083       ShouldNotReachHere();
 1084     }
 1085 #endif
 1086   }
 1087 }
 1088 
 1089 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1090                                        PhaseRegAlloc* ra_,
 1091                                        bool do_size,
 1092                                        outputStream* st) const {
 1093   assert(masm != nullptr || st  != nullptr, "sanity");
 1094   // Get registers to move
 1095   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1096   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1097   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1098   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1099 
 1100   enum RC src_second_rc = rc_class(src_second);
 1101   enum RC src_first_rc = rc_class(src_first);
 1102   enum RC dst_second_rc = rc_class(dst_second);
 1103   enum RC dst_first_rc = rc_class(dst_first);
 1104 
 1105   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1106          "must move at least 1 register" );
 1107 
 1108   if (src_first == dst_first && src_second == dst_second) {
 1109     // Self copy, no move
 1110     return 0;
 1111   }
 1112   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1113     uint ireg = ideal_reg();
 1114     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1115     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1116     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1117       // mem -> mem
 1118       int src_offset = ra_->reg2offset(src_first);
 1119       int dst_offset = ra_->reg2offset(dst_first);
 1120       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1121     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1122       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1123     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1124       int stack_offset = ra_->reg2offset(dst_first);
 1125       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1126     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1127       int stack_offset = ra_->reg2offset(src_first);
 1128       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1129     } else {
 1130       ShouldNotReachHere();
 1131     }
 1132     return 0;
 1133   }
 1134   if (src_first_rc == rc_stack) {
 1135     // mem ->
 1136     if (dst_first_rc == rc_stack) {
 1137       // mem -> mem
 1138       assert(src_second != dst_first, "overlap");
 1139       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1140           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1141         // 64-bit
 1142         int src_offset = ra_->reg2offset(src_first);
 1143         int dst_offset = ra_->reg2offset(dst_first);
 1144         if (masm) {
 1145           __ pushq(Address(rsp, src_offset));
 1146           __ popq (Address(rsp, dst_offset));
 1147 #ifndef PRODUCT
 1148         } else {
 1149           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1150                     "popq    [rsp + #%d]",
 1151                      src_offset, dst_offset);
 1152 #endif
 1153         }
 1154       } else {
 1155         // 32-bit
 1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1158         // No pushl/popl, so:
 1159         int src_offset = ra_->reg2offset(src_first);
 1160         int dst_offset = ra_->reg2offset(dst_first);
 1161         if (masm) {
 1162           __ movq(Address(rsp, -8), rax);
 1163           __ movl(rax, Address(rsp, src_offset));
 1164           __ movl(Address(rsp, dst_offset), rax);
 1165           __ movq(rax, Address(rsp, -8));
 1166 #ifndef PRODUCT
 1167         } else {
 1168           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1169                     "movl    rax, [rsp + #%d]\n\t"
 1170                     "movl    [rsp + #%d], rax\n\t"
 1171                     "movq    rax, [rsp - #8]",
 1172                      src_offset, dst_offset);
 1173 #endif
 1174         }
 1175       }
 1176       return 0;
 1177     } else if (dst_first_rc == rc_int) {
 1178       // mem -> gpr
 1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1181         // 64-bit
 1182         int offset = ra_->reg2offset(src_first);
 1183         if (masm) {
 1184           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1185 #ifndef PRODUCT
 1186         } else {
 1187           st->print("movq    %s, [rsp + #%d]\t# spill",
 1188                      Matcher::regName[dst_first],
 1189                      offset);
 1190 #endif
 1191         }
 1192       } else {
 1193         // 32-bit
 1194         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1195         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1196         int offset = ra_->reg2offset(src_first);
 1197         if (masm) {
 1198           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1199 #ifndef PRODUCT
 1200         } else {
 1201           st->print("movl    %s, [rsp + #%d]\t# spill",
 1202                      Matcher::regName[dst_first],
 1203                      offset);
 1204 #endif
 1205         }
 1206       }
 1207       return 0;
 1208     } else if (dst_first_rc == rc_float) {
 1209       // mem-> xmm
 1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1212         // 64-bit
 1213         int offset = ra_->reg2offset(src_first);
 1214         if (masm) {
 1215           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1216 #ifndef PRODUCT
 1217         } else {
 1218           st->print("%s  %s, [rsp + #%d]\t# spill",
 1219                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1220                      Matcher::regName[dst_first],
 1221                      offset);
 1222 #endif
 1223         }
 1224       } else {
 1225         // 32-bit
 1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1228         int offset = ra_->reg2offset(src_first);
 1229         if (masm) {
 1230           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1231 #ifndef PRODUCT
 1232         } else {
 1233           st->print("movss   %s, [rsp + #%d]\t# spill",
 1234                      Matcher::regName[dst_first],
 1235                      offset);
 1236 #endif
 1237         }
 1238       }
 1239       return 0;
 1240     } else if (dst_first_rc == rc_kreg) {
 1241       // mem -> kreg
 1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1244         // 64-bit
 1245         int offset = ra_->reg2offset(src_first);
 1246         if (masm) {
 1247           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1248 #ifndef PRODUCT
 1249         } else {
 1250           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1251                      Matcher::regName[dst_first],
 1252                      offset);
 1253 #endif
 1254         }
 1255       }
 1256       return 0;
 1257     }
 1258   } else if (src_first_rc == rc_int) {
 1259     // gpr ->
 1260     if (dst_first_rc == rc_stack) {
 1261       // gpr -> mem
 1262       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1263           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1264         // 64-bit
 1265         int offset = ra_->reg2offset(dst_first);
 1266         if (masm) {
 1267           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1268 #ifndef PRODUCT
 1269         } else {
 1270           st->print("movq    [rsp + #%d], %s\t# spill",
 1271                      offset,
 1272                      Matcher::regName[src_first]);
 1273 #endif
 1274         }
 1275       } else {
 1276         // 32-bit
 1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1279         int offset = ra_->reg2offset(dst_first);
 1280         if (masm) {
 1281           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1282 #ifndef PRODUCT
 1283         } else {
 1284           st->print("movl    [rsp + #%d], %s\t# spill",
 1285                      offset,
 1286                      Matcher::regName[src_first]);
 1287 #endif
 1288         }
 1289       }
 1290       return 0;
 1291     } else if (dst_first_rc == rc_int) {
 1292       // gpr -> gpr
 1293       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1294           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1295         // 64-bit
 1296         if (masm) {
 1297           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1298                   as_Register(Matcher::_regEncode[src_first]));
 1299 #ifndef PRODUCT
 1300         } else {
 1301           st->print("movq    %s, %s\t# spill",
 1302                      Matcher::regName[dst_first],
 1303                      Matcher::regName[src_first]);
 1304 #endif
 1305         }
 1306         return 0;
 1307       } else {
 1308         // 32-bit
 1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1311         if (masm) {
 1312           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1313                   as_Register(Matcher::_regEncode[src_first]));
 1314 #ifndef PRODUCT
 1315         } else {
 1316           st->print("movl    %s, %s\t# spill",
 1317                      Matcher::regName[dst_first],
 1318                      Matcher::regName[src_first]);
 1319 #endif
 1320         }
 1321         return 0;
 1322       }
 1323     } else if (dst_first_rc == rc_float) {
 1324       // gpr -> xmm
 1325       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1326           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1327         // 64-bit
 1328         if (masm) {
 1329           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1330 #ifndef PRODUCT
 1331         } else {
 1332           st->print("movdq   %s, %s\t# spill",
 1333                      Matcher::regName[dst_first],
 1334                      Matcher::regName[src_first]);
 1335 #endif
 1336         }
 1337       } else {
 1338         // 32-bit
 1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1341         if (masm) {
 1342           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1343 #ifndef PRODUCT
 1344         } else {
 1345           st->print("movdl   %s, %s\t# spill",
 1346                      Matcher::regName[dst_first],
 1347                      Matcher::regName[src_first]);
 1348 #endif
 1349         }
 1350       }
 1351       return 0;
 1352     } else if (dst_first_rc == rc_kreg) {
 1353       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1354           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1355         // 64-bit
 1356         if (masm) {
 1357           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1358   #ifndef PRODUCT
 1359         } else {
 1360            st->print("kmovq   %s, %s\t# spill",
 1361                        Matcher::regName[dst_first],
 1362                        Matcher::regName[src_first]);
 1363   #endif
 1364         }
 1365       }
 1366       Unimplemented();
 1367       return 0;
 1368     }
 1369   } else if (src_first_rc == rc_float) {
 1370     // xmm ->
 1371     if (dst_first_rc == rc_stack) {
 1372       // xmm -> mem
 1373       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1374           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1375         // 64-bit
 1376         int offset = ra_->reg2offset(dst_first);
 1377         if (masm) {
 1378           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1379 #ifndef PRODUCT
 1380         } else {
 1381           st->print("movsd   [rsp + #%d], %s\t# spill",
 1382                      offset,
 1383                      Matcher::regName[src_first]);
 1384 #endif
 1385         }
 1386       } else {
 1387         // 32-bit
 1388         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1389         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1390         int offset = ra_->reg2offset(dst_first);
 1391         if (masm) {
 1392           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1393 #ifndef PRODUCT
 1394         } else {
 1395           st->print("movss   [rsp + #%d], %s\t# spill",
 1396                      offset,
 1397                      Matcher::regName[src_first]);
 1398 #endif
 1399         }
 1400       }
 1401       return 0;
 1402     } else if (dst_first_rc == rc_int) {
 1403       // xmm -> gpr
 1404       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1405           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1406         // 64-bit
 1407         if (masm) {
 1408           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1409 #ifndef PRODUCT
 1410         } else {
 1411           st->print("movdq   %s, %s\t# spill",
 1412                      Matcher::regName[dst_first],
 1413                      Matcher::regName[src_first]);
 1414 #endif
 1415         }
 1416       } else {
 1417         // 32-bit
 1418         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1419         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1420         if (masm) {
 1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1422 #ifndef PRODUCT
 1423         } else {
 1424           st->print("movdl   %s, %s\t# spill",
 1425                      Matcher::regName[dst_first],
 1426                      Matcher::regName[src_first]);
 1427 #endif
 1428         }
 1429       }
 1430       return 0;
 1431     } else if (dst_first_rc == rc_float) {
 1432       // xmm -> xmm
 1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1435         // 64-bit
 1436         if (masm) {
 1437           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1438 #ifndef PRODUCT
 1439         } else {
 1440           st->print("%s  %s, %s\t# spill",
 1441                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1442                      Matcher::regName[dst_first],
 1443                      Matcher::regName[src_first]);
 1444 #endif
 1445         }
 1446       } else {
 1447         // 32-bit
 1448         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1449         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1450         if (masm) {
 1451           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1452 #ifndef PRODUCT
 1453         } else {
 1454           st->print("%s  %s, %s\t# spill",
 1455                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1456                      Matcher::regName[dst_first],
 1457                      Matcher::regName[src_first]);
 1458 #endif
 1459         }
 1460       }
 1461       return 0;
 1462     } else if (dst_first_rc == rc_kreg) {
 1463       assert(false, "Illegal spilling");
 1464       return 0;
 1465     }
 1466   } else if (src_first_rc == rc_kreg) {
 1467     if (dst_first_rc == rc_stack) {
 1468       // mem -> kreg
 1469       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1470           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1471         // 64-bit
 1472         int offset = ra_->reg2offset(dst_first);
 1473         if (masm) {
 1474           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1475 #ifndef PRODUCT
 1476         } else {
 1477           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1478                      offset,
 1479                      Matcher::regName[src_first]);
 1480 #endif
 1481         }
 1482       }
 1483       return 0;
 1484     } else if (dst_first_rc == rc_int) {
 1485       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1486           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1487         // 64-bit
 1488         if (masm) {
 1489           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1490 #ifndef PRODUCT
 1491         } else {
 1492          st->print("kmovq   %s, %s\t# spill",
 1493                      Matcher::regName[dst_first],
 1494                      Matcher::regName[src_first]);
 1495 #endif
 1496         }
 1497       }
 1498       Unimplemented();
 1499       return 0;
 1500     } else if (dst_first_rc == rc_kreg) {
 1501       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1502           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1503         // 64-bit
 1504         if (masm) {
 1505           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1506 #ifndef PRODUCT
 1507         } else {
 1508          st->print("kmovq   %s, %s\t# spill",
 1509                      Matcher::regName[dst_first],
 1510                      Matcher::regName[src_first]);
 1511 #endif
 1512         }
 1513       }
 1514       return 0;
 1515     } else if (dst_first_rc == rc_float) {
 1516       assert(false, "Illegal spill");
 1517       return 0;
 1518     }
 1519   }
 1520 
 1521   assert(0," foo ");
 1522   Unimplemented();
 1523   return 0;
 1524 }
 1525 
 1526 #ifndef PRODUCT
 1527 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1528   implementation(nullptr, ra_, false, st);
 1529 }
 1530 #endif
 1531 
 1532 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1533   implementation(masm, ra_, false, nullptr);
 1534 }
 1535 
 1536 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1537   return MachNode::size(ra_);
 1538 }
 1539 
 1540 //=============================================================================
 1541 #ifndef PRODUCT
 1542 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1543 {
 1544   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1545   int reg = ra_->get_reg_first(this);
 1546   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1547             Matcher::regName[reg], offset);
 1548 }
 1549 #endif
 1550 
 1551 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1552 {
 1553   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1554   int reg = ra_->get_encode(this);
 1555 
 1556   __ lea(as_Register(reg), Address(rsp, offset));
 1557 }
 1558 
 1559 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1560 {
 1561   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1562   if (ra_->get_encode(this) > 15) {
 1563     return (offset < 0x80) ? 6 : 9; // REX2
 1564   } else {
 1565     return (offset < 0x80) ? 5 : 8; // REX
 1566   }
 1567 }
 1568 
 1569 //=============================================================================
 1570 #ifndef PRODUCT
 1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1572 {
 1573   if (UseCompressedClassPointers) {
 1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1575     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1576   } else {
 1577     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1578     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1579   }
 1580   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1581 }
 1582 #endif
 1583 
 1584 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1585 {
 1586   __ ic_check(InteriorEntryAlignment);
 1587 }
 1588 
 1589 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1590 {
 1591   return MachNode::size(ra_); // too many variables; just compute it
 1592                               // the hard way
 1593 }
 1594 
 1595 
 1596 //=============================================================================
 1597 
 1598 bool Matcher::supports_vector_calling_convention(void) {
 1599   return EnableVectorSupport;
 1600 }
 1601 
 1602 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1603   assert(EnableVectorSupport, "sanity");
 1604   int lo = XMM0_num;
 1605   int hi = XMM0b_num;
 1606   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1607   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1608   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1609   return OptoRegPair(hi, lo);
 1610 }
 1611 
 1612 // Is this branch offset short enough that a short branch can be used?
 1613 //
 1614 // NOTE: If the platform does not provide any short branch variants, then
 1615 //       this method should return false for offset 0.
 1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1617   // The passed offset is relative to address of the branch.
 1618   // On 86 a branch displacement is calculated relative to address
 1619   // of a next instruction.
 1620   offset -= br_size;
 1621 
 1622   // the short version of jmpConUCF2 contains multiple branches,
 1623   // making the reach slightly less
 1624   if (rule == jmpConUCF2_rule)
 1625     return (-126 <= offset && offset <= 125);
 1626   return (-128 <= offset && offset <= 127);
 1627 }
 1628 
 1629 // Return whether or not this register is ever used as an argument.
 1630 // This function is used on startup to build the trampoline stubs in
 1631 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1632 // call in the trampoline, and arguments in those registers not be
 1633 // available to the callee.
 1634 bool Matcher::can_be_java_arg(int reg)
 1635 {
 1636   return
 1637     reg ==  RDI_num || reg == RDI_H_num ||
 1638     reg ==  RSI_num || reg == RSI_H_num ||
 1639     reg ==  RDX_num || reg == RDX_H_num ||
 1640     reg ==  RCX_num || reg == RCX_H_num ||
 1641     reg ==   R8_num || reg ==  R8_H_num ||
 1642     reg ==   R9_num || reg ==  R9_H_num ||
 1643     reg ==  R12_num || reg == R12_H_num ||
 1644     reg == XMM0_num || reg == XMM0b_num ||
 1645     reg == XMM1_num || reg == XMM1b_num ||
 1646     reg == XMM2_num || reg == XMM2b_num ||
 1647     reg == XMM3_num || reg == XMM3b_num ||
 1648     reg == XMM4_num || reg == XMM4b_num ||
 1649     reg == XMM5_num || reg == XMM5b_num ||
 1650     reg == XMM6_num || reg == XMM6b_num ||
 1651     reg == XMM7_num || reg == XMM7b_num;
 1652 }
 1653 
 1654 bool Matcher::is_spillable_arg(int reg)
 1655 {
 1656   return can_be_java_arg(reg);
 1657 }
 1658 
 1659 uint Matcher::int_pressure_limit()
 1660 {
 1661   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1662 }
 1663 
 1664 uint Matcher::float_pressure_limit()
 1665 {
 1666   // After experiment around with different values, the following default threshold
 1667   // works best for LCM's register pressure scheduling on x64.
 1668   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1669   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1670   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1671 }
 1672 
 1673 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1674   // In 64 bit mode a code which use multiply when
 1675   // devisor is constant is faster than hardware
 1676   // DIV instruction (it uses MulHiL).
 1677   return false;
 1678 }
 1679 
 1680 // Register for DIVI projection of divmodI
 1681 RegMask Matcher::divI_proj_mask() {
 1682   return INT_RAX_REG_mask();
 1683 }
 1684 
 1685 // Register for MODI projection of divmodI
 1686 RegMask Matcher::modI_proj_mask() {
 1687   return INT_RDX_REG_mask();
 1688 }
 1689 
 1690 // Register for DIVL projection of divmodL
 1691 RegMask Matcher::divL_proj_mask() {
 1692   return LONG_RAX_REG_mask();
 1693 }
 1694 
 1695 // Register for MODL projection of divmodL
 1696 RegMask Matcher::modL_proj_mask() {
 1697   return LONG_RDX_REG_mask();
 1698 }
 1699 
 1700 // Register for saving SP into on method handle invokes. Not used on x86_64.
 1701 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 1702     return NO_REG_mask();
 1703 }
 1704 
 1705 %}
 1706 
 1707 //----------ENCODING BLOCK-----------------------------------------------------
 1708 // This block specifies the encoding classes used by the compiler to
 1709 // output byte streams.  Encoding classes are parameterized macros
 1710 // used by Machine Instruction Nodes in order to generate the bit
 1711 // encoding of the instruction.  Operands specify their base encoding
 1712 // interface with the interface keyword.  There are currently
 1713 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1714 // COND_INTER.  REG_INTER causes an operand to generate a function
 1715 // which returns its register number when queried.  CONST_INTER causes
 1716 // an operand to generate a function which returns the value of the
 1717 // constant when queried.  MEMORY_INTER causes an operand to generate
 1718 // four functions which return the Base Register, the Index Register,
 1719 // the Scale Value, and the Offset Value of the operand when queried.
 1720 // COND_INTER causes an operand to generate six functions which return
 1721 // the encoding code (ie - encoding bits for the instruction)
 1722 // associated with each basic boolean condition for a conditional
 1723 // instruction.
 1724 //
 1725 // Instructions specify two basic values for encoding.  Again, a
 1726 // function is available to check if the constant displacement is an
 1727 // oop. They use the ins_encode keyword to specify their encoding
 1728 // classes (which must be a sequence of enc_class names, and their
 1729 // parameters, specified in the encoding block), and they use the
 1730 // opcode keyword to specify, in order, their primary, secondary, and
 1731 // tertiary opcode.  Only the opcode sections which a particular
 1732 // instruction needs for encoding need to be specified.
 1733 encode %{
 1734   enc_class cdql_enc(no_rax_rdx_RegI div)
 1735   %{
 1736     // Full implementation of Java idiv and irem; checks for
 1737     // special case as described in JVM spec., p.243 & p.271.
 1738     //
 1739     //         normal case                           special case
 1740     //
 1741     // input : rax: dividend                         min_int
 1742     //         reg: divisor                          -1
 1743     //
 1744     // output: rax: quotient  (= rax idiv reg)       min_int
 1745     //         rdx: remainder (= rax irem reg)       0
 1746     //
 1747     //  Code sequnce:
 1748     //
 1749     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1750     //    5:   75 07/08                jne    e <normal>
 1751     //    7:   33 d2                   xor    %edx,%edx
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1755     //    c:   74 03/04                je     11 <done>
 1756     // 000000000000000e <normal>:
 1757     //    e:   99                      cltd
 1758     //  [div >= 8 -> offset + 1]
 1759     //  [REX_B]
 1760     //    f:   f7 f9                   idiv   $div
 1761     // 0000000000000011 <done>:
 1762     Label normal;
 1763     Label done;
 1764 
 1765     // cmp    $0x80000000,%eax
 1766     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1767 
 1768     // jne    e <normal>
 1769     __ jccb(Assembler::notEqual, normal);
 1770 
 1771     // xor    %edx,%edx
 1772     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1773 
 1774     // cmp    $0xffffffffffffffff,%ecx
 1775     __ cmpl($div$$Register, -1);
 1776 
 1777     // je     11 <done>
 1778     __ jccb(Assembler::equal, done);
 1779 
 1780     // <normal>
 1781     // cltd
 1782     __ bind(normal);
 1783     __ cdql();
 1784 
 1785     // idivl
 1786     // <done>
 1787     __ idivl($div$$Register);
 1788     __ bind(done);
 1789   %}
 1790 
 1791   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1792   %{
 1793     // Full implementation of Java ldiv and lrem; checks for
 1794     // special case as described in JVM spec., p.243 & p.271.
 1795     //
 1796     //         normal case                           special case
 1797     //
 1798     // input : rax: dividend                         min_long
 1799     //         reg: divisor                          -1
 1800     //
 1801     // output: rax: quotient  (= rax idiv reg)       min_long
 1802     //         rdx: remainder (= rax irem reg)       0
 1803     //
 1804     //  Code sequnce:
 1805     //
 1806     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1807     //    7:   00 00 80
 1808     //    a:   48 39 d0                cmp    %rdx,%rax
 1809     //    d:   75 08                   jne    17 <normal>
 1810     //    f:   33 d2                   xor    %edx,%edx
 1811     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1812     //   15:   74 05                   je     1c <done>
 1813     // 0000000000000017 <normal>:
 1814     //   17:   48 99                   cqto
 1815     //   19:   48 f7 f9                idiv   $div
 1816     // 000000000000001c <done>:
 1817     Label normal;
 1818     Label done;
 1819 
 1820     // mov    $0x8000000000000000,%rdx
 1821     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1822 
 1823     // cmp    %rdx,%rax
 1824     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1825 
 1826     // jne    17 <normal>
 1827     __ jccb(Assembler::notEqual, normal);
 1828 
 1829     // xor    %edx,%edx
 1830     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1831 
 1832     // cmp    $0xffffffffffffffff,$div
 1833     __ cmpq($div$$Register, -1);
 1834 
 1835     // je     1e <done>
 1836     __ jccb(Assembler::equal, done);
 1837 
 1838     // <normal>
 1839     // cqto
 1840     __ bind(normal);
 1841     __ cdqq();
 1842 
 1843     // idivq (note: must be emitted by the user of this rule)
 1844     // <done>
 1845     __ idivq($div$$Register);
 1846     __ bind(done);
 1847   %}
 1848 
 1849   enc_class clear_avx %{
 1850     DEBUG_ONLY(int off0 = __ offset());
 1851     if (generate_vzeroupper(Compile::current())) {
 1852       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1853       // Clear upper bits of YMM registers when current compiled code uses
 1854       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1855       __ vzeroupper();
 1856     }
 1857     DEBUG_ONLY(int off1 = __ offset());
 1858     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1859   %}
 1860 
 1861   enc_class Java_To_Runtime(method meth) %{
 1862     __ lea(r10, RuntimeAddress((address)$meth$$method));
 1863     __ call(r10);
 1864     __ post_call_nop();
 1865   %}
 1866 
 1867   enc_class Java_Static_Call(method meth)
 1868   %{
 1869     // JAVA STATIC CALL
 1870     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1871     // determine who we intended to call.
 1872     if (!_method) {
 1873       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1874     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1875       // The NOP here is purely to ensure that eliding a call to
 1876       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1877       __ addr_nop_5();
 1878       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1879     } else {
 1880       int method_index = resolved_method_index(masm);
 1881       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1882                                                   : static_call_Relocation::spec(method_index);
 1883       address mark = __ pc();
 1884       int call_offset = __ offset();
 1885       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1886       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1887         // Calls of the same statically bound method can share
 1888         // a stub to the interpreter.
 1889         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1890       } else {
 1891         // Emit stubs for static call.
 1892         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1893         __ clear_inst_mark();
 1894         if (stub == nullptr) {
 1895           ciEnv::current()->record_failure("CodeCache is full");
 1896           return;
 1897         }
 1898       }
 1899     }
 1900     __ post_call_nop();
 1901   %}
 1902 
 1903   enc_class Java_Dynamic_Call(method meth) %{
 1904     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1905     __ post_call_nop();
 1906   %}
 1907 
 1908 %}
 1909 
 1910 
 1911 
 1912 //----------FRAME--------------------------------------------------------------
 1913 // Definition of frame structure and management information.
 1914 //
 1915 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1916 //                             |   (to get allocators register number
 1917 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1918 //  r   CALLER     |        |
 1919 //  o     |        +--------+      pad to even-align allocators stack-slot
 1920 //  w     V        |  pad0  |        numbers; owned by CALLER
 1921 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1922 //  h     ^        |   in   |  5
 1923 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1924 //  |     |        |        |  3
 1925 //  |     |        +--------+
 1926 //  V     |        | old out|      Empty on Intel, window on Sparc
 1927 //        |    old |preserve|      Must be even aligned.
 1928 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1929 //        |        |   in   |  3   area for Intel ret address
 1930 //     Owned by    |preserve|      Empty on Sparc.
 1931 //       SELF      +--------+
 1932 //        |        |  pad2  |  2   pad to align old SP
 1933 //        |        +--------+  1
 1934 //        |        | locks  |  0
 1935 //        |        +--------+----> OptoReg::stack0(), even aligned
 1936 //        |        |  pad1  | 11   pad to align new SP
 1937 //        |        +--------+
 1938 //        |        |        | 10
 1939 //        |        | spills |  9   spills
 1940 //        V        |        |  8   (pad0 slot for callee)
 1941 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1942 //        ^        |  out   |  7
 1943 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1944 //     Owned by    +--------+
 1945 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1946 //        |    new |preserve|      Must be even-aligned.
 1947 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1948 //        |        |        |
 1949 //
 1950 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1951 //         known from SELF's arguments and the Java calling convention.
 1952 //         Region 6-7 is determined per call site.
 1953 // Note 2: If the calling convention leaves holes in the incoming argument
 1954 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1955 //         are owned by the CALLEE.  Holes should not be necessary in the
 1956 //         incoming area, as the Java calling convention is completely under
 1957 //         the control of the AD file.  Doubles can be sorted and packed to
 1958 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1959 //         varargs C calling conventions.
 1960 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1961 //         even aligned with pad0 as needed.
 1962 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1963 //         region 6-11 is even aligned; it may be padded out more so that
 1964 //         the region from SP to FP meets the minimum stack alignment.
 1965 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1966 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1967 //         SP meets the minimum alignment.
 1968 
 1969 frame
 1970 %{
 1971   // These three registers define part of the calling convention
 1972   // between compiled code and the interpreter.
 1973   inline_cache_reg(RAX);                // Inline Cache Register
 1974 
 1975   // Optional: name the operand used by cisc-spilling to access
 1976   // [stack_pointer + offset]
 1977   cisc_spilling_operand_name(indOffset32);
 1978 
 1979   // Number of stack slots consumed by locking an object
 1980   sync_stack_slots(2);
 1981 
 1982   // Compiled code's Frame Pointer
 1983   frame_pointer(RSP);
 1984 
 1985   // Interpreter stores its frame pointer in a register which is
 1986   // stored to the stack by I2CAdaptors.
 1987   // I2CAdaptors convert from interpreted java to compiled java.
 1988   interpreter_frame_pointer(RBP);
 1989 
 1990   // Stack alignment requirement
 1991   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1992 
 1993   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1994   // for calls to C.  Supports the var-args backing area for register parms.
 1995   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1996 
 1997   // The after-PROLOG location of the return address.  Location of
 1998   // return address specifies a type (REG or STACK) and a number
 1999   // representing the register number (i.e. - use a register name) or
 2000   // stack slot.
 2001   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 2002   // Otherwise, it is above the locks and verification slot and alignment word
 2003   return_addr(STACK - 2 +
 2004               align_up((Compile::current()->in_preserve_stack_slots() +
 2005                         Compile::current()->fixed_slots()),
 2006                        stack_alignment_in_slots()));
 2007 
 2008   // Location of compiled Java return values.  Same as C for now.
 2009   return_value
 2010   %{
 2011     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2012            "only return normal values");
 2013 
 2014     static const int lo[Op_RegL + 1] = {
 2015       0,
 2016       0,
 2017       RAX_num,  // Op_RegN
 2018       RAX_num,  // Op_RegI
 2019       RAX_num,  // Op_RegP
 2020       XMM0_num, // Op_RegF
 2021       XMM0_num, // Op_RegD
 2022       RAX_num   // Op_RegL
 2023     };
 2024     static const int hi[Op_RegL + 1] = {
 2025       0,
 2026       0,
 2027       OptoReg::Bad, // Op_RegN
 2028       OptoReg::Bad, // Op_RegI
 2029       RAX_H_num,    // Op_RegP
 2030       OptoReg::Bad, // Op_RegF
 2031       XMM0b_num,    // Op_RegD
 2032       RAX_H_num     // Op_RegL
 2033     };
 2034     // Excluded flags and vector registers.
 2035     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2036     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2037   %}
 2038 %}
 2039 
 2040 //----------ATTRIBUTES---------------------------------------------------------
 2041 //----------Operand Attributes-------------------------------------------------
 2042 op_attrib op_cost(0);        // Required cost attribute
 2043 
 2044 //----------Instruction Attributes---------------------------------------------
 2045 ins_attrib ins_cost(100);       // Required cost attribute
 2046 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2047 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2048                                 // a non-matching short branch variant
 2049                                 // of some long branch?
 2050 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2051                                 // be a power of 2) specifies the
 2052                                 // alignment that some part of the
 2053                                 // instruction (not necessarily the
 2054                                 // start) requires.  If > 1, a
 2055                                 // compute_padding() function must be
 2056                                 // provided for the instruction
 2057 
 2058 // Whether this node is expanded during code emission into a sequence of
 2059 // instructions and the first instruction can perform an implicit null check.
 2060 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 2061 
 2062 //----------OPERANDS-----------------------------------------------------------
 2063 // Operand definitions must precede instruction definitions for correct parsing
 2064 // in the ADLC because operands constitute user defined types which are used in
 2065 // instruction definitions.
 2066 
 2067 //----------Simple Operands----------------------------------------------------
 2068 // Immediate Operands
 2069 // Integer Immediate
 2070 operand immI()
 2071 %{
 2072   match(ConI);
 2073 
 2074   op_cost(10);
 2075   format %{ %}
 2076   interface(CONST_INTER);
 2077 %}
 2078 
 2079 // Constant for test vs zero
 2080 operand immI_0()
 2081 %{
 2082   predicate(n->get_int() == 0);
 2083   match(ConI);
 2084 
 2085   op_cost(0);
 2086   format %{ %}
 2087   interface(CONST_INTER);
 2088 %}
 2089 
 2090 // Constant for increment
 2091 operand immI_1()
 2092 %{
 2093   predicate(n->get_int() == 1);
 2094   match(ConI);
 2095 
 2096   op_cost(0);
 2097   format %{ %}
 2098   interface(CONST_INTER);
 2099 %}
 2100 
 2101 // Constant for decrement
 2102 operand immI_M1()
 2103 %{
 2104   predicate(n->get_int() == -1);
 2105   match(ConI);
 2106 
 2107   op_cost(0);
 2108   format %{ %}
 2109   interface(CONST_INTER);
 2110 %}
 2111 
 2112 operand immI_2()
 2113 %{
 2114   predicate(n->get_int() == 2);
 2115   match(ConI);
 2116 
 2117   op_cost(0);
 2118   format %{ %}
 2119   interface(CONST_INTER);
 2120 %}
 2121 
 2122 operand immI_4()
 2123 %{
 2124   predicate(n->get_int() == 4);
 2125   match(ConI);
 2126 
 2127   op_cost(0);
 2128   format %{ %}
 2129   interface(CONST_INTER);
 2130 %}
 2131 
 2132 operand immI_8()
 2133 %{
 2134   predicate(n->get_int() == 8);
 2135   match(ConI);
 2136 
 2137   op_cost(0);
 2138   format %{ %}
 2139   interface(CONST_INTER);
 2140 %}
 2141 
 2142 // Valid scale values for addressing modes
 2143 operand immI2()
 2144 %{
 2145   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2146   match(ConI);
 2147 
 2148   format %{ %}
 2149   interface(CONST_INTER);
 2150 %}
 2151 
 2152 operand immU7()
 2153 %{
 2154   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2155   match(ConI);
 2156 
 2157   op_cost(5);
 2158   format %{ %}
 2159   interface(CONST_INTER);
 2160 %}
 2161 
 2162 operand immI8()
 2163 %{
 2164   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2165   match(ConI);
 2166 
 2167   op_cost(5);
 2168   format %{ %}
 2169   interface(CONST_INTER);
 2170 %}
 2171 
 2172 operand immU8()
 2173 %{
 2174   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2175   match(ConI);
 2176 
 2177   op_cost(5);
 2178   format %{ %}
 2179   interface(CONST_INTER);
 2180 %}
 2181 
 2182 operand immI16()
 2183 %{
 2184   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2185   match(ConI);
 2186 
 2187   op_cost(10);
 2188   format %{ %}
 2189   interface(CONST_INTER);
 2190 %}
 2191 
 2192 // Int Immediate non-negative
 2193 operand immU31()
 2194 %{
 2195   predicate(n->get_int() >= 0);
 2196   match(ConI);
 2197 
 2198   op_cost(0);
 2199   format %{ %}
 2200   interface(CONST_INTER);
 2201 %}
 2202 
 2203 // Pointer Immediate
 2204 operand immP()
 2205 %{
 2206   match(ConP);
 2207 
 2208   op_cost(10);
 2209   format %{ %}
 2210   interface(CONST_INTER);
 2211 %}
 2212 
 2213 // Null Pointer Immediate
 2214 operand immP0()
 2215 %{
 2216   predicate(n->get_ptr() == 0);
 2217   match(ConP);
 2218 
 2219   op_cost(5);
 2220   format %{ %}
 2221   interface(CONST_INTER);
 2222 %}
 2223 
 2224 // Pointer Immediate
 2225 operand immN() %{
 2226   match(ConN);
 2227 
 2228   op_cost(10);
 2229   format %{ %}
 2230   interface(CONST_INTER);
 2231 %}
 2232 
 2233 operand immNKlass() %{
 2234   match(ConNKlass);
 2235 
 2236   op_cost(10);
 2237   format %{ %}
 2238   interface(CONST_INTER);
 2239 %}
 2240 
 2241 // Null Pointer Immediate
 2242 operand immN0() %{
 2243   predicate(n->get_narrowcon() == 0);
 2244   match(ConN);
 2245 
 2246   op_cost(5);
 2247   format %{ %}
 2248   interface(CONST_INTER);
 2249 %}
 2250 
 2251 operand immP31()
 2252 %{
 2253   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2254             && (n->get_ptr() >> 31) == 0);
 2255   match(ConP);
 2256 
 2257   op_cost(5);
 2258   format %{ %}
 2259   interface(CONST_INTER);
 2260 %}
 2261 
 2262 
 2263 // Long Immediate
 2264 operand immL()
 2265 %{
 2266   match(ConL);
 2267 
 2268   op_cost(20);
 2269   format %{ %}
 2270   interface(CONST_INTER);
 2271 %}
 2272 
 2273 // Long Immediate 8-bit
 2274 operand immL8()
 2275 %{
 2276   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2277   match(ConL);
 2278 
 2279   op_cost(5);
 2280   format %{ %}
 2281   interface(CONST_INTER);
 2282 %}
 2283 
 2284 // Long Immediate 32-bit unsigned
 2285 operand immUL32()
 2286 %{
 2287   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2288   match(ConL);
 2289 
 2290   op_cost(10);
 2291   format %{ %}
 2292   interface(CONST_INTER);
 2293 %}
 2294 
 2295 // Long Immediate 32-bit signed
 2296 operand immL32()
 2297 %{
 2298   predicate(n->get_long() == (int) (n->get_long()));
 2299   match(ConL);
 2300 
 2301   op_cost(15);
 2302   format %{ %}
 2303   interface(CONST_INTER);
 2304 %}
 2305 
 2306 operand immL_Pow2()
 2307 %{
 2308   predicate(is_power_of_2((julong)n->get_long()));
 2309   match(ConL);
 2310 
 2311   op_cost(15);
 2312   format %{ %}
 2313   interface(CONST_INTER);
 2314 %}
 2315 
 2316 operand immL_NotPow2()
 2317 %{
 2318   predicate(is_power_of_2((julong)~n->get_long()));
 2319   match(ConL);
 2320 
 2321   op_cost(15);
 2322   format %{ %}
 2323   interface(CONST_INTER);
 2324 %}
 2325 
 2326 // Long Immediate zero
 2327 operand immL0()
 2328 %{
 2329   predicate(n->get_long() == 0L);
 2330   match(ConL);
 2331 
 2332   op_cost(10);
 2333   format %{ %}
 2334   interface(CONST_INTER);
 2335 %}
 2336 
 2337 // Constant for increment
 2338 operand immL1()
 2339 %{
 2340   predicate(n->get_long() == 1);
 2341   match(ConL);
 2342 
 2343   format %{ %}
 2344   interface(CONST_INTER);
 2345 %}
 2346 
 2347 // Constant for decrement
 2348 operand immL_M1()
 2349 %{
 2350   predicate(n->get_long() == -1);
 2351   match(ConL);
 2352 
 2353   format %{ %}
 2354   interface(CONST_INTER);
 2355 %}
 2356 
 2357 // Long Immediate: low 32-bit mask
 2358 operand immL_32bits()
 2359 %{
 2360   predicate(n->get_long() == 0xFFFFFFFFL);
 2361   match(ConL);
 2362   op_cost(20);
 2363 
 2364   format %{ %}
 2365   interface(CONST_INTER);
 2366 %}
 2367 
 2368 // Int Immediate: 2^n-1, positive
 2369 operand immI_Pow2M1()
 2370 %{
 2371   predicate((n->get_int() > 0)
 2372             && is_power_of_2((juint)n->get_int() + 1));
 2373   match(ConI);
 2374 
 2375   op_cost(20);
 2376   format %{ %}
 2377   interface(CONST_INTER);
 2378 %}
 2379 
 2380 // Float Immediate zero
 2381 operand immF0()
 2382 %{
 2383   predicate(jint_cast(n->getf()) == 0);
 2384   match(ConF);
 2385 
 2386   op_cost(5);
 2387   format %{ %}
 2388   interface(CONST_INTER);
 2389 %}
 2390 
 2391 // Float Immediate
 2392 operand immF()
 2393 %{
 2394   match(ConF);
 2395 
 2396   op_cost(15);
 2397   format %{ %}
 2398   interface(CONST_INTER);
 2399 %}
 2400 
 2401 // Half Float Immediate
 2402 operand immH()
 2403 %{
 2404   match(ConH);
 2405 
 2406   op_cost(15);
 2407   format %{ %}
 2408   interface(CONST_INTER);
 2409 %}
 2410 
 2411 // Double Immediate zero
 2412 operand immD0()
 2413 %{
 2414   predicate(jlong_cast(n->getd()) == 0);
 2415   match(ConD);
 2416 
 2417   op_cost(5);
 2418   format %{ %}
 2419   interface(CONST_INTER);
 2420 %}
 2421 
 2422 // Double Immediate
 2423 operand immD()
 2424 %{
 2425   match(ConD);
 2426 
 2427   op_cost(15);
 2428   format %{ %}
 2429   interface(CONST_INTER);
 2430 %}
 2431 
 2432 // Immediates for special shifts (sign extend)
 2433 
 2434 // Constants for increment
 2435 operand immI_16()
 2436 %{
 2437   predicate(n->get_int() == 16);
 2438   match(ConI);
 2439 
 2440   format %{ %}
 2441   interface(CONST_INTER);
 2442 %}
 2443 
 2444 operand immI_24()
 2445 %{
 2446   predicate(n->get_int() == 24);
 2447   match(ConI);
 2448 
 2449   format %{ %}
 2450   interface(CONST_INTER);
 2451 %}
 2452 
 2453 // Constant for byte-wide masking
 2454 operand immI_255()
 2455 %{
 2456   predicate(n->get_int() == 255);
 2457   match(ConI);
 2458 
 2459   format %{ %}
 2460   interface(CONST_INTER);
 2461 %}
 2462 
 2463 // Constant for short-wide masking
 2464 operand immI_65535()
 2465 %{
 2466   predicate(n->get_int() == 65535);
 2467   match(ConI);
 2468 
 2469   format %{ %}
 2470   interface(CONST_INTER);
 2471 %}
 2472 
 2473 // Constant for byte-wide masking
 2474 operand immL_255()
 2475 %{
 2476   predicate(n->get_long() == 255);
 2477   match(ConL);
 2478 
 2479   format %{ %}
 2480   interface(CONST_INTER);
 2481 %}
 2482 
 2483 // Constant for short-wide masking
 2484 operand immL_65535()
 2485 %{
 2486   predicate(n->get_long() == 65535);
 2487   match(ConL);
 2488 
 2489   format %{ %}
 2490   interface(CONST_INTER);
 2491 %}
 2492 
 2493 operand kReg()
 2494 %{
 2495   constraint(ALLOC_IN_RC(vectmask_reg));
 2496   match(RegVectMask);
 2497   format %{%}
 2498   interface(REG_INTER);
 2499 %}
 2500 
 2501 // Register Operands
 2502 // Integer Register
 2503 operand rRegI()
 2504 %{
 2505   constraint(ALLOC_IN_RC(int_reg));
 2506   match(RegI);
 2507 
 2508   match(rax_RegI);
 2509   match(rbx_RegI);
 2510   match(rcx_RegI);
 2511   match(rdx_RegI);
 2512   match(rdi_RegI);
 2513 
 2514   format %{ %}
 2515   interface(REG_INTER);
 2516 %}
 2517 
 2518 // Special Registers
 2519 operand rax_RegI()
 2520 %{
 2521   constraint(ALLOC_IN_RC(int_rax_reg));
 2522   match(RegI);
 2523   match(rRegI);
 2524 
 2525   format %{ "RAX" %}
 2526   interface(REG_INTER);
 2527 %}
 2528 
 2529 // Special Registers
 2530 operand rbx_RegI()
 2531 %{
 2532   constraint(ALLOC_IN_RC(int_rbx_reg));
 2533   match(RegI);
 2534   match(rRegI);
 2535 
 2536   format %{ "RBX" %}
 2537   interface(REG_INTER);
 2538 %}
 2539 
 2540 operand rcx_RegI()
 2541 %{
 2542   constraint(ALLOC_IN_RC(int_rcx_reg));
 2543   match(RegI);
 2544   match(rRegI);
 2545 
 2546   format %{ "RCX" %}
 2547   interface(REG_INTER);
 2548 %}
 2549 
 2550 operand rdx_RegI()
 2551 %{
 2552   constraint(ALLOC_IN_RC(int_rdx_reg));
 2553   match(RegI);
 2554   match(rRegI);
 2555 
 2556   format %{ "RDX" %}
 2557   interface(REG_INTER);
 2558 %}
 2559 
 2560 operand rdi_RegI()
 2561 %{
 2562   constraint(ALLOC_IN_RC(int_rdi_reg));
 2563   match(RegI);
 2564   match(rRegI);
 2565 
 2566   format %{ "RDI" %}
 2567   interface(REG_INTER);
 2568 %}
 2569 
 2570 operand no_rax_rdx_RegI()
 2571 %{
 2572   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2573   match(RegI);
 2574   match(rbx_RegI);
 2575   match(rcx_RegI);
 2576   match(rdi_RegI);
 2577 
 2578   format %{ %}
 2579   interface(REG_INTER);
 2580 %}
 2581 
 2582 operand no_rbp_r13_RegI()
 2583 %{
 2584   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2585   match(RegI);
 2586   match(rRegI);
 2587   match(rax_RegI);
 2588   match(rbx_RegI);
 2589   match(rcx_RegI);
 2590   match(rdx_RegI);
 2591   match(rdi_RegI);
 2592 
 2593   format %{ %}
 2594   interface(REG_INTER);
 2595 %}
 2596 
 2597 // Pointer Register
 2598 operand any_RegP()
 2599 %{
 2600   constraint(ALLOC_IN_RC(any_reg));
 2601   match(RegP);
 2602   match(rax_RegP);
 2603   match(rbx_RegP);
 2604   match(rdi_RegP);
 2605   match(rsi_RegP);
 2606   match(rbp_RegP);
 2607   match(r15_RegP);
 2608   match(rRegP);
 2609 
 2610   format %{ %}
 2611   interface(REG_INTER);
 2612 %}
 2613 
 2614 operand rRegP()
 2615 %{
 2616   constraint(ALLOC_IN_RC(ptr_reg));
 2617   match(RegP);
 2618   match(rax_RegP);
 2619   match(rbx_RegP);
 2620   match(rdi_RegP);
 2621   match(rsi_RegP);
 2622   match(rbp_RegP);  // See Q&A below about
 2623   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2624 
 2625   format %{ %}
 2626   interface(REG_INTER);
 2627 %}
 2628 
 2629 operand rRegN() %{
 2630   constraint(ALLOC_IN_RC(int_reg));
 2631   match(RegN);
 2632 
 2633   format %{ %}
 2634   interface(REG_INTER);
 2635 %}
 2636 
 2637 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2638 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2639 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2640 // The output of an instruction is controlled by the allocator, which respects
 2641 // register class masks, not match rules.  Unless an instruction mentions
 2642 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2643 // by the allocator as an input.
 2644 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2645 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2646 // result, RBP is not included in the output of the instruction either.
 2647 
 2648 // This operand is not allowed to use RBP even if
 2649 // RBP is not used to hold the frame pointer.
 2650 operand no_rbp_RegP()
 2651 %{
 2652   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2653   match(RegP);
 2654   match(rbx_RegP);
 2655   match(rsi_RegP);
 2656   match(rdi_RegP);
 2657 
 2658   format %{ %}
 2659   interface(REG_INTER);
 2660 %}
 2661 
 2662 // Special Registers
 2663 // Return a pointer value
 2664 operand rax_RegP()
 2665 %{
 2666   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2667   match(RegP);
 2668   match(rRegP);
 2669 
 2670   format %{ %}
 2671   interface(REG_INTER);
 2672 %}
 2673 
 2674 // Special Registers
 2675 // Return a compressed pointer value
 2676 operand rax_RegN()
 2677 %{
 2678   constraint(ALLOC_IN_RC(int_rax_reg));
 2679   match(RegN);
 2680   match(rRegN);
 2681 
 2682   format %{ %}
 2683   interface(REG_INTER);
 2684 %}
 2685 
 2686 // Used in AtomicAdd
 2687 operand rbx_RegP()
 2688 %{
 2689   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2690   match(RegP);
 2691   match(rRegP);
 2692 
 2693   format %{ %}
 2694   interface(REG_INTER);
 2695 %}
 2696 
 2697 operand rsi_RegP()
 2698 %{
 2699   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2700   match(RegP);
 2701   match(rRegP);
 2702 
 2703   format %{ %}
 2704   interface(REG_INTER);
 2705 %}
 2706 
 2707 operand rbp_RegP()
 2708 %{
 2709   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2710   match(RegP);
 2711   match(rRegP);
 2712 
 2713   format %{ %}
 2714   interface(REG_INTER);
 2715 %}
 2716 
 2717 // Used in rep stosq
 2718 operand rdi_RegP()
 2719 %{
 2720   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2721   match(RegP);
 2722   match(rRegP);
 2723 
 2724   format %{ %}
 2725   interface(REG_INTER);
 2726 %}
 2727 
 2728 operand r15_RegP()
 2729 %{
 2730   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2731   match(RegP);
 2732   match(rRegP);
 2733 
 2734   format %{ %}
 2735   interface(REG_INTER);
 2736 %}
 2737 
 2738 operand rRegL()
 2739 %{
 2740   constraint(ALLOC_IN_RC(long_reg));
 2741   match(RegL);
 2742   match(rax_RegL);
 2743   match(rdx_RegL);
 2744 
 2745   format %{ %}
 2746   interface(REG_INTER);
 2747 %}
 2748 
 2749 // Special Registers
 2750 operand no_rax_rdx_RegL()
 2751 %{
 2752   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2753   match(RegL);
 2754   match(rRegL);
 2755 
 2756   format %{ %}
 2757   interface(REG_INTER);
 2758 %}
 2759 
 2760 operand rax_RegL()
 2761 %{
 2762   constraint(ALLOC_IN_RC(long_rax_reg));
 2763   match(RegL);
 2764   match(rRegL);
 2765 
 2766   format %{ "RAX" %}
 2767   interface(REG_INTER);
 2768 %}
 2769 
 2770 operand rcx_RegL()
 2771 %{
 2772   constraint(ALLOC_IN_RC(long_rcx_reg));
 2773   match(RegL);
 2774   match(rRegL);
 2775 
 2776   format %{ %}
 2777   interface(REG_INTER);
 2778 %}
 2779 
 2780 operand rdx_RegL()
 2781 %{
 2782   constraint(ALLOC_IN_RC(long_rdx_reg));
 2783   match(RegL);
 2784   match(rRegL);
 2785 
 2786   format %{ %}
 2787   interface(REG_INTER);
 2788 %}
 2789 
 2790 operand r11_RegL()
 2791 %{
 2792   constraint(ALLOC_IN_RC(long_r11_reg));
 2793   match(RegL);
 2794   match(rRegL);
 2795 
 2796   format %{ %}
 2797   interface(REG_INTER);
 2798 %}
 2799 
 2800 operand no_rbp_r13_RegL()
 2801 %{
 2802   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2803   match(RegL);
 2804   match(rRegL);
 2805   match(rax_RegL);
 2806   match(rcx_RegL);
 2807   match(rdx_RegL);
 2808 
 2809   format %{ %}
 2810   interface(REG_INTER);
 2811 %}
 2812 
 2813 // Flags register, used as output of compare instructions
 2814 operand rFlagsReg()
 2815 %{
 2816   constraint(ALLOC_IN_RC(int_flags));
 2817   match(RegFlags);
 2818 
 2819   format %{ "RFLAGS" %}
 2820   interface(REG_INTER);
 2821 %}
 2822 
 2823 // Flags register, used as output of FLOATING POINT compare instructions
 2824 operand rFlagsRegU()
 2825 %{
 2826   constraint(ALLOC_IN_RC(int_flags));
 2827   match(RegFlags);
 2828 
 2829   format %{ "RFLAGS_U" %}
 2830   interface(REG_INTER);
 2831 %}
 2832 
 2833 operand rFlagsRegUCF() %{
 2834   constraint(ALLOC_IN_RC(int_flags));
 2835   match(RegFlags);
 2836   predicate(false);
 2837 
 2838   format %{ "RFLAGS_U_CF" %}
 2839   interface(REG_INTER);
 2840 %}
 2841 
 2842 // Float register operands
 2843 operand regF() %{
 2844    constraint(ALLOC_IN_RC(float_reg));
 2845    match(RegF);
 2846 
 2847    format %{ %}
 2848    interface(REG_INTER);
 2849 %}
 2850 
 2851 // Float register operands
 2852 operand legRegF() %{
 2853    constraint(ALLOC_IN_RC(float_reg_legacy));
 2854    match(RegF);
 2855 
 2856    format %{ %}
 2857    interface(REG_INTER);
 2858 %}
 2859 
 2860 // Float register operands
 2861 operand vlRegF() %{
 2862    constraint(ALLOC_IN_RC(float_reg_vl));
 2863    match(RegF);
 2864 
 2865    format %{ %}
 2866    interface(REG_INTER);
 2867 %}
 2868 
 2869 // Double register operands
 2870 operand regD() %{
 2871    constraint(ALLOC_IN_RC(double_reg));
 2872    match(RegD);
 2873 
 2874    format %{ %}
 2875    interface(REG_INTER);
 2876 %}
 2877 
 2878 // Double register operands
 2879 operand legRegD() %{
 2880    constraint(ALLOC_IN_RC(double_reg_legacy));
 2881    match(RegD);
 2882 
 2883    format %{ %}
 2884    interface(REG_INTER);
 2885 %}
 2886 
 2887 // Double register operands
 2888 operand vlRegD() %{
 2889    constraint(ALLOC_IN_RC(double_reg_vl));
 2890    match(RegD);
 2891 
 2892    format %{ %}
 2893    interface(REG_INTER);
 2894 %}
 2895 
 2896 //----------Memory Operands----------------------------------------------------
 2897 // Direct Memory Operand
 2898 // operand direct(immP addr)
 2899 // %{
 2900 //   match(addr);
 2901 
 2902 //   format %{ "[$addr]" %}
 2903 //   interface(MEMORY_INTER) %{
 2904 //     base(0xFFFFFFFF);
 2905 //     index(0x4);
 2906 //     scale(0x0);
 2907 //     disp($addr);
 2908 //   %}
 2909 // %}
 2910 
 2911 // Indirect Memory Operand
 2912 operand indirect(any_RegP reg)
 2913 %{
 2914   constraint(ALLOC_IN_RC(ptr_reg));
 2915   match(reg);
 2916 
 2917   format %{ "[$reg]" %}
 2918   interface(MEMORY_INTER) %{
 2919     base($reg);
 2920     index(0x4);
 2921     scale(0x0);
 2922     disp(0x0);
 2923   %}
 2924 %}
 2925 
 2926 // Indirect Memory Plus Short Offset Operand
 2927 operand indOffset8(any_RegP reg, immL8 off)
 2928 %{
 2929   constraint(ALLOC_IN_RC(ptr_reg));
 2930   match(AddP reg off);
 2931 
 2932   format %{ "[$reg + $off (8-bit)]" %}
 2933   interface(MEMORY_INTER) %{
 2934     base($reg);
 2935     index(0x4);
 2936     scale(0x0);
 2937     disp($off);
 2938   %}
 2939 %}
 2940 
 2941 // Indirect Memory Plus Long Offset Operand
 2942 operand indOffset32(any_RegP reg, immL32 off)
 2943 %{
 2944   constraint(ALLOC_IN_RC(ptr_reg));
 2945   match(AddP reg off);
 2946 
 2947   format %{ "[$reg + $off (32-bit)]" %}
 2948   interface(MEMORY_INTER) %{
 2949     base($reg);
 2950     index(0x4);
 2951     scale(0x0);
 2952     disp($off);
 2953   %}
 2954 %}
 2955 
 2956 // Indirect Memory Plus Index Register Plus Offset Operand
 2957 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2958 %{
 2959   constraint(ALLOC_IN_RC(ptr_reg));
 2960   match(AddP (AddP reg lreg) off);
 2961 
 2962   op_cost(10);
 2963   format %{"[$reg + $off + $lreg]" %}
 2964   interface(MEMORY_INTER) %{
 2965     base($reg);
 2966     index($lreg);
 2967     scale(0x0);
 2968     disp($off);
 2969   %}
 2970 %}
 2971 
 2972 // Indirect Memory Plus Index Register Plus Offset Operand
 2973 operand indIndex(any_RegP reg, rRegL lreg)
 2974 %{
 2975   constraint(ALLOC_IN_RC(ptr_reg));
 2976   match(AddP reg lreg);
 2977 
 2978   op_cost(10);
 2979   format %{"[$reg + $lreg]" %}
 2980   interface(MEMORY_INTER) %{
 2981     base($reg);
 2982     index($lreg);
 2983     scale(0x0);
 2984     disp(0x0);
 2985   %}
 2986 %}
 2987 
 2988 // Indirect Memory Times Scale Plus Index Register
 2989 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2990 %{
 2991   constraint(ALLOC_IN_RC(ptr_reg));
 2992   match(AddP reg (LShiftL lreg scale));
 2993 
 2994   op_cost(10);
 2995   format %{"[$reg + $lreg << $scale]" %}
 2996   interface(MEMORY_INTER) %{
 2997     base($reg);
 2998     index($lreg);
 2999     scale($scale);
 3000     disp(0x0);
 3001   %}
 3002 %}
 3003 
 3004 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3005 %{
 3006   constraint(ALLOC_IN_RC(ptr_reg));
 3007   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3008   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3009 
 3010   op_cost(10);
 3011   format %{"[$reg + pos $idx << $scale]" %}
 3012   interface(MEMORY_INTER) %{
 3013     base($reg);
 3014     index($idx);
 3015     scale($scale);
 3016     disp(0x0);
 3017   %}
 3018 %}
 3019 
 3020 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3021 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3022 %{
 3023   constraint(ALLOC_IN_RC(ptr_reg));
 3024   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3025 
 3026   op_cost(10);
 3027   format %{"[$reg + $off + $lreg << $scale]" %}
 3028   interface(MEMORY_INTER) %{
 3029     base($reg);
 3030     index($lreg);
 3031     scale($scale);
 3032     disp($off);
 3033   %}
 3034 %}
 3035 
 3036 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3037 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3038 %{
 3039   constraint(ALLOC_IN_RC(ptr_reg));
 3040   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3041   match(AddP (AddP reg (ConvI2L idx)) off);
 3042 
 3043   op_cost(10);
 3044   format %{"[$reg + $off + $idx]" %}
 3045   interface(MEMORY_INTER) %{
 3046     base($reg);
 3047     index($idx);
 3048     scale(0x0);
 3049     disp($off);
 3050   %}
 3051 %}
 3052 
 3053 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3054 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3055 %{
 3056   constraint(ALLOC_IN_RC(ptr_reg));
 3057   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3058   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3059 
 3060   op_cost(10);
 3061   format %{"[$reg + $off + $idx << $scale]" %}
 3062   interface(MEMORY_INTER) %{
 3063     base($reg);
 3064     index($idx);
 3065     scale($scale);
 3066     disp($off);
 3067   %}
 3068 %}
 3069 
 3070 // Indirect Narrow Oop Plus Offset Operand
 3071 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3072 // we can't free r12 even with CompressedOops::base() == nullptr.
 3073 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3074   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3075   constraint(ALLOC_IN_RC(ptr_reg));
 3076   match(AddP (DecodeN reg) off);
 3077 
 3078   op_cost(10);
 3079   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3080   interface(MEMORY_INTER) %{
 3081     base(0xc); // R12
 3082     index($reg);
 3083     scale(0x3);
 3084     disp($off);
 3085   %}
 3086 %}
 3087 
 3088 // Indirect Memory Operand
 3089 operand indirectNarrow(rRegN reg)
 3090 %{
 3091   predicate(CompressedOops::shift() == 0);
 3092   constraint(ALLOC_IN_RC(ptr_reg));
 3093   match(DecodeN reg);
 3094 
 3095   format %{ "[$reg]" %}
 3096   interface(MEMORY_INTER) %{
 3097     base($reg);
 3098     index(0x4);
 3099     scale(0x0);
 3100     disp(0x0);
 3101   %}
 3102 %}
 3103 
 3104 // Indirect Memory Plus Short Offset Operand
 3105 operand indOffset8Narrow(rRegN reg, immL8 off)
 3106 %{
 3107   predicate(CompressedOops::shift() == 0);
 3108   constraint(ALLOC_IN_RC(ptr_reg));
 3109   match(AddP (DecodeN reg) off);
 3110 
 3111   format %{ "[$reg + $off (8-bit)]" %}
 3112   interface(MEMORY_INTER) %{
 3113     base($reg);
 3114     index(0x4);
 3115     scale(0x0);
 3116     disp($off);
 3117   %}
 3118 %}
 3119 
 3120 // Indirect Memory Plus Long Offset Operand
 3121 operand indOffset32Narrow(rRegN reg, immL32 off)
 3122 %{
 3123   predicate(CompressedOops::shift() == 0);
 3124   constraint(ALLOC_IN_RC(ptr_reg));
 3125   match(AddP (DecodeN reg) off);
 3126 
 3127   format %{ "[$reg + $off (32-bit)]" %}
 3128   interface(MEMORY_INTER) %{
 3129     base($reg);
 3130     index(0x4);
 3131     scale(0x0);
 3132     disp($off);
 3133   %}
 3134 %}
 3135 
 3136 // Indirect Memory Plus Index Register Plus Offset Operand
 3137 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3138 %{
 3139   predicate(CompressedOops::shift() == 0);
 3140   constraint(ALLOC_IN_RC(ptr_reg));
 3141   match(AddP (AddP (DecodeN reg) lreg) off);
 3142 
 3143   op_cost(10);
 3144   format %{"[$reg + $off + $lreg]" %}
 3145   interface(MEMORY_INTER) %{
 3146     base($reg);
 3147     index($lreg);
 3148     scale(0x0);
 3149     disp($off);
 3150   %}
 3151 %}
 3152 
 3153 // Indirect Memory Plus Index Register Plus Offset Operand
 3154 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3155 %{
 3156   predicate(CompressedOops::shift() == 0);
 3157   constraint(ALLOC_IN_RC(ptr_reg));
 3158   match(AddP (DecodeN reg) lreg);
 3159 
 3160   op_cost(10);
 3161   format %{"[$reg + $lreg]" %}
 3162   interface(MEMORY_INTER) %{
 3163     base($reg);
 3164     index($lreg);
 3165     scale(0x0);
 3166     disp(0x0);
 3167   %}
 3168 %}
 3169 
 3170 // Indirect Memory Times Scale Plus Index Register
 3171 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3172 %{
 3173   predicate(CompressedOops::shift() == 0);
 3174   constraint(ALLOC_IN_RC(ptr_reg));
 3175   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3176 
 3177   op_cost(10);
 3178   format %{"[$reg + $lreg << $scale]" %}
 3179   interface(MEMORY_INTER) %{
 3180     base($reg);
 3181     index($lreg);
 3182     scale($scale);
 3183     disp(0x0);
 3184   %}
 3185 %}
 3186 
 3187 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3188 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3189 %{
 3190   predicate(CompressedOops::shift() == 0);
 3191   constraint(ALLOC_IN_RC(ptr_reg));
 3192   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3193 
 3194   op_cost(10);
 3195   format %{"[$reg + $off + $lreg << $scale]" %}
 3196   interface(MEMORY_INTER) %{
 3197     base($reg);
 3198     index($lreg);
 3199     scale($scale);
 3200     disp($off);
 3201   %}
 3202 %}
 3203 
 3204 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3205 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3206 %{
 3207   constraint(ALLOC_IN_RC(ptr_reg));
 3208   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3209   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3210 
 3211   op_cost(10);
 3212   format %{"[$reg + $off + $idx]" %}
 3213   interface(MEMORY_INTER) %{
 3214     base($reg);
 3215     index($idx);
 3216     scale(0x0);
 3217     disp($off);
 3218   %}
 3219 %}
 3220 
 3221 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3222 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3223 %{
 3224   constraint(ALLOC_IN_RC(ptr_reg));
 3225   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3226   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3227 
 3228   op_cost(10);
 3229   format %{"[$reg + $off + $idx << $scale]" %}
 3230   interface(MEMORY_INTER) %{
 3231     base($reg);
 3232     index($idx);
 3233     scale($scale);
 3234     disp($off);
 3235   %}
 3236 %}
 3237 
 3238 //----------Special Memory Operands--------------------------------------------
 3239 // Stack Slot Operand - This operand is used for loading and storing temporary
 3240 //                      values on the stack where a match requires a value to
 3241 //                      flow through memory.
 3242 operand stackSlotP(sRegP reg)
 3243 %{
 3244   constraint(ALLOC_IN_RC(stack_slots));
 3245   // No match rule because this operand is only generated in matching
 3246 
 3247   format %{ "[$reg]" %}
 3248   interface(MEMORY_INTER) %{
 3249     base(0x4);   // RSP
 3250     index(0x4);  // No Index
 3251     scale(0x0);  // No Scale
 3252     disp($reg);  // Stack Offset
 3253   %}
 3254 %}
 3255 
 3256 operand stackSlotI(sRegI reg)
 3257 %{
 3258   constraint(ALLOC_IN_RC(stack_slots));
 3259   // No match rule because this operand is only generated in matching
 3260 
 3261   format %{ "[$reg]" %}
 3262   interface(MEMORY_INTER) %{
 3263     base(0x4);   // RSP
 3264     index(0x4);  // No Index
 3265     scale(0x0);  // No Scale
 3266     disp($reg);  // Stack Offset
 3267   %}
 3268 %}
 3269 
 3270 operand stackSlotF(sRegF reg)
 3271 %{
 3272   constraint(ALLOC_IN_RC(stack_slots));
 3273   // No match rule because this operand is only generated in matching
 3274 
 3275   format %{ "[$reg]" %}
 3276   interface(MEMORY_INTER) %{
 3277     base(0x4);   // RSP
 3278     index(0x4);  // No Index
 3279     scale(0x0);  // No Scale
 3280     disp($reg);  // Stack Offset
 3281   %}
 3282 %}
 3283 
 3284 operand stackSlotD(sRegD reg)
 3285 %{
 3286   constraint(ALLOC_IN_RC(stack_slots));
 3287   // No match rule because this operand is only generated in matching
 3288 
 3289   format %{ "[$reg]" %}
 3290   interface(MEMORY_INTER) %{
 3291     base(0x4);   // RSP
 3292     index(0x4);  // No Index
 3293     scale(0x0);  // No Scale
 3294     disp($reg);  // Stack Offset
 3295   %}
 3296 %}
 3297 operand stackSlotL(sRegL reg)
 3298 %{
 3299   constraint(ALLOC_IN_RC(stack_slots));
 3300   // No match rule because this operand is only generated in matching
 3301 
 3302   format %{ "[$reg]" %}
 3303   interface(MEMORY_INTER) %{
 3304     base(0x4);   // RSP
 3305     index(0x4);  // No Index
 3306     scale(0x0);  // No Scale
 3307     disp($reg);  // Stack Offset
 3308   %}
 3309 %}
 3310 
 3311 //----------Conditional Branch Operands----------------------------------------
 3312 // Comparison Op  - This is the operation of the comparison, and is limited to
 3313 //                  the following set of codes:
 3314 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3315 //
 3316 // Other attributes of the comparison, such as unsignedness, are specified
 3317 // by the comparison instruction that sets a condition code flags register.
 3318 // That result is represented by a flags operand whose subtype is appropriate
 3319 // to the unsignedness (etc.) of the comparison.
 3320 //
 3321 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3322 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3323 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3324 
 3325 // Comparison Code
 3326 operand cmpOp()
 3327 %{
 3328   match(Bool);
 3329 
 3330   format %{ "" %}
 3331   interface(COND_INTER) %{
 3332     equal(0x4, "e");
 3333     not_equal(0x5, "ne");
 3334     less(0xC, "l");
 3335     greater_equal(0xD, "ge");
 3336     less_equal(0xE, "le");
 3337     greater(0xF, "g");
 3338     overflow(0x0, "o");
 3339     no_overflow(0x1, "no");
 3340   %}
 3341 %}
 3342 
 3343 // Comparison Code, unsigned compare.  Used by FP also, with
 3344 // C2 (unordered) turned into GT or LT already.  The other bits
 3345 // C0 and C3 are turned into Carry & Zero flags.
 3346 operand cmpOpU()
 3347 %{
 3348   match(Bool);
 3349 
 3350   format %{ "" %}
 3351   interface(COND_INTER) %{
 3352     equal(0x4, "e");
 3353     not_equal(0x5, "ne");
 3354     less(0x2, "b");
 3355     greater_equal(0x3, "ae");
 3356     less_equal(0x6, "be");
 3357     greater(0x7, "a");
 3358     overflow(0x0, "o");
 3359     no_overflow(0x1, "no");
 3360   %}
 3361 %}
 3362 
 3363 
 3364 // Floating comparisons that don't require any fixup for the unordered case,
 3365 // If both inputs of the comparison are the same, ZF is always set so we
 3366 // don't need to use cmpOpUCF2 for eq/ne
 3367 operand cmpOpUCF() %{
 3368   match(Bool);
 3369   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3370             n->as_Bool()->_test._test == BoolTest::ge ||
 3371             n->as_Bool()->_test._test == BoolTest::le ||
 3372             n->as_Bool()->_test._test == BoolTest::gt ||
 3373             n->in(1)->in(1) == n->in(1)->in(2));
 3374   format %{ "" %}
 3375   interface(COND_INTER) %{
 3376     equal(0xb, "np");
 3377     not_equal(0xa, "p");
 3378     less(0x2, "b");
 3379     greater_equal(0x3, "ae");
 3380     less_equal(0x6, "be");
 3381     greater(0x7, "a");
 3382     overflow(0x0, "o");
 3383     no_overflow(0x1, "no");
 3384   %}
 3385 %}
 3386 
 3387 
 3388 // Floating comparisons that can be fixed up with extra conditional jumps
 3389 operand cmpOpUCF2() %{
 3390   match(Bool);
 3391   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3392              n->as_Bool()->_test._test == BoolTest::eq) &&
 3393             n->in(1)->in(1) != n->in(1)->in(2));
 3394   format %{ "" %}
 3395   interface(COND_INTER) %{
 3396     equal(0x4, "e");
 3397     not_equal(0x5, "ne");
 3398     less(0x2, "b");
 3399     greater_equal(0x3, "ae");
 3400     less_equal(0x6, "be");
 3401     greater(0x7, "a");
 3402     overflow(0x0, "o");
 3403     no_overflow(0x1, "no");
 3404   %}
 3405 %}
 3406 
 3407 //----------OPERAND CLASSES----------------------------------------------------
 3408 // Operand Classes are groups of operands that are used as to simplify
 3409 // instruction definitions by not requiring the AD writer to specify separate
 3410 // instructions for every form of operand when the instruction accepts
 3411 // multiple operand types with the same basic encoding and format.  The classic
 3412 // case of this is memory operands.
 3413 
 3414 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3415                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3416                indCompressedOopOffset,
 3417                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3418                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3419                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3420 
 3421 //----------PIPELINE-----------------------------------------------------------
 3422 // Rules which define the behavior of the target architectures pipeline.
 3423 pipeline %{
 3424 
 3425 //----------ATTRIBUTES---------------------------------------------------------
 3426 attributes %{
 3427   variable_size_instructions;        // Fixed size instructions
 3428   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3429   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3430   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3431   instruction_fetch_units = 1;       // of 16 bytes
 3432 
 3433   // List of nop instructions
 3434   nops( MachNop );
 3435 %}
 3436 
 3437 //----------RESOURCES----------------------------------------------------------
 3438 // Resources are the functional units available to the machine
 3439 
 3440 // Generic P2/P3 pipeline
 3441 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3442 // 3 instructions decoded per cycle.
 3443 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3444 // 3 ALU op, only ALU0 handles mul instructions.
 3445 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3446            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3447            BR, FPU,
 3448            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3449 
 3450 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3451 // Pipeline Description specifies the stages in the machine's pipeline
 3452 
 3453 // Generic P2/P3 pipeline
 3454 pipe_desc(S0, S1, S2, S3, S4, S5);
 3455 
 3456 //----------PIPELINE CLASSES---------------------------------------------------
 3457 // Pipeline Classes describe the stages in which input and output are
 3458 // referenced by the hardware pipeline.
 3459 
 3460 // Naming convention: ialu or fpu
 3461 // Then: _reg
 3462 // Then: _reg if there is a 2nd register
 3463 // Then: _long if it's a pair of instructions implementing a long
 3464 // Then: _fat if it requires the big decoder
 3465 //   Or: _mem if it requires the big decoder and a memory unit.
 3466 
 3467 // Integer ALU reg operation
 3468 pipe_class ialu_reg(rRegI dst)
 3469 %{
 3470     single_instruction;
 3471     dst    : S4(write);
 3472     dst    : S3(read);
 3473     DECODE : S0;        // any decoder
 3474     ALU    : S3;        // any alu
 3475 %}
 3476 
 3477 // Long ALU reg operation
 3478 pipe_class ialu_reg_long(rRegL dst)
 3479 %{
 3480     instruction_count(2);
 3481     dst    : S4(write);
 3482     dst    : S3(read);
 3483     DECODE : S0(2);     // any 2 decoders
 3484     ALU    : S3(2);     // both alus
 3485 %}
 3486 
 3487 // Integer ALU reg operation using big decoder
 3488 pipe_class ialu_reg_fat(rRegI dst)
 3489 %{
 3490     single_instruction;
 3491     dst    : S4(write);
 3492     dst    : S3(read);
 3493     D0     : S0;        // big decoder only
 3494     ALU    : S3;        // any alu
 3495 %}
 3496 
 3497 // Integer ALU reg-reg operation
 3498 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3499 %{
 3500     single_instruction;
 3501     dst    : S4(write);
 3502     src    : S3(read);
 3503     DECODE : S0;        // any decoder
 3504     ALU    : S3;        // any alu
 3505 %}
 3506 
 3507 // Integer ALU reg-reg operation
 3508 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3509 %{
 3510     single_instruction;
 3511     dst    : S4(write);
 3512     src    : S3(read);
 3513     D0     : S0;        // big decoder only
 3514     ALU    : S3;        // any alu
 3515 %}
 3516 
 3517 // Integer ALU reg-mem operation
 3518 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3519 %{
 3520     single_instruction;
 3521     dst    : S5(write);
 3522     mem    : S3(read);
 3523     D0     : S0;        // big decoder only
 3524     ALU    : S4;        // any alu
 3525     MEM    : S3;        // any mem
 3526 %}
 3527 
 3528 // Integer mem operation (prefetch)
 3529 pipe_class ialu_mem(memory mem)
 3530 %{
 3531     single_instruction;
 3532     mem    : S3(read);
 3533     D0     : S0;        // big decoder only
 3534     MEM    : S3;        // any mem
 3535 %}
 3536 
 3537 // Integer Store to Memory
 3538 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3539 %{
 3540     single_instruction;
 3541     mem    : S3(read);
 3542     src    : S5(read);
 3543     D0     : S0;        // big decoder only
 3544     ALU    : S4;        // any alu
 3545     MEM    : S3;
 3546 %}
 3547 
 3548 // // Long Store to Memory
 3549 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3550 // %{
 3551 //     instruction_count(2);
 3552 //     mem    : S3(read);
 3553 //     src    : S5(read);
 3554 //     D0     : S0(2);          // big decoder only; twice
 3555 //     ALU    : S4(2);     // any 2 alus
 3556 //     MEM    : S3(2);  // Both mems
 3557 // %}
 3558 
 3559 // Integer Store to Memory
 3560 pipe_class ialu_mem_imm(memory mem)
 3561 %{
 3562     single_instruction;
 3563     mem    : S3(read);
 3564     D0     : S0;        // big decoder only
 3565     ALU    : S4;        // any alu
 3566     MEM    : S3;
 3567 %}
 3568 
 3569 // Integer ALU0 reg-reg operation
 3570 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3571 %{
 3572     single_instruction;
 3573     dst    : S4(write);
 3574     src    : S3(read);
 3575     D0     : S0;        // Big decoder only
 3576     ALU0   : S3;        // only alu0
 3577 %}
 3578 
 3579 // Integer ALU0 reg-mem operation
 3580 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3581 %{
 3582     single_instruction;
 3583     dst    : S5(write);
 3584     mem    : S3(read);
 3585     D0     : S0;        // big decoder only
 3586     ALU0   : S4;        // ALU0 only
 3587     MEM    : S3;        // any mem
 3588 %}
 3589 
 3590 // Integer ALU reg-reg operation
 3591 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3592 %{
 3593     single_instruction;
 3594     cr     : S4(write);
 3595     src1   : S3(read);
 3596     src2   : S3(read);
 3597     DECODE : S0;        // any decoder
 3598     ALU    : S3;        // any alu
 3599 %}
 3600 
 3601 // Integer ALU reg-imm operation
 3602 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3603 %{
 3604     single_instruction;
 3605     cr     : S4(write);
 3606     src1   : S3(read);
 3607     DECODE : S0;        // any decoder
 3608     ALU    : S3;        // any alu
 3609 %}
 3610 
 3611 // Integer ALU reg-mem operation
 3612 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3613 %{
 3614     single_instruction;
 3615     cr     : S4(write);
 3616     src1   : S3(read);
 3617     src2   : S3(read);
 3618     D0     : S0;        // big decoder only
 3619     ALU    : S4;        // any alu
 3620     MEM    : S3;
 3621 %}
 3622 
 3623 // Conditional move reg-reg
 3624 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3625 %{
 3626     instruction_count(4);
 3627     y      : S4(read);
 3628     q      : S3(read);
 3629     p      : S3(read);
 3630     DECODE : S0(4);     // any decoder
 3631 %}
 3632 
 3633 // Conditional move reg-reg
 3634 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3635 %{
 3636     single_instruction;
 3637     dst    : S4(write);
 3638     src    : S3(read);
 3639     cr     : S3(read);
 3640     DECODE : S0;        // any decoder
 3641 %}
 3642 
 3643 // Conditional move reg-mem
 3644 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3645 %{
 3646     single_instruction;
 3647     dst    : S4(write);
 3648     src    : S3(read);
 3649     cr     : S3(read);
 3650     DECODE : S0;        // any decoder
 3651     MEM    : S3;
 3652 %}
 3653 
 3654 // Conditional move reg-reg long
 3655 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3656 %{
 3657     single_instruction;
 3658     dst    : S4(write);
 3659     src    : S3(read);
 3660     cr     : S3(read);
 3661     DECODE : S0(2);     // any 2 decoders
 3662 %}
 3663 
 3664 // Float reg-reg operation
 3665 pipe_class fpu_reg(regD dst)
 3666 %{
 3667     instruction_count(2);
 3668     dst    : S3(read);
 3669     DECODE : S0(2);     // any 2 decoders
 3670     FPU    : S3;
 3671 %}
 3672 
 3673 // Float reg-reg operation
 3674 pipe_class fpu_reg_reg(regD dst, regD src)
 3675 %{
 3676     instruction_count(2);
 3677     dst    : S4(write);
 3678     src    : S3(read);
 3679     DECODE : S0(2);     // any 2 decoders
 3680     FPU    : S3;
 3681 %}
 3682 
 3683 // Float reg-reg operation
 3684 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3685 %{
 3686     instruction_count(3);
 3687     dst    : S4(write);
 3688     src1   : S3(read);
 3689     src2   : S3(read);
 3690     DECODE : S0(3);     // any 3 decoders
 3691     FPU    : S3(2);
 3692 %}
 3693 
 3694 // Float reg-reg operation
 3695 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3696 %{
 3697     instruction_count(4);
 3698     dst    : S4(write);
 3699     src1   : S3(read);
 3700     src2   : S3(read);
 3701     src3   : S3(read);
 3702     DECODE : S0(4);     // any 3 decoders
 3703     FPU    : S3(2);
 3704 %}
 3705 
 3706 // Float reg-reg operation
 3707 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3708 %{
 3709     instruction_count(4);
 3710     dst    : S4(write);
 3711     src1   : S3(read);
 3712     src2   : S3(read);
 3713     src3   : S3(read);
 3714     DECODE : S1(3);     // any 3 decoders
 3715     D0     : S0;        // Big decoder only
 3716     FPU    : S3(2);
 3717     MEM    : S3;
 3718 %}
 3719 
 3720 // Float reg-mem operation
 3721 pipe_class fpu_reg_mem(regD dst, memory mem)
 3722 %{
 3723     instruction_count(2);
 3724     dst    : S5(write);
 3725     mem    : S3(read);
 3726     D0     : S0;        // big decoder only
 3727     DECODE : S1;        // any decoder for FPU POP
 3728     FPU    : S4;
 3729     MEM    : S3;        // any mem
 3730 %}
 3731 
 3732 // Float reg-mem operation
 3733 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3734 %{
 3735     instruction_count(3);
 3736     dst    : S5(write);
 3737     src1   : S3(read);
 3738     mem    : S3(read);
 3739     D0     : S0;        // big decoder only
 3740     DECODE : S1(2);     // any decoder for FPU POP
 3741     FPU    : S4;
 3742     MEM    : S3;        // any mem
 3743 %}
 3744 
 3745 // Float mem-reg operation
 3746 pipe_class fpu_mem_reg(memory mem, regD src)
 3747 %{
 3748     instruction_count(2);
 3749     src    : S5(read);
 3750     mem    : S3(read);
 3751     DECODE : S0;        // any decoder for FPU PUSH
 3752     D0     : S1;        // big decoder only
 3753     FPU    : S4;
 3754     MEM    : S3;        // any mem
 3755 %}
 3756 
 3757 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3758 %{
 3759     instruction_count(3);
 3760     src1   : S3(read);
 3761     src2   : S3(read);
 3762     mem    : S3(read);
 3763     DECODE : S0(2);     // any decoder for FPU PUSH
 3764     D0     : S1;        // big decoder only
 3765     FPU    : S4;
 3766     MEM    : S3;        // any mem
 3767 %}
 3768 
 3769 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3770 %{
 3771     instruction_count(3);
 3772     src1   : S3(read);
 3773     src2   : S3(read);
 3774     mem    : S4(read);
 3775     DECODE : S0;        // any decoder for FPU PUSH
 3776     D0     : S0(2);     // big decoder only
 3777     FPU    : S4;
 3778     MEM    : S3(2);     // any mem
 3779 %}
 3780 
 3781 pipe_class fpu_mem_mem(memory dst, memory src1)
 3782 %{
 3783     instruction_count(2);
 3784     src1   : S3(read);
 3785     dst    : S4(read);
 3786     D0     : S0(2);     // big decoder only
 3787     MEM    : S3(2);     // any mem
 3788 %}
 3789 
 3790 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3791 %{
 3792     instruction_count(3);
 3793     src1   : S3(read);
 3794     src2   : S3(read);
 3795     dst    : S4(read);
 3796     D0     : S0(3);     // big decoder only
 3797     FPU    : S4;
 3798     MEM    : S3(3);     // any mem
 3799 %}
 3800 
 3801 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3802 %{
 3803     instruction_count(3);
 3804     src1   : S4(read);
 3805     mem    : S4(read);
 3806     DECODE : S0;        // any decoder for FPU PUSH
 3807     D0     : S0(2);     // big decoder only
 3808     FPU    : S4;
 3809     MEM    : S3(2);     // any mem
 3810 %}
 3811 
 3812 // Float load constant
 3813 pipe_class fpu_reg_con(regD dst)
 3814 %{
 3815     instruction_count(2);
 3816     dst    : S5(write);
 3817     D0     : S0;        // big decoder only for the load
 3818     DECODE : S1;        // any decoder for FPU POP
 3819     FPU    : S4;
 3820     MEM    : S3;        // any mem
 3821 %}
 3822 
 3823 // Float load constant
 3824 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3825 %{
 3826     instruction_count(3);
 3827     dst    : S5(write);
 3828     src    : S3(read);
 3829     D0     : S0;        // big decoder only for the load
 3830     DECODE : S1(2);     // any decoder for FPU POP
 3831     FPU    : S4;
 3832     MEM    : S3;        // any mem
 3833 %}
 3834 
 3835 // UnConditional branch
 3836 pipe_class pipe_jmp(label labl)
 3837 %{
 3838     single_instruction;
 3839     BR   : S3;
 3840 %}
 3841 
 3842 // Conditional branch
 3843 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3844 %{
 3845     single_instruction;
 3846     cr    : S1(read);
 3847     BR    : S3;
 3848 %}
 3849 
 3850 // Allocation idiom
 3851 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3852 %{
 3853     instruction_count(1); force_serialization;
 3854     fixed_latency(6);
 3855     heap_ptr : S3(read);
 3856     DECODE   : S0(3);
 3857     D0       : S2;
 3858     MEM      : S3;
 3859     ALU      : S3(2);
 3860     dst      : S5(write);
 3861     BR       : S5;
 3862 %}
 3863 
 3864 // Generic big/slow expanded idiom
 3865 pipe_class pipe_slow()
 3866 %{
 3867     instruction_count(10); multiple_bundles; force_serialization;
 3868     fixed_latency(100);
 3869     D0  : S0(2);
 3870     MEM : S3(2);
 3871 %}
 3872 
 3873 // The real do-nothing guy
 3874 pipe_class empty()
 3875 %{
 3876     instruction_count(0);
 3877 %}
 3878 
 3879 // Define the class for the Nop node
 3880 define
 3881 %{
 3882    MachNop = empty;
 3883 %}
 3884 
 3885 %}
 3886 
 3887 //----------INSTRUCTIONS-------------------------------------------------------
 3888 //
 3889 // match      -- States which machine-independent subtree may be replaced
 3890 //               by this instruction.
 3891 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3892 //               selection to identify a minimum cost tree of machine
 3893 //               instructions that matches a tree of machine-independent
 3894 //               instructions.
 3895 // format     -- A string providing the disassembly for this instruction.
 3896 //               The value of an instruction's operand may be inserted
 3897 //               by referring to it with a '$' prefix.
 3898 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3899 //               to within an encode class as $primary, $secondary, and $tertiary
 3900 //               rrspectively.  The primary opcode is commonly used to
 3901 //               indicate the type of machine instruction, while secondary
 3902 //               and tertiary are often used for prefix options or addressing
 3903 //               modes.
 3904 // ins_encode -- A list of encode classes with parameters. The encode class
 3905 //               name must have been defined in an 'enc_class' specification
 3906 //               in the encode section of the architecture description.
 3907 
 3908 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3909 // Load Float
 3910 instruct MoveF2VL(vlRegF dst, regF src) %{
 3911   match(Set dst src);
 3912   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3913   ins_encode %{
 3914     ShouldNotReachHere();
 3915   %}
 3916   ins_pipe( fpu_reg_reg );
 3917 %}
 3918 
 3919 // Load Float
 3920 instruct MoveF2LEG(legRegF dst, regF src) %{
 3921   match(Set dst src);
 3922   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3923   ins_encode %{
 3924     ShouldNotReachHere();
 3925   %}
 3926   ins_pipe( fpu_reg_reg );
 3927 %}
 3928 
 3929 // Load Float
 3930 instruct MoveVL2F(regF dst, vlRegF src) %{
 3931   match(Set dst src);
 3932   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3933   ins_encode %{
 3934     ShouldNotReachHere();
 3935   %}
 3936   ins_pipe( fpu_reg_reg );
 3937 %}
 3938 
 3939 // Load Float
 3940 instruct MoveLEG2F(regF dst, legRegF src) %{
 3941   match(Set dst src);
 3942   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3943   ins_encode %{
 3944     ShouldNotReachHere();
 3945   %}
 3946   ins_pipe( fpu_reg_reg );
 3947 %}
 3948 
 3949 // Load Double
 3950 instruct MoveD2VL(vlRegD dst, regD src) %{
 3951   match(Set dst src);
 3952   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3953   ins_encode %{
 3954     ShouldNotReachHere();
 3955   %}
 3956   ins_pipe( fpu_reg_reg );
 3957 %}
 3958 
 3959 // Load Double
 3960 instruct MoveD2LEG(legRegD dst, regD src) %{
 3961   match(Set dst src);
 3962   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3963   ins_encode %{
 3964     ShouldNotReachHere();
 3965   %}
 3966   ins_pipe( fpu_reg_reg );
 3967 %}
 3968 
 3969 // Load Double
 3970 instruct MoveVL2D(regD dst, vlRegD src) %{
 3971   match(Set dst src);
 3972   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3973   ins_encode %{
 3974     ShouldNotReachHere();
 3975   %}
 3976   ins_pipe( fpu_reg_reg );
 3977 %}
 3978 
 3979 // Load Double
 3980 instruct MoveLEG2D(regD dst, legRegD src) %{
 3981   match(Set dst src);
 3982   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3983   ins_encode %{
 3984     ShouldNotReachHere();
 3985   %}
 3986   ins_pipe( fpu_reg_reg );
 3987 %}
 3988 
 3989 //----------Load/Store/Move Instructions---------------------------------------
 3990 //----------Load Instructions--------------------------------------------------
 3991 
 3992 // Load Byte (8 bit signed)
 3993 instruct loadB(rRegI dst, memory mem)
 3994 %{
 3995   match(Set dst (LoadB mem));
 3996 
 3997   ins_cost(125);
 3998   format %{ "movsbl  $dst, $mem\t# byte" %}
 3999 
 4000   ins_encode %{
 4001     __ movsbl($dst$$Register, $mem$$Address);
 4002   %}
 4003 
 4004   ins_pipe(ialu_reg_mem);
 4005 %}
 4006 
 4007 // Load Byte (8 bit signed) into Long Register
 4008 instruct loadB2L(rRegL dst, memory mem)
 4009 %{
 4010   match(Set dst (ConvI2L (LoadB mem)));
 4011 
 4012   ins_cost(125);
 4013   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4014 
 4015   ins_encode %{
 4016     __ movsbq($dst$$Register, $mem$$Address);
 4017   %}
 4018 
 4019   ins_pipe(ialu_reg_mem);
 4020 %}
 4021 
 4022 // Load Unsigned Byte (8 bit UNsigned)
 4023 instruct loadUB(rRegI dst, memory mem)
 4024 %{
 4025   match(Set dst (LoadUB mem));
 4026 
 4027   ins_cost(125);
 4028   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4029 
 4030   ins_encode %{
 4031     __ movzbl($dst$$Register, $mem$$Address);
 4032   %}
 4033 
 4034   ins_pipe(ialu_reg_mem);
 4035 %}
 4036 
 4037 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4038 instruct loadUB2L(rRegL dst, memory mem)
 4039 %{
 4040   match(Set dst (ConvI2L (LoadUB mem)));
 4041 
 4042   ins_cost(125);
 4043   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4044 
 4045   ins_encode %{
 4046     __ movzbq($dst$$Register, $mem$$Address);
 4047   %}
 4048 
 4049   ins_pipe(ialu_reg_mem);
 4050 %}
 4051 
 4052 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4053 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4054   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4055   effect(KILL cr);
 4056 
 4057   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4058             "andl    $dst, right_n_bits($mask, 8)" %}
 4059   ins_encode %{
 4060     Register Rdst = $dst$$Register;
 4061     __ movzbq(Rdst, $mem$$Address);
 4062     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4063   %}
 4064   ins_pipe(ialu_reg_mem);
 4065 %}
 4066 
 4067 // Load Short (16 bit signed)
 4068 instruct loadS(rRegI dst, memory mem)
 4069 %{
 4070   match(Set dst (LoadS mem));
 4071 
 4072   ins_cost(125);
 4073   format %{ "movswl $dst, $mem\t# short" %}
 4074 
 4075   ins_encode %{
 4076     __ movswl($dst$$Register, $mem$$Address);
 4077   %}
 4078 
 4079   ins_pipe(ialu_reg_mem);
 4080 %}
 4081 
 4082 // Load Short (16 bit signed) to Byte (8 bit signed)
 4083 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4084   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4085 
 4086   ins_cost(125);
 4087   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4088   ins_encode %{
 4089     __ movsbl($dst$$Register, $mem$$Address);
 4090   %}
 4091   ins_pipe(ialu_reg_mem);
 4092 %}
 4093 
 4094 // Load Short (16 bit signed) into Long Register
 4095 instruct loadS2L(rRegL dst, memory mem)
 4096 %{
 4097   match(Set dst (ConvI2L (LoadS mem)));
 4098 
 4099   ins_cost(125);
 4100   format %{ "movswq $dst, $mem\t# short -> long" %}
 4101 
 4102   ins_encode %{
 4103     __ movswq($dst$$Register, $mem$$Address);
 4104   %}
 4105 
 4106   ins_pipe(ialu_reg_mem);
 4107 %}
 4108 
 4109 // Load Unsigned Short/Char (16 bit UNsigned)
 4110 instruct loadUS(rRegI dst, memory mem)
 4111 %{
 4112   match(Set dst (LoadUS mem));
 4113 
 4114   ins_cost(125);
 4115   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4116 
 4117   ins_encode %{
 4118     __ movzwl($dst$$Register, $mem$$Address);
 4119   %}
 4120 
 4121   ins_pipe(ialu_reg_mem);
 4122 %}
 4123 
 4124 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4125 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4126   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4127 
 4128   ins_cost(125);
 4129   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4130   ins_encode %{
 4131     __ movsbl($dst$$Register, $mem$$Address);
 4132   %}
 4133   ins_pipe(ialu_reg_mem);
 4134 %}
 4135 
 4136 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4137 instruct loadUS2L(rRegL dst, memory mem)
 4138 %{
 4139   match(Set dst (ConvI2L (LoadUS mem)));
 4140 
 4141   ins_cost(125);
 4142   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4143 
 4144   ins_encode %{
 4145     __ movzwq($dst$$Register, $mem$$Address);
 4146   %}
 4147 
 4148   ins_pipe(ialu_reg_mem);
 4149 %}
 4150 
 4151 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4152 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4153   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4154 
 4155   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4156   ins_encode %{
 4157     __ movzbq($dst$$Register, $mem$$Address);
 4158   %}
 4159   ins_pipe(ialu_reg_mem);
 4160 %}
 4161 
 4162 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4163 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4164   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4165   effect(KILL cr);
 4166 
 4167   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4168             "andl    $dst, right_n_bits($mask, 16)" %}
 4169   ins_encode %{
 4170     Register Rdst = $dst$$Register;
 4171     __ movzwq(Rdst, $mem$$Address);
 4172     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4173   %}
 4174   ins_pipe(ialu_reg_mem);
 4175 %}
 4176 
 4177 // Load Integer
 4178 instruct loadI(rRegI dst, memory mem)
 4179 %{
 4180   match(Set dst (LoadI mem));
 4181 
 4182   ins_cost(125);
 4183   format %{ "movl    $dst, $mem\t# int" %}
 4184 
 4185   ins_encode %{
 4186     __ movl($dst$$Register, $mem$$Address);
 4187   %}
 4188 
 4189   ins_pipe(ialu_reg_mem);
 4190 %}
 4191 
 4192 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4193 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4194   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4195 
 4196   ins_cost(125);
 4197   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4198   ins_encode %{
 4199     __ movsbl($dst$$Register, $mem$$Address);
 4200   %}
 4201   ins_pipe(ialu_reg_mem);
 4202 %}
 4203 
 4204 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4205 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4206   match(Set dst (AndI (LoadI mem) mask));
 4207 
 4208   ins_cost(125);
 4209   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4210   ins_encode %{
 4211     __ movzbl($dst$$Register, $mem$$Address);
 4212   %}
 4213   ins_pipe(ialu_reg_mem);
 4214 %}
 4215 
 4216 // Load Integer (32 bit signed) to Short (16 bit signed)
 4217 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4218   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4219 
 4220   ins_cost(125);
 4221   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4222   ins_encode %{
 4223     __ movswl($dst$$Register, $mem$$Address);
 4224   %}
 4225   ins_pipe(ialu_reg_mem);
 4226 %}
 4227 
 4228 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4229 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4230   match(Set dst (AndI (LoadI mem) mask));
 4231 
 4232   ins_cost(125);
 4233   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4234   ins_encode %{
 4235     __ movzwl($dst$$Register, $mem$$Address);
 4236   %}
 4237   ins_pipe(ialu_reg_mem);
 4238 %}
 4239 
 4240 // Load Integer into Long Register
 4241 instruct loadI2L(rRegL dst, memory mem)
 4242 %{
 4243   match(Set dst (ConvI2L (LoadI mem)));
 4244 
 4245   ins_cost(125);
 4246   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4247 
 4248   ins_encode %{
 4249     __ movslq($dst$$Register, $mem$$Address);
 4250   %}
 4251 
 4252   ins_pipe(ialu_reg_mem);
 4253 %}
 4254 
 4255 // Load Integer with mask 0xFF into Long Register
 4256 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4257   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4258 
 4259   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4260   ins_encode %{
 4261     __ movzbq($dst$$Register, $mem$$Address);
 4262   %}
 4263   ins_pipe(ialu_reg_mem);
 4264 %}
 4265 
 4266 // Load Integer with mask 0xFFFF into Long Register
 4267 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4268   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4269 
 4270   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4271   ins_encode %{
 4272     __ movzwq($dst$$Register, $mem$$Address);
 4273   %}
 4274   ins_pipe(ialu_reg_mem);
 4275 %}
 4276 
 4277 // Load Integer with a 31-bit mask into Long Register
 4278 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4279   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4280   effect(KILL cr);
 4281 
 4282   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4283             "andl    $dst, $mask" %}
 4284   ins_encode %{
 4285     Register Rdst = $dst$$Register;
 4286     __ movl(Rdst, $mem$$Address);
 4287     __ andl(Rdst, $mask$$constant);
 4288   %}
 4289   ins_pipe(ialu_reg_mem);
 4290 %}
 4291 
 4292 // Load Unsigned Integer into Long Register
 4293 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4294 %{
 4295   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4296 
 4297   ins_cost(125);
 4298   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4299 
 4300   ins_encode %{
 4301     __ movl($dst$$Register, $mem$$Address);
 4302   %}
 4303 
 4304   ins_pipe(ialu_reg_mem);
 4305 %}
 4306 
 4307 // Load Long
 4308 instruct loadL(rRegL dst, memory mem)
 4309 %{
 4310   match(Set dst (LoadL mem));
 4311 
 4312   ins_cost(125);
 4313   format %{ "movq    $dst, $mem\t# long" %}
 4314 
 4315   ins_encode %{
 4316     __ movq($dst$$Register, $mem$$Address);
 4317   %}
 4318 
 4319   ins_pipe(ialu_reg_mem); // XXX
 4320 %}
 4321 
 4322 // Load Range
 4323 instruct loadRange(rRegI dst, memory mem)
 4324 %{
 4325   match(Set dst (LoadRange mem));
 4326 
 4327   ins_cost(125); // XXX
 4328   format %{ "movl    $dst, $mem\t# range" %}
 4329   ins_encode %{
 4330     __ movl($dst$$Register, $mem$$Address);
 4331   %}
 4332   ins_pipe(ialu_reg_mem);
 4333 %}
 4334 
 4335 // Load Pointer
 4336 instruct loadP(rRegP dst, memory mem)
 4337 %{
 4338   match(Set dst (LoadP mem));
 4339   predicate(n->as_Load()->barrier_data() == 0);
 4340 
 4341   ins_cost(125); // XXX
 4342   format %{ "movq    $dst, $mem\t# ptr" %}
 4343   ins_encode %{
 4344     __ movq($dst$$Register, $mem$$Address);
 4345   %}
 4346   ins_pipe(ialu_reg_mem); // XXX
 4347 %}
 4348 
 4349 // Load Compressed Pointer
 4350 instruct loadN(rRegN dst, memory mem)
 4351 %{
 4352    predicate(n->as_Load()->barrier_data() == 0);
 4353    match(Set dst (LoadN mem));
 4354 
 4355    ins_cost(125); // XXX
 4356    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4357    ins_encode %{
 4358      __ movl($dst$$Register, $mem$$Address);
 4359    %}
 4360    ins_pipe(ialu_reg_mem); // XXX
 4361 %}
 4362 
 4363 
 4364 // Load Klass Pointer
 4365 instruct loadKlass(rRegP dst, memory mem)
 4366 %{
 4367   match(Set dst (LoadKlass mem));
 4368 
 4369   ins_cost(125); // XXX
 4370   format %{ "movq    $dst, $mem\t# class" %}
 4371   ins_encode %{
 4372     __ movq($dst$$Register, $mem$$Address);
 4373   %}
 4374   ins_pipe(ialu_reg_mem); // XXX
 4375 %}
 4376 
 4377 // Load narrow Klass Pointer
 4378 instruct loadNKlass(rRegN dst, memory mem)
 4379 %{
 4380   predicate(!UseCompactObjectHeaders);
 4381   match(Set dst (LoadNKlass mem));
 4382 
 4383   ins_cost(125); // XXX
 4384   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4385   ins_encode %{
 4386     __ movl($dst$$Register, $mem$$Address);
 4387   %}
 4388   ins_pipe(ialu_reg_mem); // XXX
 4389 %}
 4390 
 4391 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4392 %{
 4393   predicate(UseCompactObjectHeaders);
 4394   match(Set dst (LoadNKlass mem));
 4395   effect(KILL cr);
 4396   ins_cost(125);
 4397   format %{
 4398     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4399     "shrl    $dst, markWord::klass_shift"
 4400   %}
 4401   ins_encode %{
 4402     // The incoming address is pointing into obj-start + Type::klass_offset(). We need to extract
 4403     // obj-start, so that we can load from the object's mark-word instead.
 4404     Register d = $dst$$Register;
 4405     Address  s = ($mem$$Address).plus_disp(-Type::klass_offset());
 4406     if (UseAPX) {
 4407       __ eshrl(d, s, markWord::klass_shift, false);
 4408     } else {
 4409       __ movl(d, s);
 4410       __ shrl(d, markWord::klass_shift);
 4411     }
 4412   %}
 4413   ins_pipe(ialu_reg_mem);
 4414 %}
 4415 
 4416 // Load Float
 4417 instruct loadF(regF dst, memory mem)
 4418 %{
 4419   match(Set dst (LoadF mem));
 4420 
 4421   ins_cost(145); // XXX
 4422   format %{ "movss   $dst, $mem\t# float" %}
 4423   ins_encode %{
 4424     __ movflt($dst$$XMMRegister, $mem$$Address);
 4425   %}
 4426   ins_pipe(pipe_slow); // XXX
 4427 %}
 4428 
 4429 // Load Double
 4430 instruct loadD_partial(regD dst, memory mem)
 4431 %{
 4432   predicate(!UseXmmLoadAndClearUpper);
 4433   match(Set dst (LoadD mem));
 4434 
 4435   ins_cost(145); // XXX
 4436   format %{ "movlpd  $dst, $mem\t# double" %}
 4437   ins_encode %{
 4438     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4439   %}
 4440   ins_pipe(pipe_slow); // XXX
 4441 %}
 4442 
 4443 instruct loadD(regD dst, memory mem)
 4444 %{
 4445   predicate(UseXmmLoadAndClearUpper);
 4446   match(Set dst (LoadD mem));
 4447 
 4448   ins_cost(145); // XXX
 4449   format %{ "movsd   $dst, $mem\t# double" %}
 4450   ins_encode %{
 4451     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4452   %}
 4453   ins_pipe(pipe_slow); // XXX
 4454 %}
 4455 
 4456 // max = java.lang.Math.max(float a, float b)
 4457 instruct maxF_avx10_reg(regF dst, regF a, regF b) %{
 4458   predicate(VM_Version::supports_avx10_2());
 4459   match(Set dst (MaxF a b));
 4460   format %{ "maxF $dst, $a, $b" %}
 4461   ins_encode %{
 4462     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4463   %}
 4464   ins_pipe( pipe_slow );
 4465 %}
 4466 
 4467 // max = java.lang.Math.max(float a, float b)
 4468 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4469   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4470   match(Set dst (MaxF a b));
 4471   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4472   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4473   ins_encode %{
 4474     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4475   %}
 4476   ins_pipe( pipe_slow );
 4477 %}
 4478 
 4479 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4480   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4481   match(Set dst (MaxF a b));
 4482   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4483 
 4484   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4485   ins_encode %{
 4486     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4487                     false /*min*/, true /*single*/);
 4488   %}
 4489   ins_pipe( pipe_slow );
 4490 %}
 4491 
 4492 // max = java.lang.Math.max(double a, double b)
 4493 instruct maxD_avx10_reg(regD dst, regD a, regD b) %{
 4494   predicate(VM_Version::supports_avx10_2());
 4495   match(Set dst (MaxD a b));
 4496   format %{ "maxD $dst, $a, $b" %}
 4497   ins_encode %{
 4498     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4499   %}
 4500   ins_pipe( pipe_slow );
 4501 %}
 4502 
 4503 // max = java.lang.Math.max(double a, double b)
 4504 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4505   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4506   match(Set dst (MaxD a b));
 4507   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4508   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4509   ins_encode %{
 4510     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4511   %}
 4512   ins_pipe( pipe_slow );
 4513 %}
 4514 
 4515 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4516   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4517   match(Set dst (MaxD a b));
 4518   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4519 
 4520   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4521   ins_encode %{
 4522     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4523                     false /*min*/, false /*single*/);
 4524   %}
 4525   ins_pipe( pipe_slow );
 4526 %}
 4527 
 4528 // max = java.lang.Math.min(float a, float b)
 4529 instruct minF_avx10_reg(regF dst, regF a, regF b) %{
 4530   predicate(VM_Version::supports_avx10_2());
 4531   match(Set dst (MinF a b));
 4532   format %{ "minF $dst, $a, $b" %}
 4533   ins_encode %{
 4534     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4535   %}
 4536   ins_pipe( pipe_slow );
 4537 %}
 4538 
 4539 // min = java.lang.Math.min(float a, float b)
 4540 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4541   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4542   match(Set dst (MinF a b));
 4543   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4544   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4545   ins_encode %{
 4546     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4547   %}
 4548   ins_pipe( pipe_slow );
 4549 %}
 4550 
 4551 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4552   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4553   match(Set dst (MinF a b));
 4554   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4555 
 4556   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4557   ins_encode %{
 4558     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4559                     true /*min*/, true /*single*/);
 4560   %}
 4561   ins_pipe( pipe_slow );
 4562 %}
 4563 
 4564 // max = java.lang.Math.min(double a, double b)
 4565 instruct minD_avx10_reg(regD dst, regD a, regD b) %{
 4566   predicate(VM_Version::supports_avx10_2());
 4567   match(Set dst (MinD a b));
 4568   format %{ "minD $dst, $a, $b" %}
 4569   ins_encode %{
 4570     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4571   %}
 4572   ins_pipe( pipe_slow );
 4573 %}
 4574 
 4575 // min = java.lang.Math.min(double a, double b)
 4576 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4577   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4578   match(Set dst (MinD a b));
 4579   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4580     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4581   ins_encode %{
 4582     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4583   %}
 4584   ins_pipe( pipe_slow );
 4585 %}
 4586 
 4587 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4588   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4589   match(Set dst (MinD a b));
 4590   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4591 
 4592   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4593   ins_encode %{
 4594     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4595                     true /*min*/, false /*single*/);
 4596   %}
 4597   ins_pipe( pipe_slow );
 4598 %}
 4599 
 4600 // Load Effective Address
 4601 instruct leaP8(rRegP dst, indOffset8 mem)
 4602 %{
 4603   match(Set dst mem);
 4604 
 4605   ins_cost(110); // XXX
 4606   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4607   ins_encode %{
 4608     __ leaq($dst$$Register, $mem$$Address);
 4609   %}
 4610   ins_pipe(ialu_reg_reg_fat);
 4611 %}
 4612 
 4613 instruct leaP32(rRegP dst, indOffset32 mem)
 4614 %{
 4615   match(Set dst mem);
 4616 
 4617   ins_cost(110);
 4618   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4619   ins_encode %{
 4620     __ leaq($dst$$Register, $mem$$Address);
 4621   %}
 4622   ins_pipe(ialu_reg_reg_fat);
 4623 %}
 4624 
 4625 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4626 %{
 4627   match(Set dst mem);
 4628 
 4629   ins_cost(110);
 4630   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4631   ins_encode %{
 4632     __ leaq($dst$$Register, $mem$$Address);
 4633   %}
 4634   ins_pipe(ialu_reg_reg_fat);
 4635 %}
 4636 
 4637 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4638 %{
 4639   match(Set dst mem);
 4640 
 4641   ins_cost(110);
 4642   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4643   ins_encode %{
 4644     __ leaq($dst$$Register, $mem$$Address);
 4645   %}
 4646   ins_pipe(ialu_reg_reg_fat);
 4647 %}
 4648 
 4649 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4650 %{
 4651   match(Set dst mem);
 4652 
 4653   ins_cost(110);
 4654   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4655   ins_encode %{
 4656     __ leaq($dst$$Register, $mem$$Address);
 4657   %}
 4658   ins_pipe(ialu_reg_reg_fat);
 4659 %}
 4660 
 4661 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4662 %{
 4663   match(Set dst mem);
 4664 
 4665   ins_cost(110);
 4666   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4667   ins_encode %{
 4668     __ leaq($dst$$Register, $mem$$Address);
 4669   %}
 4670   ins_pipe(ialu_reg_reg_fat);
 4671 %}
 4672 
 4673 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4674 %{
 4675   match(Set dst mem);
 4676 
 4677   ins_cost(110);
 4678   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4679   ins_encode %{
 4680     __ leaq($dst$$Register, $mem$$Address);
 4681   %}
 4682   ins_pipe(ialu_reg_reg_fat);
 4683 %}
 4684 
 4685 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4686 %{
 4687   match(Set dst mem);
 4688 
 4689   ins_cost(110);
 4690   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4691   ins_encode %{
 4692     __ leaq($dst$$Register, $mem$$Address);
 4693   %}
 4694   ins_pipe(ialu_reg_reg_fat);
 4695 %}
 4696 
 4697 // Load Effective Address which uses Narrow (32-bits) oop
 4698 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4699 %{
 4700   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4701   match(Set dst mem);
 4702 
 4703   ins_cost(110);
 4704   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4705   ins_encode %{
 4706     __ leaq($dst$$Register, $mem$$Address);
 4707   %}
 4708   ins_pipe(ialu_reg_reg_fat);
 4709 %}
 4710 
 4711 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4712 %{
 4713   predicate(CompressedOops::shift() == 0);
 4714   match(Set dst mem);
 4715 
 4716   ins_cost(110); // XXX
 4717   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4718   ins_encode %{
 4719     __ leaq($dst$$Register, $mem$$Address);
 4720   %}
 4721   ins_pipe(ialu_reg_reg_fat);
 4722 %}
 4723 
 4724 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4725 %{
 4726   predicate(CompressedOops::shift() == 0);
 4727   match(Set dst mem);
 4728 
 4729   ins_cost(110);
 4730   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4731   ins_encode %{
 4732     __ leaq($dst$$Register, $mem$$Address);
 4733   %}
 4734   ins_pipe(ialu_reg_reg_fat);
 4735 %}
 4736 
 4737 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4738 %{
 4739   predicate(CompressedOops::shift() == 0);
 4740   match(Set dst mem);
 4741 
 4742   ins_cost(110);
 4743   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4744   ins_encode %{
 4745     __ leaq($dst$$Register, $mem$$Address);
 4746   %}
 4747   ins_pipe(ialu_reg_reg_fat);
 4748 %}
 4749 
 4750 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4751 %{
 4752   predicate(CompressedOops::shift() == 0);
 4753   match(Set dst mem);
 4754 
 4755   ins_cost(110);
 4756   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4757   ins_encode %{
 4758     __ leaq($dst$$Register, $mem$$Address);
 4759   %}
 4760   ins_pipe(ialu_reg_reg_fat);
 4761 %}
 4762 
 4763 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4764 %{
 4765   predicate(CompressedOops::shift() == 0);
 4766   match(Set dst mem);
 4767 
 4768   ins_cost(110);
 4769   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4770   ins_encode %{
 4771     __ leaq($dst$$Register, $mem$$Address);
 4772   %}
 4773   ins_pipe(ialu_reg_reg_fat);
 4774 %}
 4775 
 4776 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4777 %{
 4778   predicate(CompressedOops::shift() == 0);
 4779   match(Set dst mem);
 4780 
 4781   ins_cost(110);
 4782   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4783   ins_encode %{
 4784     __ leaq($dst$$Register, $mem$$Address);
 4785   %}
 4786   ins_pipe(ialu_reg_reg_fat);
 4787 %}
 4788 
 4789 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4790 %{
 4791   predicate(CompressedOops::shift() == 0);
 4792   match(Set dst mem);
 4793 
 4794   ins_cost(110);
 4795   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4796   ins_encode %{
 4797     __ leaq($dst$$Register, $mem$$Address);
 4798   %}
 4799   ins_pipe(ialu_reg_reg_fat);
 4800 %}
 4801 
 4802 instruct loadConI(rRegI dst, immI src)
 4803 %{
 4804   match(Set dst src);
 4805 
 4806   format %{ "movl    $dst, $src\t# int" %}
 4807   ins_encode %{
 4808     __ movl($dst$$Register, $src$$constant);
 4809   %}
 4810   ins_pipe(ialu_reg_fat); // XXX
 4811 %}
 4812 
 4813 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4814 %{
 4815   match(Set dst src);
 4816   effect(KILL cr);
 4817 
 4818   ins_cost(50);
 4819   format %{ "xorl    $dst, $dst\t# int" %}
 4820   ins_encode %{
 4821     __ xorl($dst$$Register, $dst$$Register);
 4822   %}
 4823   ins_pipe(ialu_reg);
 4824 %}
 4825 
 4826 instruct loadConL(rRegL dst, immL src)
 4827 %{
 4828   match(Set dst src);
 4829 
 4830   ins_cost(150);
 4831   format %{ "movq    $dst, $src\t# long" %}
 4832   ins_encode %{
 4833     __ mov64($dst$$Register, $src$$constant);
 4834   %}
 4835   ins_pipe(ialu_reg);
 4836 %}
 4837 
 4838 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4839 %{
 4840   match(Set dst src);
 4841   effect(KILL cr);
 4842 
 4843   ins_cost(50);
 4844   format %{ "xorl    $dst, $dst\t# long" %}
 4845   ins_encode %{
 4846     __ xorl($dst$$Register, $dst$$Register);
 4847   %}
 4848   ins_pipe(ialu_reg); // XXX
 4849 %}
 4850 
 4851 instruct loadConUL32(rRegL dst, immUL32 src)
 4852 %{
 4853   match(Set dst src);
 4854 
 4855   ins_cost(60);
 4856   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4857   ins_encode %{
 4858     __ movl($dst$$Register, $src$$constant);
 4859   %}
 4860   ins_pipe(ialu_reg);
 4861 %}
 4862 
 4863 instruct loadConL32(rRegL dst, immL32 src)
 4864 %{
 4865   match(Set dst src);
 4866 
 4867   ins_cost(70);
 4868   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4869   ins_encode %{
 4870     __ movq($dst$$Register, $src$$constant);
 4871   %}
 4872   ins_pipe(ialu_reg);
 4873 %}
 4874 
 4875 instruct loadConP(rRegP dst, immP con) %{
 4876   match(Set dst con);
 4877 
 4878   format %{ "movq    $dst, $con\t# ptr" %}
 4879   ins_encode %{
 4880     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4881   %}
 4882   ins_pipe(ialu_reg_fat); // XXX
 4883 %}
 4884 
 4885 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4886 %{
 4887   match(Set dst src);
 4888   effect(KILL cr);
 4889 
 4890   ins_cost(50);
 4891   format %{ "xorl    $dst, $dst\t# ptr" %}
 4892   ins_encode %{
 4893     __ xorl($dst$$Register, $dst$$Register);
 4894   %}
 4895   ins_pipe(ialu_reg);
 4896 %}
 4897 
 4898 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4899 %{
 4900   match(Set dst src);
 4901   effect(KILL cr);
 4902 
 4903   ins_cost(60);
 4904   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4905   ins_encode %{
 4906     __ movl($dst$$Register, $src$$constant);
 4907   %}
 4908   ins_pipe(ialu_reg);
 4909 %}
 4910 
 4911 instruct loadConF(regF dst, immF con) %{
 4912   match(Set dst con);
 4913   ins_cost(125);
 4914   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4915   ins_encode %{
 4916     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4917   %}
 4918   ins_pipe(pipe_slow);
 4919 %}
 4920 
 4921 instruct loadConH(regF dst, immH con) %{
 4922   match(Set dst con);
 4923   ins_cost(125);
 4924   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4925   ins_encode %{
 4926     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4927   %}
 4928   ins_pipe(pipe_slow);
 4929 %}
 4930 
 4931 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4932   match(Set dst src);
 4933   effect(KILL cr);
 4934   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4935   ins_encode %{
 4936     __ xorq($dst$$Register, $dst$$Register);
 4937   %}
 4938   ins_pipe(ialu_reg);
 4939 %}
 4940 
 4941 instruct loadConN(rRegN dst, immN src) %{
 4942   match(Set dst src);
 4943 
 4944   ins_cost(125);
 4945   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4946   ins_encode %{
 4947     address con = (address)$src$$constant;
 4948     if (con == nullptr) {
 4949       ShouldNotReachHere();
 4950     } else {
 4951       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4952     }
 4953   %}
 4954   ins_pipe(ialu_reg_fat); // XXX
 4955 %}
 4956 
 4957 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4958   match(Set dst src);
 4959 
 4960   ins_cost(125);
 4961   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4962   ins_encode %{
 4963     address con = (address)$src$$constant;
 4964     if (con == nullptr) {
 4965       ShouldNotReachHere();
 4966     } else {
 4967       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4968     }
 4969   %}
 4970   ins_pipe(ialu_reg_fat); // XXX
 4971 %}
 4972 
 4973 instruct loadConF0(regF dst, immF0 src)
 4974 %{
 4975   match(Set dst src);
 4976   ins_cost(100);
 4977 
 4978   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4979   ins_encode %{
 4980     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4981   %}
 4982   ins_pipe(pipe_slow);
 4983 %}
 4984 
 4985 // Use the same format since predicate() can not be used here.
 4986 instruct loadConD(regD dst, immD con) %{
 4987   match(Set dst con);
 4988   ins_cost(125);
 4989   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 4990   ins_encode %{
 4991     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 4992   %}
 4993   ins_pipe(pipe_slow);
 4994 %}
 4995 
 4996 instruct loadConD0(regD dst, immD0 src)
 4997 %{
 4998   match(Set dst src);
 4999   ins_cost(100);
 5000 
 5001   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5002   ins_encode %{
 5003     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 5004   %}
 5005   ins_pipe(pipe_slow);
 5006 %}
 5007 
 5008 instruct loadSSI(rRegI dst, stackSlotI src)
 5009 %{
 5010   match(Set dst src);
 5011 
 5012   ins_cost(125);
 5013   format %{ "movl    $dst, $src\t# int stk" %}
 5014   ins_encode %{
 5015     __ movl($dst$$Register, $src$$Address);
 5016   %}
 5017   ins_pipe(ialu_reg_mem);
 5018 %}
 5019 
 5020 instruct loadSSL(rRegL dst, stackSlotL src)
 5021 %{
 5022   match(Set dst src);
 5023 
 5024   ins_cost(125);
 5025   format %{ "movq    $dst, $src\t# long stk" %}
 5026   ins_encode %{
 5027     __ movq($dst$$Register, $src$$Address);
 5028   %}
 5029   ins_pipe(ialu_reg_mem);
 5030 %}
 5031 
 5032 instruct loadSSP(rRegP dst, stackSlotP src)
 5033 %{
 5034   match(Set dst src);
 5035 
 5036   ins_cost(125);
 5037   format %{ "movq    $dst, $src\t# ptr stk" %}
 5038   ins_encode %{
 5039     __ movq($dst$$Register, $src$$Address);
 5040   %}
 5041   ins_pipe(ialu_reg_mem);
 5042 %}
 5043 
 5044 instruct loadSSF(regF dst, stackSlotF src)
 5045 %{
 5046   match(Set dst src);
 5047 
 5048   ins_cost(125);
 5049   format %{ "movss   $dst, $src\t# float stk" %}
 5050   ins_encode %{
 5051     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5052   %}
 5053   ins_pipe(pipe_slow); // XXX
 5054 %}
 5055 
 5056 // Use the same format since predicate() can not be used here.
 5057 instruct loadSSD(regD dst, stackSlotD src)
 5058 %{
 5059   match(Set dst src);
 5060 
 5061   ins_cost(125);
 5062   format %{ "movsd   $dst, $src\t# double stk" %}
 5063   ins_encode  %{
 5064     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5065   %}
 5066   ins_pipe(pipe_slow); // XXX
 5067 %}
 5068 
 5069 // Prefetch instructions for allocation.
 5070 // Must be safe to execute with invalid address (cannot fault).
 5071 
 5072 instruct prefetchAlloc( memory mem ) %{
 5073   predicate(AllocatePrefetchInstr==3);
 5074   match(PrefetchAllocation mem);
 5075   ins_cost(125);
 5076 
 5077   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5078   ins_encode %{
 5079     __ prefetchw($mem$$Address);
 5080   %}
 5081   ins_pipe(ialu_mem);
 5082 %}
 5083 
 5084 instruct prefetchAllocNTA( memory mem ) %{
 5085   predicate(AllocatePrefetchInstr==0);
 5086   match(PrefetchAllocation mem);
 5087   ins_cost(125);
 5088 
 5089   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5090   ins_encode %{
 5091     __ prefetchnta($mem$$Address);
 5092   %}
 5093   ins_pipe(ialu_mem);
 5094 %}
 5095 
 5096 instruct prefetchAllocT0( memory mem ) %{
 5097   predicate(AllocatePrefetchInstr==1);
 5098   match(PrefetchAllocation mem);
 5099   ins_cost(125);
 5100 
 5101   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5102   ins_encode %{
 5103     __ prefetcht0($mem$$Address);
 5104   %}
 5105   ins_pipe(ialu_mem);
 5106 %}
 5107 
 5108 instruct prefetchAllocT2( memory mem ) %{
 5109   predicate(AllocatePrefetchInstr==2);
 5110   match(PrefetchAllocation mem);
 5111   ins_cost(125);
 5112 
 5113   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5114   ins_encode %{
 5115     __ prefetcht2($mem$$Address);
 5116   %}
 5117   ins_pipe(ialu_mem);
 5118 %}
 5119 
 5120 //----------Store Instructions-------------------------------------------------
 5121 
 5122 // Store Byte
 5123 instruct storeB(memory mem, rRegI src)
 5124 %{
 5125   match(Set mem (StoreB mem src));
 5126 
 5127   ins_cost(125); // XXX
 5128   format %{ "movb    $mem, $src\t# byte" %}
 5129   ins_encode %{
 5130     __ movb($mem$$Address, $src$$Register);
 5131   %}
 5132   ins_pipe(ialu_mem_reg);
 5133 %}
 5134 
 5135 // Store Char/Short
 5136 instruct storeC(memory mem, rRegI src)
 5137 %{
 5138   match(Set mem (StoreC mem src));
 5139 
 5140   ins_cost(125); // XXX
 5141   format %{ "movw    $mem, $src\t# char/short" %}
 5142   ins_encode %{
 5143     __ movw($mem$$Address, $src$$Register);
 5144   %}
 5145   ins_pipe(ialu_mem_reg);
 5146 %}
 5147 
 5148 // Store Integer
 5149 instruct storeI(memory mem, rRegI src)
 5150 %{
 5151   match(Set mem (StoreI mem src));
 5152 
 5153   ins_cost(125); // XXX
 5154   format %{ "movl    $mem, $src\t# int" %}
 5155   ins_encode %{
 5156     __ movl($mem$$Address, $src$$Register);
 5157   %}
 5158   ins_pipe(ialu_mem_reg);
 5159 %}
 5160 
 5161 // Store Long
 5162 instruct storeL(memory mem, rRegL src)
 5163 %{
 5164   match(Set mem (StoreL mem src));
 5165 
 5166   ins_cost(125); // XXX
 5167   format %{ "movq    $mem, $src\t# long" %}
 5168   ins_encode %{
 5169     __ movq($mem$$Address, $src$$Register);
 5170   %}
 5171   ins_pipe(ialu_mem_reg); // XXX
 5172 %}
 5173 
 5174 // Store Pointer
 5175 instruct storeP(memory mem, any_RegP src)
 5176 %{
 5177   predicate(n->as_Store()->barrier_data() == 0);
 5178   match(Set mem (StoreP mem src));
 5179 
 5180   ins_cost(125); // XXX
 5181   format %{ "movq    $mem, $src\t# ptr" %}
 5182   ins_encode %{
 5183     __ movq($mem$$Address, $src$$Register);
 5184   %}
 5185   ins_pipe(ialu_mem_reg);
 5186 %}
 5187 
 5188 instruct storeImmP0(memory mem, immP0 zero)
 5189 %{
 5190   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5191   match(Set mem (StoreP mem zero));
 5192 
 5193   ins_cost(125); // XXX
 5194   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5195   ins_encode %{
 5196     __ movq($mem$$Address, r12);
 5197   %}
 5198   ins_pipe(ialu_mem_reg);
 5199 %}
 5200 
 5201 // Store Null Pointer, mark word, or other simple pointer constant.
 5202 instruct storeImmP(memory mem, immP31 src)
 5203 %{
 5204   predicate(n->as_Store()->barrier_data() == 0);
 5205   match(Set mem (StoreP mem src));
 5206 
 5207   ins_cost(150); // XXX
 5208   format %{ "movq    $mem, $src\t# ptr" %}
 5209   ins_encode %{
 5210     __ movq($mem$$Address, $src$$constant);
 5211   %}
 5212   ins_pipe(ialu_mem_imm);
 5213 %}
 5214 
 5215 // Store Compressed Pointer
 5216 instruct storeN(memory mem, rRegN src)
 5217 %{
 5218   predicate(n->as_Store()->barrier_data() == 0);
 5219   match(Set mem (StoreN mem src));
 5220 
 5221   ins_cost(125); // XXX
 5222   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5223   ins_encode %{
 5224     __ movl($mem$$Address, $src$$Register);
 5225   %}
 5226   ins_pipe(ialu_mem_reg);
 5227 %}
 5228 
 5229 instruct storeNKlass(memory mem, rRegN src)
 5230 %{
 5231   match(Set mem (StoreNKlass mem src));
 5232 
 5233   ins_cost(125); // XXX
 5234   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5235   ins_encode %{
 5236     __ movl($mem$$Address, $src$$Register);
 5237   %}
 5238   ins_pipe(ialu_mem_reg);
 5239 %}
 5240 
 5241 instruct storeImmN0(memory mem, immN0 zero)
 5242 %{
 5243   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5244   match(Set mem (StoreN mem zero));
 5245 
 5246   ins_cost(125); // XXX
 5247   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5248   ins_encode %{
 5249     __ movl($mem$$Address, r12);
 5250   %}
 5251   ins_pipe(ialu_mem_reg);
 5252 %}
 5253 
 5254 instruct storeImmN(memory mem, immN src)
 5255 %{
 5256   predicate(n->as_Store()->barrier_data() == 0);
 5257   match(Set mem (StoreN mem src));
 5258 
 5259   ins_cost(150); // XXX
 5260   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5261   ins_encode %{
 5262     address con = (address)$src$$constant;
 5263     if (con == nullptr) {
 5264       __ movl($mem$$Address, 0);
 5265     } else {
 5266       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5267     }
 5268   %}
 5269   ins_pipe(ialu_mem_imm);
 5270 %}
 5271 
 5272 instruct storeImmNKlass(memory mem, immNKlass src)
 5273 %{
 5274   match(Set mem (StoreNKlass mem src));
 5275 
 5276   ins_cost(150); // XXX
 5277   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5278   ins_encode %{
 5279     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5280   %}
 5281   ins_pipe(ialu_mem_imm);
 5282 %}
 5283 
 5284 // Store Integer Immediate
 5285 instruct storeImmI0(memory mem, immI_0 zero)
 5286 %{
 5287   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5288   match(Set mem (StoreI mem zero));
 5289 
 5290   ins_cost(125); // XXX
 5291   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5292   ins_encode %{
 5293     __ movl($mem$$Address, r12);
 5294   %}
 5295   ins_pipe(ialu_mem_reg);
 5296 %}
 5297 
 5298 instruct storeImmI(memory mem, immI src)
 5299 %{
 5300   match(Set mem (StoreI mem src));
 5301 
 5302   ins_cost(150);
 5303   format %{ "movl    $mem, $src\t# int" %}
 5304   ins_encode %{
 5305     __ movl($mem$$Address, $src$$constant);
 5306   %}
 5307   ins_pipe(ialu_mem_imm);
 5308 %}
 5309 
 5310 // Store Long Immediate
 5311 instruct storeImmL0(memory mem, immL0 zero)
 5312 %{
 5313   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5314   match(Set mem (StoreL mem zero));
 5315 
 5316   ins_cost(125); // XXX
 5317   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5318   ins_encode %{
 5319     __ movq($mem$$Address, r12);
 5320   %}
 5321   ins_pipe(ialu_mem_reg);
 5322 %}
 5323 
 5324 instruct storeImmL(memory mem, immL32 src)
 5325 %{
 5326   match(Set mem (StoreL mem src));
 5327 
 5328   ins_cost(150);
 5329   format %{ "movq    $mem, $src\t# long" %}
 5330   ins_encode %{
 5331     __ movq($mem$$Address, $src$$constant);
 5332   %}
 5333   ins_pipe(ialu_mem_imm);
 5334 %}
 5335 
 5336 // Store Short/Char Immediate
 5337 instruct storeImmC0(memory mem, immI_0 zero)
 5338 %{
 5339   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5340   match(Set mem (StoreC mem zero));
 5341 
 5342   ins_cost(125); // XXX
 5343   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5344   ins_encode %{
 5345     __ movw($mem$$Address, r12);
 5346   %}
 5347   ins_pipe(ialu_mem_reg);
 5348 %}
 5349 
 5350 instruct storeImmI16(memory mem, immI16 src)
 5351 %{
 5352   predicate(UseStoreImmI16);
 5353   match(Set mem (StoreC mem src));
 5354 
 5355   ins_cost(150);
 5356   format %{ "movw    $mem, $src\t# short/char" %}
 5357   ins_encode %{
 5358     __ movw($mem$$Address, $src$$constant);
 5359   %}
 5360   ins_pipe(ialu_mem_imm);
 5361 %}
 5362 
 5363 // Store Byte Immediate
 5364 instruct storeImmB0(memory mem, immI_0 zero)
 5365 %{
 5366   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5367   match(Set mem (StoreB mem zero));
 5368 
 5369   ins_cost(125); // XXX
 5370   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5371   ins_encode %{
 5372     __ movb($mem$$Address, r12);
 5373   %}
 5374   ins_pipe(ialu_mem_reg);
 5375 %}
 5376 
 5377 instruct storeImmB(memory mem, immI8 src)
 5378 %{
 5379   match(Set mem (StoreB mem src));
 5380 
 5381   ins_cost(150); // XXX
 5382   format %{ "movb    $mem, $src\t# byte" %}
 5383   ins_encode %{
 5384     __ movb($mem$$Address, $src$$constant);
 5385   %}
 5386   ins_pipe(ialu_mem_imm);
 5387 %}
 5388 
 5389 // Store Float
 5390 instruct storeF(memory mem, regF src)
 5391 %{
 5392   match(Set mem (StoreF mem src));
 5393 
 5394   ins_cost(95); // XXX
 5395   format %{ "movss   $mem, $src\t# float" %}
 5396   ins_encode %{
 5397     __ movflt($mem$$Address, $src$$XMMRegister);
 5398   %}
 5399   ins_pipe(pipe_slow); // XXX
 5400 %}
 5401 
 5402 // Store immediate Float value (it is faster than store from XMM register)
 5403 instruct storeF0(memory mem, immF0 zero)
 5404 %{
 5405   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5406   match(Set mem (StoreF mem zero));
 5407 
 5408   ins_cost(25); // XXX
 5409   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5410   ins_encode %{
 5411     __ movl($mem$$Address, r12);
 5412   %}
 5413   ins_pipe(ialu_mem_reg);
 5414 %}
 5415 
 5416 instruct storeF_imm(memory mem, immF src)
 5417 %{
 5418   match(Set mem (StoreF mem src));
 5419 
 5420   ins_cost(50);
 5421   format %{ "movl    $mem, $src\t# float" %}
 5422   ins_encode %{
 5423     __ movl($mem$$Address, jint_cast($src$$constant));
 5424   %}
 5425   ins_pipe(ialu_mem_imm);
 5426 %}
 5427 
 5428 // Store Double
 5429 instruct storeD(memory mem, regD src)
 5430 %{
 5431   match(Set mem (StoreD mem src));
 5432 
 5433   ins_cost(95); // XXX
 5434   format %{ "movsd   $mem, $src\t# double" %}
 5435   ins_encode %{
 5436     __ movdbl($mem$$Address, $src$$XMMRegister);
 5437   %}
 5438   ins_pipe(pipe_slow); // XXX
 5439 %}
 5440 
 5441 // Store immediate double 0.0 (it is faster than store from XMM register)
 5442 instruct storeD0_imm(memory mem, immD0 src)
 5443 %{
 5444   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5445   match(Set mem (StoreD mem src));
 5446 
 5447   ins_cost(50);
 5448   format %{ "movq    $mem, $src\t# double 0." %}
 5449   ins_encode %{
 5450     __ movq($mem$$Address, $src$$constant);
 5451   %}
 5452   ins_pipe(ialu_mem_imm);
 5453 %}
 5454 
 5455 instruct storeD0(memory mem, immD0 zero)
 5456 %{
 5457   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5458   match(Set mem (StoreD mem zero));
 5459 
 5460   ins_cost(25); // XXX
 5461   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5462   ins_encode %{
 5463     __ movq($mem$$Address, r12);
 5464   %}
 5465   ins_pipe(ialu_mem_reg);
 5466 %}
 5467 
 5468 instruct storeSSI(stackSlotI dst, rRegI src)
 5469 %{
 5470   match(Set dst src);
 5471 
 5472   ins_cost(100);
 5473   format %{ "movl    $dst, $src\t# int stk" %}
 5474   ins_encode %{
 5475     __ movl($dst$$Address, $src$$Register);
 5476   %}
 5477   ins_pipe( ialu_mem_reg );
 5478 %}
 5479 
 5480 instruct storeSSL(stackSlotL dst, rRegL src)
 5481 %{
 5482   match(Set dst src);
 5483 
 5484   ins_cost(100);
 5485   format %{ "movq    $dst, $src\t# long stk" %}
 5486   ins_encode %{
 5487     __ movq($dst$$Address, $src$$Register);
 5488   %}
 5489   ins_pipe(ialu_mem_reg);
 5490 %}
 5491 
 5492 instruct storeSSP(stackSlotP dst, rRegP src)
 5493 %{
 5494   match(Set dst src);
 5495 
 5496   ins_cost(100);
 5497   format %{ "movq    $dst, $src\t# ptr stk" %}
 5498   ins_encode %{
 5499     __ movq($dst$$Address, $src$$Register);
 5500   %}
 5501   ins_pipe(ialu_mem_reg);
 5502 %}
 5503 
 5504 instruct storeSSF(stackSlotF dst, regF src)
 5505 %{
 5506   match(Set dst src);
 5507 
 5508   ins_cost(95); // XXX
 5509   format %{ "movss   $dst, $src\t# float stk" %}
 5510   ins_encode %{
 5511     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5512   %}
 5513   ins_pipe(pipe_slow); // XXX
 5514 %}
 5515 
 5516 instruct storeSSD(stackSlotD dst, regD src)
 5517 %{
 5518   match(Set dst src);
 5519 
 5520   ins_cost(95); // XXX
 5521   format %{ "movsd   $dst, $src\t# double stk" %}
 5522   ins_encode %{
 5523     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5524   %}
 5525   ins_pipe(pipe_slow); // XXX
 5526 %}
 5527 
 5528 instruct cacheWB(indirect addr)
 5529 %{
 5530   predicate(VM_Version::supports_data_cache_line_flush());
 5531   match(CacheWB addr);
 5532 
 5533   ins_cost(100);
 5534   format %{"cache wb $addr" %}
 5535   ins_encode %{
 5536     assert($addr->index_position() < 0, "should be");
 5537     assert($addr$$disp == 0, "should be");
 5538     __ cache_wb(Address($addr$$base$$Register, 0));
 5539   %}
 5540   ins_pipe(pipe_slow); // XXX
 5541 %}
 5542 
 5543 instruct cacheWBPreSync()
 5544 %{
 5545   predicate(VM_Version::supports_data_cache_line_flush());
 5546   match(CacheWBPreSync);
 5547 
 5548   ins_cost(100);
 5549   format %{"cache wb presync" %}
 5550   ins_encode %{
 5551     __ cache_wbsync(true);
 5552   %}
 5553   ins_pipe(pipe_slow); // XXX
 5554 %}
 5555 
 5556 instruct cacheWBPostSync()
 5557 %{
 5558   predicate(VM_Version::supports_data_cache_line_flush());
 5559   match(CacheWBPostSync);
 5560 
 5561   ins_cost(100);
 5562   format %{"cache wb postsync" %}
 5563   ins_encode %{
 5564     __ cache_wbsync(false);
 5565   %}
 5566   ins_pipe(pipe_slow); // XXX
 5567 %}
 5568 
 5569 //----------BSWAP Instructions-------------------------------------------------
 5570 instruct bytes_reverse_int(rRegI dst) %{
 5571   match(Set dst (ReverseBytesI dst));
 5572 
 5573   format %{ "bswapl  $dst" %}
 5574   ins_encode %{
 5575     __ bswapl($dst$$Register);
 5576   %}
 5577   ins_pipe( ialu_reg );
 5578 %}
 5579 
 5580 instruct bytes_reverse_long(rRegL dst) %{
 5581   match(Set dst (ReverseBytesL dst));
 5582 
 5583   format %{ "bswapq  $dst" %}
 5584   ins_encode %{
 5585     __ bswapq($dst$$Register);
 5586   %}
 5587   ins_pipe( ialu_reg);
 5588 %}
 5589 
 5590 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5591   match(Set dst (ReverseBytesUS dst));
 5592   effect(KILL cr);
 5593 
 5594   format %{ "bswapl  $dst\n\t"
 5595             "shrl    $dst,16\n\t" %}
 5596   ins_encode %{
 5597     __ bswapl($dst$$Register);
 5598     __ shrl($dst$$Register, 16);
 5599   %}
 5600   ins_pipe( ialu_reg );
 5601 %}
 5602 
 5603 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5604   match(Set dst (ReverseBytesS dst));
 5605   effect(KILL cr);
 5606 
 5607   format %{ "bswapl  $dst\n\t"
 5608             "sar     $dst,16\n\t" %}
 5609   ins_encode %{
 5610     __ bswapl($dst$$Register);
 5611     __ sarl($dst$$Register, 16);
 5612   %}
 5613   ins_pipe( ialu_reg );
 5614 %}
 5615 
 5616 //---------- Zeros Count Instructions ------------------------------------------
 5617 
 5618 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5619   predicate(UseCountLeadingZerosInstruction);
 5620   match(Set dst (CountLeadingZerosI src));
 5621   effect(KILL cr);
 5622 
 5623   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5624   ins_encode %{
 5625     __ lzcntl($dst$$Register, $src$$Register);
 5626   %}
 5627   ins_pipe(ialu_reg);
 5628 %}
 5629 
 5630 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5631   predicate(UseCountLeadingZerosInstruction);
 5632   match(Set dst (CountLeadingZerosI (LoadI src)));
 5633   effect(KILL cr);
 5634   ins_cost(175);
 5635   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5636   ins_encode %{
 5637     __ lzcntl($dst$$Register, $src$$Address);
 5638   %}
 5639   ins_pipe(ialu_reg_mem);
 5640 %}
 5641 
 5642 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5643   predicate(!UseCountLeadingZerosInstruction);
 5644   match(Set dst (CountLeadingZerosI src));
 5645   effect(KILL cr);
 5646 
 5647   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5648             "jnz     skip\n\t"
 5649             "movl    $dst, -1\n"
 5650       "skip:\n\t"
 5651             "negl    $dst\n\t"
 5652             "addl    $dst, 31" %}
 5653   ins_encode %{
 5654     Register Rdst = $dst$$Register;
 5655     Register Rsrc = $src$$Register;
 5656     Label skip;
 5657     __ bsrl(Rdst, Rsrc);
 5658     __ jccb(Assembler::notZero, skip);
 5659     __ movl(Rdst, -1);
 5660     __ bind(skip);
 5661     __ negl(Rdst);
 5662     __ addl(Rdst, BitsPerInt - 1);
 5663   %}
 5664   ins_pipe(ialu_reg);
 5665 %}
 5666 
 5667 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5668   predicate(UseCountLeadingZerosInstruction);
 5669   match(Set dst (CountLeadingZerosL src));
 5670   effect(KILL cr);
 5671 
 5672   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5673   ins_encode %{
 5674     __ lzcntq($dst$$Register, $src$$Register);
 5675   %}
 5676   ins_pipe(ialu_reg);
 5677 %}
 5678 
 5679 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5680   predicate(UseCountLeadingZerosInstruction);
 5681   match(Set dst (CountLeadingZerosL (LoadL src)));
 5682   effect(KILL cr);
 5683   ins_cost(175);
 5684   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5685   ins_encode %{
 5686     __ lzcntq($dst$$Register, $src$$Address);
 5687   %}
 5688   ins_pipe(ialu_reg_mem);
 5689 %}
 5690 
 5691 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5692   predicate(!UseCountLeadingZerosInstruction);
 5693   match(Set dst (CountLeadingZerosL src));
 5694   effect(KILL cr);
 5695 
 5696   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5697             "jnz     skip\n\t"
 5698             "movl    $dst, -1\n"
 5699       "skip:\n\t"
 5700             "negl    $dst\n\t"
 5701             "addl    $dst, 63" %}
 5702   ins_encode %{
 5703     Register Rdst = $dst$$Register;
 5704     Register Rsrc = $src$$Register;
 5705     Label skip;
 5706     __ bsrq(Rdst, Rsrc);
 5707     __ jccb(Assembler::notZero, skip);
 5708     __ movl(Rdst, -1);
 5709     __ bind(skip);
 5710     __ negl(Rdst);
 5711     __ addl(Rdst, BitsPerLong - 1);
 5712   %}
 5713   ins_pipe(ialu_reg);
 5714 %}
 5715 
 5716 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5717   predicate(UseCountTrailingZerosInstruction);
 5718   match(Set dst (CountTrailingZerosI src));
 5719   effect(KILL cr);
 5720 
 5721   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5722   ins_encode %{
 5723     __ tzcntl($dst$$Register, $src$$Register);
 5724   %}
 5725   ins_pipe(ialu_reg);
 5726 %}
 5727 
 5728 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5729   predicate(UseCountTrailingZerosInstruction);
 5730   match(Set dst (CountTrailingZerosI (LoadI src)));
 5731   effect(KILL cr);
 5732   ins_cost(175);
 5733   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5734   ins_encode %{
 5735     __ tzcntl($dst$$Register, $src$$Address);
 5736   %}
 5737   ins_pipe(ialu_reg_mem);
 5738 %}
 5739 
 5740 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5741   predicate(!UseCountTrailingZerosInstruction);
 5742   match(Set dst (CountTrailingZerosI src));
 5743   effect(KILL cr);
 5744 
 5745   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5746             "jnz     done\n\t"
 5747             "movl    $dst, 32\n"
 5748       "done:" %}
 5749   ins_encode %{
 5750     Register Rdst = $dst$$Register;
 5751     Label done;
 5752     __ bsfl(Rdst, $src$$Register);
 5753     __ jccb(Assembler::notZero, done);
 5754     __ movl(Rdst, BitsPerInt);
 5755     __ bind(done);
 5756   %}
 5757   ins_pipe(ialu_reg);
 5758 %}
 5759 
 5760 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5761   predicate(UseCountTrailingZerosInstruction);
 5762   match(Set dst (CountTrailingZerosL src));
 5763   effect(KILL cr);
 5764 
 5765   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5766   ins_encode %{
 5767     __ tzcntq($dst$$Register, $src$$Register);
 5768   %}
 5769   ins_pipe(ialu_reg);
 5770 %}
 5771 
 5772 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5773   predicate(UseCountTrailingZerosInstruction);
 5774   match(Set dst (CountTrailingZerosL (LoadL src)));
 5775   effect(KILL cr);
 5776   ins_cost(175);
 5777   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5778   ins_encode %{
 5779     __ tzcntq($dst$$Register, $src$$Address);
 5780   %}
 5781   ins_pipe(ialu_reg_mem);
 5782 %}
 5783 
 5784 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5785   predicate(!UseCountTrailingZerosInstruction);
 5786   match(Set dst (CountTrailingZerosL src));
 5787   effect(KILL cr);
 5788 
 5789   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5790             "jnz     done\n\t"
 5791             "movl    $dst, 64\n"
 5792       "done:" %}
 5793   ins_encode %{
 5794     Register Rdst = $dst$$Register;
 5795     Label done;
 5796     __ bsfq(Rdst, $src$$Register);
 5797     __ jccb(Assembler::notZero, done);
 5798     __ movl(Rdst, BitsPerLong);
 5799     __ bind(done);
 5800   %}
 5801   ins_pipe(ialu_reg);
 5802 %}
 5803 
 5804 //--------------- Reverse Operation Instructions ----------------
 5805 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5806   predicate(!VM_Version::supports_gfni());
 5807   match(Set dst (ReverseI src));
 5808   effect(TEMP dst, TEMP rtmp, KILL cr);
 5809   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5810   ins_encode %{
 5811     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5812   %}
 5813   ins_pipe( ialu_reg );
 5814 %}
 5815 
 5816 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5817   predicate(VM_Version::supports_gfni());
 5818   match(Set dst (ReverseI src));
 5819   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5820   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5821   ins_encode %{
 5822     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5823   %}
 5824   ins_pipe( ialu_reg );
 5825 %}
 5826 
 5827 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5828   predicate(!VM_Version::supports_gfni());
 5829   match(Set dst (ReverseL src));
 5830   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5831   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5832   ins_encode %{
 5833     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5834   %}
 5835   ins_pipe( ialu_reg );
 5836 %}
 5837 
 5838 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5839   predicate(VM_Version::supports_gfni());
 5840   match(Set dst (ReverseL src));
 5841   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5842   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5843   ins_encode %{
 5844     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5845   %}
 5846   ins_pipe( ialu_reg );
 5847 %}
 5848 
 5849 //---------- Population Count Instructions -------------------------------------
 5850 
 5851 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5852   predicate(UsePopCountInstruction);
 5853   match(Set dst (PopCountI src));
 5854   effect(KILL cr);
 5855 
 5856   format %{ "popcnt  $dst, $src" %}
 5857   ins_encode %{
 5858     __ popcntl($dst$$Register, $src$$Register);
 5859   %}
 5860   ins_pipe(ialu_reg);
 5861 %}
 5862 
 5863 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5864   predicate(UsePopCountInstruction);
 5865   match(Set dst (PopCountI (LoadI mem)));
 5866   effect(KILL cr);
 5867 
 5868   format %{ "popcnt  $dst, $mem" %}
 5869   ins_encode %{
 5870     __ popcntl($dst$$Register, $mem$$Address);
 5871   %}
 5872   ins_pipe(ialu_reg);
 5873 %}
 5874 
 5875 // Note: Long.bitCount(long) returns an int.
 5876 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5877   predicate(UsePopCountInstruction);
 5878   match(Set dst (PopCountL src));
 5879   effect(KILL cr);
 5880 
 5881   format %{ "popcnt  $dst, $src" %}
 5882   ins_encode %{
 5883     __ popcntq($dst$$Register, $src$$Register);
 5884   %}
 5885   ins_pipe(ialu_reg);
 5886 %}
 5887 
 5888 // Note: Long.bitCount(long) returns an int.
 5889 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5890   predicate(UsePopCountInstruction);
 5891   match(Set dst (PopCountL (LoadL mem)));
 5892   effect(KILL cr);
 5893 
 5894   format %{ "popcnt  $dst, $mem" %}
 5895   ins_encode %{
 5896     __ popcntq($dst$$Register, $mem$$Address);
 5897   %}
 5898   ins_pipe(ialu_reg);
 5899 %}
 5900 
 5901 
 5902 //----------MemBar Instructions-----------------------------------------------
 5903 // Memory barrier flavors
 5904 
 5905 instruct membar_acquire()
 5906 %{
 5907   match(MemBarAcquire);
 5908   match(LoadFence);
 5909   ins_cost(0);
 5910 
 5911   size(0);
 5912   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5913   ins_encode();
 5914   ins_pipe(empty);
 5915 %}
 5916 
 5917 instruct membar_acquire_lock()
 5918 %{
 5919   match(MemBarAcquireLock);
 5920   ins_cost(0);
 5921 
 5922   size(0);
 5923   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5924   ins_encode();
 5925   ins_pipe(empty);
 5926 %}
 5927 
 5928 instruct membar_release()
 5929 %{
 5930   match(MemBarRelease);
 5931   match(StoreFence);
 5932   ins_cost(0);
 5933 
 5934   size(0);
 5935   format %{ "MEMBAR-release ! (empty encoding)" %}
 5936   ins_encode();
 5937   ins_pipe(empty);
 5938 %}
 5939 
 5940 instruct membar_release_lock()
 5941 %{
 5942   match(MemBarReleaseLock);
 5943   ins_cost(0);
 5944 
 5945   size(0);
 5946   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5947   ins_encode();
 5948   ins_pipe(empty);
 5949 %}
 5950 
 5951 instruct membar_volatile(rFlagsReg cr) %{
 5952   match(MemBarVolatile);
 5953   effect(KILL cr);
 5954   ins_cost(400);
 5955 
 5956   format %{
 5957     $$template
 5958     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5959   %}
 5960   ins_encode %{
 5961     __ membar(Assembler::StoreLoad);
 5962   %}
 5963   ins_pipe(pipe_slow);
 5964 %}
 5965 
 5966 instruct unnecessary_membar_volatile()
 5967 %{
 5968   match(MemBarVolatile);
 5969   predicate(Matcher::post_store_load_barrier(n));
 5970   ins_cost(0);
 5971 
 5972   size(0);
 5973   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5974   ins_encode();
 5975   ins_pipe(empty);
 5976 %}
 5977 
 5978 instruct membar_storestore() %{
 5979   match(MemBarStoreStore);
 5980   match(StoreStoreFence);
 5981   ins_cost(0);
 5982 
 5983   size(0);
 5984   format %{ "MEMBAR-storestore (empty encoding)" %}
 5985   ins_encode( );
 5986   ins_pipe(empty);
 5987 %}
 5988 
 5989 //----------Move Instructions--------------------------------------------------
 5990 
 5991 instruct castX2P(rRegP dst, rRegL src)
 5992 %{
 5993   match(Set dst (CastX2P src));
 5994 
 5995   format %{ "movq    $dst, $src\t# long->ptr" %}
 5996   ins_encode %{
 5997     if ($dst$$reg != $src$$reg) {
 5998       __ movptr($dst$$Register, $src$$Register);
 5999     }
 6000   %}
 6001   ins_pipe(ialu_reg_reg); // XXX
 6002 %}
 6003 
 6004 instruct castP2X(rRegL dst, rRegP src)
 6005 %{
 6006   match(Set dst (CastP2X src));
 6007 
 6008   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6009   ins_encode %{
 6010     if ($dst$$reg != $src$$reg) {
 6011       __ movptr($dst$$Register, $src$$Register);
 6012     }
 6013   %}
 6014   ins_pipe(ialu_reg_reg); // XXX
 6015 %}
 6016 
 6017 // Convert oop into int for vectors alignment masking
 6018 instruct convP2I(rRegI dst, rRegP src)
 6019 %{
 6020   match(Set dst (ConvL2I (CastP2X src)));
 6021 
 6022   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6023   ins_encode %{
 6024     __ movl($dst$$Register, $src$$Register);
 6025   %}
 6026   ins_pipe(ialu_reg_reg); // XXX
 6027 %}
 6028 
 6029 // Convert compressed oop into int for vectors alignment masking
 6030 // in case of 32bit oops (heap < 4Gb).
 6031 instruct convN2I(rRegI dst, rRegN src)
 6032 %{
 6033   predicate(CompressedOops::shift() == 0);
 6034   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6035 
 6036   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6037   ins_encode %{
 6038     __ movl($dst$$Register, $src$$Register);
 6039   %}
 6040   ins_pipe(ialu_reg_reg); // XXX
 6041 %}
 6042 
 6043 // Convert oop pointer into compressed form
 6044 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6045   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6046   match(Set dst (EncodeP src));
 6047   effect(KILL cr);
 6048   format %{ "encode_heap_oop $dst,$src" %}
 6049   ins_encode %{
 6050     Register s = $src$$Register;
 6051     Register d = $dst$$Register;
 6052     if (s != d) {
 6053       __ movq(d, s);
 6054     }
 6055     __ encode_heap_oop(d);
 6056   %}
 6057   ins_pipe(ialu_reg_long);
 6058 %}
 6059 
 6060 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6061   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6062   match(Set dst (EncodeP src));
 6063   effect(KILL cr);
 6064   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6065   ins_encode %{
 6066     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6067   %}
 6068   ins_pipe(ialu_reg_long);
 6069 %}
 6070 
 6071 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6072   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6073             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6074   match(Set dst (DecodeN src));
 6075   effect(KILL cr);
 6076   format %{ "decode_heap_oop $dst,$src" %}
 6077   ins_encode %{
 6078     Register s = $src$$Register;
 6079     Register d = $dst$$Register;
 6080     if (s != d) {
 6081       __ movq(d, s);
 6082     }
 6083     __ decode_heap_oop(d);
 6084   %}
 6085   ins_pipe(ialu_reg_long);
 6086 %}
 6087 
 6088 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6089   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6090             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6091   match(Set dst (DecodeN src));
 6092   effect(KILL cr);
 6093   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6094   ins_encode %{
 6095     Register s = $src$$Register;
 6096     Register d = $dst$$Register;
 6097     if (s != d) {
 6098       __ decode_heap_oop_not_null(d, s);
 6099     } else {
 6100       __ decode_heap_oop_not_null(d);
 6101     }
 6102   %}
 6103   ins_pipe(ialu_reg_long);
 6104 %}
 6105 
 6106 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6107   match(Set dst (EncodePKlass src));
 6108   effect(TEMP dst, KILL cr);
 6109   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6110   ins_encode %{
 6111     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6112   %}
 6113   ins_pipe(ialu_reg_long);
 6114 %}
 6115 
 6116 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6117   match(Set dst (DecodeNKlass src));
 6118   effect(TEMP dst, KILL cr);
 6119   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6120   ins_encode %{
 6121     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6122   %}
 6123   ins_pipe(ialu_reg_long);
 6124 %}
 6125 
 6126 //----------Conditional Move---------------------------------------------------
 6127 // Jump
 6128 // dummy instruction for generating temp registers
 6129 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6130   match(Jump (LShiftL switch_val shift));
 6131   ins_cost(350);
 6132   predicate(false);
 6133   effect(TEMP dest);
 6134 
 6135   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6136             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6137   ins_encode %{
 6138     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6139     // to do that and the compiler is using that register as one it can allocate.
 6140     // So we build it all by hand.
 6141     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6142     // ArrayAddress dispatch(table, index);
 6143     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6144     __ lea($dest$$Register, $constantaddress);
 6145     __ jmp(dispatch);
 6146   %}
 6147   ins_pipe(pipe_jmp);
 6148 %}
 6149 
 6150 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6151   match(Jump (AddL (LShiftL switch_val shift) offset));
 6152   ins_cost(350);
 6153   effect(TEMP dest);
 6154 
 6155   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6156             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6157   ins_encode %{
 6158     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6159     // to do that and the compiler is using that register as one it can allocate.
 6160     // So we build it all by hand.
 6161     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6162     // ArrayAddress dispatch(table, index);
 6163     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6164     __ lea($dest$$Register, $constantaddress);
 6165     __ jmp(dispatch);
 6166   %}
 6167   ins_pipe(pipe_jmp);
 6168 %}
 6169 
 6170 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6171   match(Jump switch_val);
 6172   ins_cost(350);
 6173   effect(TEMP dest);
 6174 
 6175   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6176             "jmp     [$dest + $switch_val]\n\t" %}
 6177   ins_encode %{
 6178     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6179     // to do that and the compiler is using that register as one it can allocate.
 6180     // So we build it all by hand.
 6181     // Address index(noreg, switch_reg, Address::times_1);
 6182     // ArrayAddress dispatch(table, index);
 6183     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6184     __ lea($dest$$Register, $constantaddress);
 6185     __ jmp(dispatch);
 6186   %}
 6187   ins_pipe(pipe_jmp);
 6188 %}
 6189 
 6190 // Conditional move
 6191 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6192 %{
 6193   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6194   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6195 
 6196   ins_cost(100); // XXX
 6197   format %{ "setbn$cop $dst\t# signed, int" %}
 6198   ins_encode %{
 6199     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6200     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6201   %}
 6202   ins_pipe(ialu_reg);
 6203 %}
 6204 
 6205 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6206 %{
 6207   predicate(!UseAPX);
 6208   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6209 
 6210   ins_cost(200); // XXX
 6211   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6212   ins_encode %{
 6213     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6214   %}
 6215   ins_pipe(pipe_cmov_reg);
 6216 %}
 6217 
 6218 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6219 %{
 6220   predicate(UseAPX);
 6221   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6222 
 6223   ins_cost(200);
 6224   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6225   ins_encode %{
 6226     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6227   %}
 6228   ins_pipe(pipe_cmov_reg);
 6229 %}
 6230 
 6231 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6232 %{
 6233   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6234   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6235 
 6236   ins_cost(100); // XXX
 6237   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6238   ins_encode %{
 6239     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6240     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6241   %}
 6242   ins_pipe(ialu_reg);
 6243 %}
 6244 
 6245 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6246   predicate(!UseAPX);
 6247   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6248 
 6249   ins_cost(200); // XXX
 6250   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6251   ins_encode %{
 6252     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6253   %}
 6254   ins_pipe(pipe_cmov_reg);
 6255 %}
 6256 
 6257 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6258   predicate(UseAPX);
 6259   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6260 
 6261   ins_cost(200);
 6262   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6263   ins_encode %{
 6264     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6265   %}
 6266   ins_pipe(pipe_cmov_reg);
 6267 %}
 6268 
 6269 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6270 %{
 6271   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6272   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6273 
 6274   ins_cost(100); // XXX
 6275   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6276   ins_encode %{
 6277     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6278     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6279   %}
 6280   ins_pipe(ialu_reg);
 6281 %}
 6282 
 6283 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6284   predicate(!UseAPX);
 6285   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6286   ins_cost(200);
 6287   expand %{
 6288     cmovI_regU(cop, cr, dst, src);
 6289   %}
 6290 %}
 6291 
 6292 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6293   predicate(UseAPX);
 6294   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6295   ins_cost(200);
 6296   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6297   ins_encode %{
 6298     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6299   %}
 6300   ins_pipe(pipe_cmov_reg);
 6301 %}
 6302 
 6303 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6304   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6305   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6306 
 6307   ins_cost(200); // XXX
 6308   format %{ "cmovpl  $dst, $src\n\t"
 6309             "cmovnel $dst, $src" %}
 6310   ins_encode %{
 6311     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6312     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6313   %}
 6314   ins_pipe(pipe_cmov_reg);
 6315 %}
 6316 
 6317 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6318   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6319   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6320   effect(TEMP dst);
 6321 
 6322   ins_cost(200);
 6323   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6324             "cmovnel  $dst, $src2" %}
 6325   ins_encode %{
 6326     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6327     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6328   %}
 6329   ins_pipe(pipe_cmov_reg);
 6330 %}
 6331 
 6332 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6333 // inputs of the CMove
 6334 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6335   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6336   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6337   effect(TEMP dst);
 6338 
 6339   ins_cost(200); // XXX
 6340   format %{ "cmovpl  $dst, $src\n\t"
 6341             "cmovnel $dst, $src" %}
 6342   ins_encode %{
 6343     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6344     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6345   %}
 6346   ins_pipe(pipe_cmov_reg);
 6347 %}
 6348 
 6349 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6350 // and parity flag bit is set if any of the operand is a NaN.
 6351 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6352   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6353   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6354   effect(TEMP dst);
 6355 
 6356   ins_cost(200);
 6357   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6358             "cmovnel  $dst, $src2" %}
 6359   ins_encode %{
 6360     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6361     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6362   %}
 6363   ins_pipe(pipe_cmov_reg);
 6364 %}
 6365 
 6366 // Conditional move
 6367 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6368   predicate(!UseAPX);
 6369   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6370 
 6371   ins_cost(250); // XXX
 6372   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6373   ins_encode %{
 6374     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6375   %}
 6376   ins_pipe(pipe_cmov_mem);
 6377 %}
 6378 
 6379 // Conditional move
 6380 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6381 %{
 6382   predicate(UseAPX);
 6383   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6384 
 6385   ins_cost(250);
 6386   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6387   ins_encode %{
 6388     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6389   %}
 6390   ins_pipe(pipe_cmov_mem);
 6391 %}
 6392 
 6393 // Conditional move
 6394 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6395 %{
 6396   predicate(!UseAPX);
 6397   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6398 
 6399   ins_cost(250); // XXX
 6400   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6401   ins_encode %{
 6402     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6403   %}
 6404   ins_pipe(pipe_cmov_mem);
 6405 %}
 6406 
 6407 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6408   predicate(!UseAPX);
 6409   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6410   ins_cost(250);
 6411   expand %{
 6412     cmovI_memU(cop, cr, dst, src);
 6413   %}
 6414 %}
 6415 
 6416 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6417 %{
 6418   predicate(UseAPX);
 6419   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6420 
 6421   ins_cost(250);
 6422   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6423   ins_encode %{
 6424     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6425   %}
 6426   ins_pipe(pipe_cmov_mem);
 6427 %}
 6428 
 6429 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
 6430 %{
 6431   predicate(UseAPX);
 6432   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6433   ins_cost(250);
 6434   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6435   ins_encode %{
 6436     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6437   %}
 6438   ins_pipe(pipe_cmov_mem);
 6439 %}
 6440 
 6441 // Conditional move
 6442 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6443 %{
 6444   predicate(!UseAPX);
 6445   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6446 
 6447   ins_cost(200); // XXX
 6448   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6449   ins_encode %{
 6450     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6451   %}
 6452   ins_pipe(pipe_cmov_reg);
 6453 %}
 6454 
 6455 // Conditional move ndd
 6456 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6457 %{
 6458   predicate(UseAPX);
 6459   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6460 
 6461   ins_cost(200);
 6462   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6463   ins_encode %{
 6464     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6465   %}
 6466   ins_pipe(pipe_cmov_reg);
 6467 %}
 6468 
 6469 // Conditional move
 6470 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6471 %{
 6472   predicate(!UseAPX);
 6473   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6474 
 6475   ins_cost(200); // XXX
 6476   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6477   ins_encode %{
 6478     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6479   %}
 6480   ins_pipe(pipe_cmov_reg);
 6481 %}
 6482 
 6483 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6484   predicate(!UseAPX);
 6485   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6486   ins_cost(200);
 6487   expand %{
 6488     cmovN_regU(cop, cr, dst, src);
 6489   %}
 6490 %}
 6491 
 6492 // Conditional move ndd
 6493 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6494 %{
 6495   predicate(UseAPX);
 6496   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6497 
 6498   ins_cost(200);
 6499   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6500   ins_encode %{
 6501     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6502   %}
 6503   ins_pipe(pipe_cmov_reg);
 6504 %}
 6505 
 6506 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6507   predicate(UseAPX);
 6508   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6509   ins_cost(200);
 6510   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6511   ins_encode %{
 6512     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6513   %}
 6514   ins_pipe(pipe_cmov_reg);
 6515 %}
 6516 
 6517 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6518   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6519   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6520 
 6521   ins_cost(200); // XXX
 6522   format %{ "cmovpl  $dst, $src\n\t"
 6523             "cmovnel $dst, $src" %}
 6524   ins_encode %{
 6525     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6526     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6527   %}
 6528   ins_pipe(pipe_cmov_reg);
 6529 %}
 6530 
 6531 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6532 // inputs of the CMove
 6533 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6534   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6535   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6536 
 6537   ins_cost(200); // XXX
 6538   format %{ "cmovpl  $dst, $src\n\t"
 6539             "cmovnel $dst, $src" %}
 6540   ins_encode %{
 6541     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6542     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6543   %}
 6544   ins_pipe(pipe_cmov_reg);
 6545 %}
 6546 
 6547 // Conditional move
 6548 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6549 %{
 6550   predicate(!UseAPX);
 6551   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6552 
 6553   ins_cost(200); // XXX
 6554   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6555   ins_encode %{
 6556     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6557   %}
 6558   ins_pipe(pipe_cmov_reg);  // XXX
 6559 %}
 6560 
 6561 // Conditional move ndd
 6562 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6563 %{
 6564   predicate(UseAPX);
 6565   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6566 
 6567   ins_cost(200);
 6568   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6569   ins_encode %{
 6570     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6571   %}
 6572   ins_pipe(pipe_cmov_reg);
 6573 %}
 6574 
 6575 // Conditional move
 6576 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6577 %{
 6578   predicate(!UseAPX);
 6579   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6580 
 6581   ins_cost(200); // XXX
 6582   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6583   ins_encode %{
 6584     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6585   %}
 6586   ins_pipe(pipe_cmov_reg); // XXX
 6587 %}
 6588 
 6589 // Conditional move ndd
 6590 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6591 %{
 6592   predicate(UseAPX);
 6593   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6594 
 6595   ins_cost(200);
 6596   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6597   ins_encode %{
 6598     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6599   %}
 6600   ins_pipe(pipe_cmov_reg);
 6601 %}
 6602 
 6603 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6604   predicate(!UseAPX);
 6605   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6606   ins_cost(200);
 6607   expand %{
 6608     cmovP_regU(cop, cr, dst, src);
 6609   %}
 6610 %}
 6611 
 6612 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6613   predicate(UseAPX);
 6614   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6615   ins_cost(200);
 6616   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6617   ins_encode %{
 6618     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6619   %}
 6620   ins_pipe(pipe_cmov_reg);
 6621 %}
 6622 
 6623 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6624   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6625   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6626 
 6627   ins_cost(200); // XXX
 6628   format %{ "cmovpq  $dst, $src\n\t"
 6629             "cmovneq $dst, $src" %}
 6630   ins_encode %{
 6631     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6632     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6633   %}
 6634   ins_pipe(pipe_cmov_reg);
 6635 %}
 6636 
 6637 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6638   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6639   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6640   effect(TEMP dst);
 6641 
 6642   ins_cost(200);
 6643   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6644             "cmovneq  $dst, $src2" %}
 6645   ins_encode %{
 6646     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6647     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6648   %}
 6649   ins_pipe(pipe_cmov_reg);
 6650 %}
 6651 
 6652 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6653 // inputs of the CMove
 6654 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6655   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6656   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6657 
 6658   ins_cost(200); // XXX
 6659   format %{ "cmovpq  $dst, $src\n\t"
 6660             "cmovneq $dst, $src" %}
 6661   ins_encode %{
 6662     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6663     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6664   %}
 6665   ins_pipe(pipe_cmov_reg);
 6666 %}
 6667 
 6668 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6669   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6670   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6671   effect(TEMP dst);
 6672 
 6673   ins_cost(200);
 6674   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6675             "cmovneq  $dst, $src2" %}
 6676   ins_encode %{
 6677     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6678     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6679   %}
 6680   ins_pipe(pipe_cmov_reg);
 6681 %}
 6682 
 6683 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6684 %{
 6685   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6686   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6687 
 6688   ins_cost(100); // XXX
 6689   format %{ "setbn$cop $dst\t# signed, long" %}
 6690   ins_encode %{
 6691     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6692     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6693   %}
 6694   ins_pipe(ialu_reg);
 6695 %}
 6696 
 6697 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6698 %{
 6699   predicate(!UseAPX);
 6700   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6701 
 6702   ins_cost(200); // XXX
 6703   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6704   ins_encode %{
 6705     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6706   %}
 6707   ins_pipe(pipe_cmov_reg);  // XXX
 6708 %}
 6709 
 6710 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6711 %{
 6712   predicate(UseAPX);
 6713   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6714 
 6715   ins_cost(200);
 6716   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6717   ins_encode %{
 6718     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6719   %}
 6720   ins_pipe(pipe_cmov_reg);
 6721 %}
 6722 
 6723 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6724 %{
 6725   predicate(!UseAPX);
 6726   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6727 
 6728   ins_cost(200); // XXX
 6729   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6730   ins_encode %{
 6731     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6732   %}
 6733   ins_pipe(pipe_cmov_mem);  // XXX
 6734 %}
 6735 
 6736 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6737 %{
 6738   predicate(UseAPX);
 6739   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6740 
 6741   ins_cost(200);
 6742   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6743   ins_encode %{
 6744     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6745   %}
 6746   ins_pipe(pipe_cmov_mem);
 6747 %}
 6748 
 6749 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6750 %{
 6751   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6752   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6753 
 6754   ins_cost(100); // XXX
 6755   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6756   ins_encode %{
 6757     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6758     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6759   %}
 6760   ins_pipe(ialu_reg);
 6761 %}
 6762 
 6763 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6764 %{
 6765   predicate(!UseAPX);
 6766   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6767 
 6768   ins_cost(200); // XXX
 6769   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6770   ins_encode %{
 6771     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6772   %}
 6773   ins_pipe(pipe_cmov_reg); // XXX
 6774 %}
 6775 
 6776 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6777 %{
 6778   predicate(UseAPX);
 6779   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6780 
 6781   ins_cost(200);
 6782   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6783   ins_encode %{
 6784     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6785   %}
 6786   ins_pipe(pipe_cmov_reg);
 6787 %}
 6788 
 6789 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6790 %{
 6791   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6792   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6793 
 6794   ins_cost(100); // XXX
 6795   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6796   ins_encode %{
 6797     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6798     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6799   %}
 6800   ins_pipe(ialu_reg);
 6801 %}
 6802 
 6803 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6804   predicate(!UseAPX);
 6805   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6806   ins_cost(200);
 6807   expand %{
 6808     cmovL_regU(cop, cr, dst, src);
 6809   %}
 6810 %}
 6811 
 6812 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
 6813 %{
 6814   predicate(UseAPX);
 6815   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6816   ins_cost(200);
 6817   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6818   ins_encode %{
 6819     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6820   %}
 6821   ins_pipe(pipe_cmov_reg);
 6822 %}
 6823 
 6824 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6825   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6826   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6827 
 6828   ins_cost(200); // XXX
 6829   format %{ "cmovpq  $dst, $src\n\t"
 6830             "cmovneq $dst, $src" %}
 6831   ins_encode %{
 6832     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6833     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6834   %}
 6835   ins_pipe(pipe_cmov_reg);
 6836 %}
 6837 
 6838 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6839   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6840   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6841   effect(TEMP dst);
 6842 
 6843   ins_cost(200);
 6844   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6845             "cmovneq  $dst, $src2" %}
 6846   ins_encode %{
 6847     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6848     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6849   %}
 6850   ins_pipe(pipe_cmov_reg);
 6851 %}
 6852 
 6853 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6854 // inputs of the CMove
 6855 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6856   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6857   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6858 
 6859   ins_cost(200); // XXX
 6860   format %{ "cmovpq  $dst, $src\n\t"
 6861             "cmovneq $dst, $src" %}
 6862   ins_encode %{
 6863     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6864     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6865   %}
 6866   ins_pipe(pipe_cmov_reg);
 6867 %}
 6868 
 6869 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6870   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6871   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6872   effect(TEMP dst);
 6873 
 6874   ins_cost(200);
 6875   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6876             "cmovneq $dst, $src2" %}
 6877   ins_encode %{
 6878     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6879     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6880   %}
 6881   ins_pipe(pipe_cmov_reg);
 6882 %}
 6883 
 6884 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6885 %{
 6886   predicate(!UseAPX);
 6887   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6888 
 6889   ins_cost(200); // XXX
 6890   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6891   ins_encode %{
 6892     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6893   %}
 6894   ins_pipe(pipe_cmov_mem); // XXX
 6895 %}
 6896 
 6897 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6898   predicate(!UseAPX);
 6899   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6900   ins_cost(200);
 6901   expand %{
 6902     cmovL_memU(cop, cr, dst, src);
 6903   %}
 6904 %}
 6905 
 6906 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6907 %{
 6908   predicate(UseAPX);
 6909   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6910 
 6911   ins_cost(200);
 6912   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6913   ins_encode %{
 6914     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6915   %}
 6916   ins_pipe(pipe_cmov_mem);
 6917 %}
 6918 
 6919 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
 6920 %{
 6921   predicate(UseAPX);
 6922   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6923   ins_cost(200);
 6924   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6925   ins_encode %{
 6926     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6927   %}
 6928   ins_pipe(pipe_cmov_mem);
 6929 %}
 6930 
 6931 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6932 %{
 6933   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6934 
 6935   ins_cost(200); // XXX
 6936   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6937             "movss     $dst, $src\n"
 6938     "skip:" %}
 6939   ins_encode %{
 6940     Label Lskip;
 6941     // Invert sense of branch from sense of CMOV
 6942     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6943     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6944     __ bind(Lskip);
 6945   %}
 6946   ins_pipe(pipe_slow);
 6947 %}
 6948 
 6949 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6950 %{
 6951   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6952 
 6953   ins_cost(200); // XXX
 6954   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6955             "movss     $dst, $src\n"
 6956     "skip:" %}
 6957   ins_encode %{
 6958     Label Lskip;
 6959     // Invert sense of branch from sense of CMOV
 6960     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6961     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6962     __ bind(Lskip);
 6963   %}
 6964   ins_pipe(pipe_slow);
 6965 %}
 6966 
 6967 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6968   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6969   ins_cost(200);
 6970   expand %{
 6971     cmovF_regU(cop, cr, dst, src);
 6972   %}
 6973 %}
 6974 
 6975 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6976 %{
 6977   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6978 
 6979   ins_cost(200); // XXX
 6980   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6981             "movsd     $dst, $src\n"
 6982     "skip:" %}
 6983   ins_encode %{
 6984     Label Lskip;
 6985     // Invert sense of branch from sense of CMOV
 6986     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6987     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 6988     __ bind(Lskip);
 6989   %}
 6990   ins_pipe(pipe_slow);
 6991 %}
 6992 
 6993 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 6994 %{
 6995   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6996 
 6997   ins_cost(200); // XXX
 6998   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 6999             "movsd     $dst, $src\n"
 7000     "skip:" %}
 7001   ins_encode %{
 7002     Label Lskip;
 7003     // Invert sense of branch from sense of CMOV
 7004     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7005     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7006     __ bind(Lskip);
 7007   %}
 7008   ins_pipe(pipe_slow);
 7009 %}
 7010 
 7011 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7012   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7013   ins_cost(200);
 7014   expand %{
 7015     cmovD_regU(cop, cr, dst, src);
 7016   %}
 7017 %}
 7018 
 7019 //----------Arithmetic Instructions--------------------------------------------
 7020 //----------Addition Instructions----------------------------------------------
 7021 
 7022 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7023 %{
 7024   predicate(!UseAPX);
 7025   match(Set dst (AddI dst src));
 7026   effect(KILL cr);
 7027   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7028   format %{ "addl    $dst, $src\t# int" %}
 7029   ins_encode %{
 7030     __ addl($dst$$Register, $src$$Register);
 7031   %}
 7032   ins_pipe(ialu_reg_reg);
 7033 %}
 7034 
 7035 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 7036 %{
 7037   predicate(UseAPX);
 7038   match(Set dst (AddI src1 src2));
 7039   effect(KILL cr);
 7040   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7041 
 7042   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7043   ins_encode %{
 7044     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 7045   %}
 7046   ins_pipe(ialu_reg_reg);
 7047 %}
 7048 
 7049 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7050 %{
 7051   predicate(!UseAPX);
 7052   match(Set dst (AddI dst src));
 7053   effect(KILL cr);
 7054   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7055 
 7056   format %{ "addl    $dst, $src\t# int" %}
 7057   ins_encode %{
 7058     __ addl($dst$$Register, $src$$constant);
 7059   %}
 7060   ins_pipe( ialu_reg );
 7061 %}
 7062 
 7063 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7064 %{
 7065   predicate(UseAPX);
 7066   match(Set dst (AddI src1 src2));
 7067   effect(KILL cr);
 7068   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7069 
 7070   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7071   ins_encode %{
 7072     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7073   %}
 7074   ins_pipe( ialu_reg );
 7075 %}
 7076 
 7077 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7078 %{
 7079   predicate(UseAPX);
 7080   match(Set dst (AddI (LoadI src1) src2));
 7081   effect(KILL cr);
 7082   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7083 
 7084   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7085   ins_encode %{
 7086     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7087   %}
 7088   ins_pipe( ialu_reg );
 7089 %}
 7090 
 7091 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7092 %{
 7093   predicate(!UseAPX);
 7094   match(Set dst (AddI dst (LoadI src)));
 7095   effect(KILL cr);
 7096   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7097 
 7098   ins_cost(150); // XXX
 7099   format %{ "addl    $dst, $src\t# int" %}
 7100   ins_encode %{
 7101     __ addl($dst$$Register, $src$$Address);
 7102   %}
 7103   ins_pipe(ialu_reg_mem);
 7104 %}
 7105 
 7106 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7107 %{
 7108   predicate(UseAPX);
 7109   match(Set dst (AddI src1 (LoadI src2)));
 7110   effect(KILL cr);
 7111   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7112 
 7113   ins_cost(150);
 7114   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7115   ins_encode %{
 7116     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7117   %}
 7118   ins_pipe(ialu_reg_mem);
 7119 %}
 7120 
 7121 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7122 %{
 7123   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7124   effect(KILL cr);
 7125   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7126 
 7127   ins_cost(150); // XXX
 7128   format %{ "addl    $dst, $src\t# int" %}
 7129   ins_encode %{
 7130     __ addl($dst$$Address, $src$$Register);
 7131   %}
 7132   ins_pipe(ialu_mem_reg);
 7133 %}
 7134 
 7135 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7136 %{
 7137   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7138   effect(KILL cr);
 7139   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7140 
 7141 
 7142   ins_cost(125); // XXX
 7143   format %{ "addl    $dst, $src\t# int" %}
 7144   ins_encode %{
 7145     __ addl($dst$$Address, $src$$constant);
 7146   %}
 7147   ins_pipe(ialu_mem_imm);
 7148 %}
 7149 
 7150 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7151 %{
 7152   predicate(!UseAPX && UseIncDec);
 7153   match(Set dst (AddI dst src));
 7154   effect(KILL cr);
 7155 
 7156   format %{ "incl    $dst\t# int" %}
 7157   ins_encode %{
 7158     __ incrementl($dst$$Register);
 7159   %}
 7160   ins_pipe(ialu_reg);
 7161 %}
 7162 
 7163 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7164 %{
 7165   predicate(UseAPX && UseIncDec);
 7166   match(Set dst (AddI src val));
 7167   effect(KILL cr);
 7168 
 7169   format %{ "eincl    $dst, $src\t# int ndd" %}
 7170   ins_encode %{
 7171     __ eincl($dst$$Register, $src$$Register, false);
 7172   %}
 7173   ins_pipe(ialu_reg);
 7174 %}
 7175 
 7176 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7177 %{
 7178   predicate(UseAPX && UseIncDec);
 7179   match(Set dst (AddI (LoadI src) val));
 7180   effect(KILL cr);
 7181 
 7182   format %{ "eincl    $dst, $src\t# int ndd" %}
 7183   ins_encode %{
 7184     __ eincl($dst$$Register, $src$$Address, false);
 7185   %}
 7186   ins_pipe(ialu_reg);
 7187 %}
 7188 
 7189 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7190 %{
 7191   predicate(UseIncDec);
 7192   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7193   effect(KILL cr);
 7194 
 7195   ins_cost(125); // XXX
 7196   format %{ "incl    $dst\t# int" %}
 7197   ins_encode %{
 7198     __ incrementl($dst$$Address);
 7199   %}
 7200   ins_pipe(ialu_mem_imm);
 7201 %}
 7202 
 7203 // XXX why does that use AddI
 7204 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7205 %{
 7206   predicate(!UseAPX && UseIncDec);
 7207   match(Set dst (AddI dst src));
 7208   effect(KILL cr);
 7209 
 7210   format %{ "decl    $dst\t# int" %}
 7211   ins_encode %{
 7212     __ decrementl($dst$$Register);
 7213   %}
 7214   ins_pipe(ialu_reg);
 7215 %}
 7216 
 7217 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7218 %{
 7219   predicate(UseAPX && UseIncDec);
 7220   match(Set dst (AddI src val));
 7221   effect(KILL cr);
 7222 
 7223   format %{ "edecl    $dst, $src\t# int ndd" %}
 7224   ins_encode %{
 7225     __ edecl($dst$$Register, $src$$Register, false);
 7226   %}
 7227   ins_pipe(ialu_reg);
 7228 %}
 7229 
 7230 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7231 %{
 7232   predicate(UseAPX && UseIncDec);
 7233   match(Set dst (AddI (LoadI src) val));
 7234   effect(KILL cr);
 7235 
 7236   format %{ "edecl    $dst, $src\t# int ndd" %}
 7237   ins_encode %{
 7238     __ edecl($dst$$Register, $src$$Address, false);
 7239   %}
 7240   ins_pipe(ialu_reg);
 7241 %}
 7242 
 7243 // XXX why does that use AddI
 7244 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7245 %{
 7246   predicate(UseIncDec);
 7247   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7248   effect(KILL cr);
 7249 
 7250   ins_cost(125); // XXX
 7251   format %{ "decl    $dst\t# int" %}
 7252   ins_encode %{
 7253     __ decrementl($dst$$Address);
 7254   %}
 7255   ins_pipe(ialu_mem_imm);
 7256 %}
 7257 
 7258 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7259 %{
 7260   predicate(VM_Version::supports_fast_2op_lea());
 7261   match(Set dst (AddI (LShiftI index scale) disp));
 7262 
 7263   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7264   ins_encode %{
 7265     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7266     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7267   %}
 7268   ins_pipe(ialu_reg_reg);
 7269 %}
 7270 
 7271 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7272 %{
 7273   predicate(VM_Version::supports_fast_3op_lea());
 7274   match(Set dst (AddI (AddI base index) disp));
 7275 
 7276   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7277   ins_encode %{
 7278     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7279   %}
 7280   ins_pipe(ialu_reg_reg);
 7281 %}
 7282 
 7283 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7284 %{
 7285   predicate(VM_Version::supports_fast_2op_lea());
 7286   match(Set dst (AddI base (LShiftI index scale)));
 7287 
 7288   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7289   ins_encode %{
 7290     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7291     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7292   %}
 7293   ins_pipe(ialu_reg_reg);
 7294 %}
 7295 
 7296 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7297 %{
 7298   predicate(VM_Version::supports_fast_3op_lea());
 7299   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7300 
 7301   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7302   ins_encode %{
 7303     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7304     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7305   %}
 7306   ins_pipe(ialu_reg_reg);
 7307 %}
 7308 
 7309 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7310 %{
 7311   predicate(!UseAPX);
 7312   match(Set dst (AddL dst src));
 7313   effect(KILL cr);
 7314   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7315 
 7316   format %{ "addq    $dst, $src\t# long" %}
 7317   ins_encode %{
 7318     __ addq($dst$$Register, $src$$Register);
 7319   %}
 7320   ins_pipe(ialu_reg_reg);
 7321 %}
 7322 
 7323 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7324 %{
 7325   predicate(UseAPX);
 7326   match(Set dst (AddL src1 src2));
 7327   effect(KILL cr);
 7328   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7329 
 7330   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7331   ins_encode %{
 7332     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7333   %}
 7334   ins_pipe(ialu_reg_reg);
 7335 %}
 7336 
 7337 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7338 %{
 7339   predicate(!UseAPX);
 7340   match(Set dst (AddL dst src));
 7341   effect(KILL cr);
 7342   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7343 
 7344   format %{ "addq    $dst, $src\t# long" %}
 7345   ins_encode %{
 7346     __ addq($dst$$Register, $src$$constant);
 7347   %}
 7348   ins_pipe( ialu_reg );
 7349 %}
 7350 
 7351 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7352 %{
 7353   predicate(UseAPX);
 7354   match(Set dst (AddL src1 src2));
 7355   effect(KILL cr);
 7356   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7357 
 7358   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7359   ins_encode %{
 7360     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7361   %}
 7362   ins_pipe( ialu_reg );
 7363 %}
 7364 
 7365 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7366 %{
 7367   predicate(UseAPX);
 7368   match(Set dst (AddL (LoadL src1) src2));
 7369   effect(KILL cr);
 7370   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7371 
 7372   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7373   ins_encode %{
 7374     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7375   %}
 7376   ins_pipe( ialu_reg );
 7377 %}
 7378 
 7379 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7380 %{
 7381   predicate(!UseAPX);
 7382   match(Set dst (AddL dst (LoadL src)));
 7383   effect(KILL cr);
 7384   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7385 
 7386   ins_cost(150); // XXX
 7387   format %{ "addq    $dst, $src\t# long" %}
 7388   ins_encode %{
 7389     __ addq($dst$$Register, $src$$Address);
 7390   %}
 7391   ins_pipe(ialu_reg_mem);
 7392 %}
 7393 
 7394 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7395 %{
 7396   predicate(UseAPX);
 7397   match(Set dst (AddL src1 (LoadL src2)));
 7398   effect(KILL cr);
 7399   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7400 
 7401   ins_cost(150);
 7402   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7403   ins_encode %{
 7404     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7405   %}
 7406   ins_pipe(ialu_reg_mem);
 7407 %}
 7408 
 7409 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7410 %{
 7411   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7412   effect(KILL cr);
 7413   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7414 
 7415   ins_cost(150); // XXX
 7416   format %{ "addq    $dst, $src\t# long" %}
 7417   ins_encode %{
 7418     __ addq($dst$$Address, $src$$Register);
 7419   %}
 7420   ins_pipe(ialu_mem_reg);
 7421 %}
 7422 
 7423 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7424 %{
 7425   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7426   effect(KILL cr);
 7427   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7428 
 7429   ins_cost(125); // XXX
 7430   format %{ "addq    $dst, $src\t# long" %}
 7431   ins_encode %{
 7432     __ addq($dst$$Address, $src$$constant);
 7433   %}
 7434   ins_pipe(ialu_mem_imm);
 7435 %}
 7436 
 7437 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7438 %{
 7439   predicate(!UseAPX && UseIncDec);
 7440   match(Set dst (AddL dst src));
 7441   effect(KILL cr);
 7442 
 7443   format %{ "incq    $dst\t# long" %}
 7444   ins_encode %{
 7445     __ incrementq($dst$$Register);
 7446   %}
 7447   ins_pipe(ialu_reg);
 7448 %}
 7449 
 7450 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7451 %{
 7452   predicate(UseAPX && UseIncDec);
 7453   match(Set dst (AddL src val));
 7454   effect(KILL cr);
 7455 
 7456   format %{ "eincq    $dst, $src\t# long ndd" %}
 7457   ins_encode %{
 7458     __ eincq($dst$$Register, $src$$Register, false);
 7459   %}
 7460   ins_pipe(ialu_reg);
 7461 %}
 7462 
 7463 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7464 %{
 7465   predicate(UseAPX && UseIncDec);
 7466   match(Set dst (AddL (LoadL src) val));
 7467   effect(KILL cr);
 7468 
 7469   format %{ "eincq    $dst, $src\t# long ndd" %}
 7470   ins_encode %{
 7471     __ eincq($dst$$Register, $src$$Address, false);
 7472   %}
 7473   ins_pipe(ialu_reg);
 7474 %}
 7475 
 7476 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7477 %{
 7478   predicate(UseIncDec);
 7479   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7480   effect(KILL cr);
 7481 
 7482   ins_cost(125); // XXX
 7483   format %{ "incq    $dst\t# long" %}
 7484   ins_encode %{
 7485     __ incrementq($dst$$Address);
 7486   %}
 7487   ins_pipe(ialu_mem_imm);
 7488 %}
 7489 
 7490 // XXX why does that use AddL
 7491 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7492 %{
 7493   predicate(!UseAPX && UseIncDec);
 7494   match(Set dst (AddL dst src));
 7495   effect(KILL cr);
 7496 
 7497   format %{ "decq    $dst\t# long" %}
 7498   ins_encode %{
 7499     __ decrementq($dst$$Register);
 7500   %}
 7501   ins_pipe(ialu_reg);
 7502 %}
 7503 
 7504 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7505 %{
 7506   predicate(UseAPX && UseIncDec);
 7507   match(Set dst (AddL src val));
 7508   effect(KILL cr);
 7509 
 7510   format %{ "edecq    $dst, $src\t# long ndd" %}
 7511   ins_encode %{
 7512     __ edecq($dst$$Register, $src$$Register, false);
 7513   %}
 7514   ins_pipe(ialu_reg);
 7515 %}
 7516 
 7517 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7518 %{
 7519   predicate(UseAPX && UseIncDec);
 7520   match(Set dst (AddL (LoadL src) val));
 7521   effect(KILL cr);
 7522 
 7523   format %{ "edecq    $dst, $src\t# long ndd" %}
 7524   ins_encode %{
 7525     __ edecq($dst$$Register, $src$$Address, false);
 7526   %}
 7527   ins_pipe(ialu_reg);
 7528 %}
 7529 
 7530 // XXX why does that use AddL
 7531 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7532 %{
 7533   predicate(UseIncDec);
 7534   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7535   effect(KILL cr);
 7536 
 7537   ins_cost(125); // XXX
 7538   format %{ "decq    $dst\t# long" %}
 7539   ins_encode %{
 7540     __ decrementq($dst$$Address);
 7541   %}
 7542   ins_pipe(ialu_mem_imm);
 7543 %}
 7544 
 7545 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7546 %{
 7547   predicate(VM_Version::supports_fast_2op_lea());
 7548   match(Set dst (AddL (LShiftL index scale) disp));
 7549 
 7550   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7551   ins_encode %{
 7552     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7553     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7554   %}
 7555   ins_pipe(ialu_reg_reg);
 7556 %}
 7557 
 7558 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7559 %{
 7560   predicate(VM_Version::supports_fast_3op_lea());
 7561   match(Set dst (AddL (AddL base index) disp));
 7562 
 7563   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7564   ins_encode %{
 7565     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7566   %}
 7567   ins_pipe(ialu_reg_reg);
 7568 %}
 7569 
 7570 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7571 %{
 7572   predicate(VM_Version::supports_fast_2op_lea());
 7573   match(Set dst (AddL base (LShiftL index scale)));
 7574 
 7575   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7576   ins_encode %{
 7577     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7578     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7579   %}
 7580   ins_pipe(ialu_reg_reg);
 7581 %}
 7582 
 7583 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7584 %{
 7585   predicate(VM_Version::supports_fast_3op_lea());
 7586   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7587 
 7588   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7589   ins_encode %{
 7590     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7591     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7592   %}
 7593   ins_pipe(ialu_reg_reg);
 7594 %}
 7595 
 7596 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7597 %{
 7598   match(Set dst (AddP dst src));
 7599   effect(KILL cr);
 7600   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7601 
 7602   format %{ "addq    $dst, $src\t# ptr" %}
 7603   ins_encode %{
 7604     __ addq($dst$$Register, $src$$Register);
 7605   %}
 7606   ins_pipe(ialu_reg_reg);
 7607 %}
 7608 
 7609 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7610 %{
 7611   match(Set dst (AddP dst src));
 7612   effect(KILL cr);
 7613   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7614 
 7615   format %{ "addq    $dst, $src\t# ptr" %}
 7616   ins_encode %{
 7617     __ addq($dst$$Register, $src$$constant);
 7618   %}
 7619   ins_pipe( ialu_reg );
 7620 %}
 7621 
 7622 // XXX addP mem ops ????
 7623 
 7624 instruct checkCastPP(rRegP dst)
 7625 %{
 7626   match(Set dst (CheckCastPP dst));
 7627 
 7628   size(0);
 7629   format %{ "# checkcastPP of $dst" %}
 7630   ins_encode(/* empty encoding */);
 7631   ins_pipe(empty);
 7632 %}
 7633 
 7634 instruct castPP(rRegP dst)
 7635 %{
 7636   match(Set dst (CastPP dst));
 7637 
 7638   size(0);
 7639   format %{ "# castPP of $dst" %}
 7640   ins_encode(/* empty encoding */);
 7641   ins_pipe(empty);
 7642 %}
 7643 
 7644 instruct castII(rRegI dst)
 7645 %{
 7646   predicate(VerifyConstraintCasts == 0);
 7647   match(Set dst (CastII dst));
 7648 
 7649   size(0);
 7650   format %{ "# castII of $dst" %}
 7651   ins_encode(/* empty encoding */);
 7652   ins_cost(0);
 7653   ins_pipe(empty);
 7654 %}
 7655 
 7656 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7657 %{
 7658   predicate(VerifyConstraintCasts > 0);
 7659   match(Set dst (CastII dst));
 7660 
 7661   effect(KILL cr);
 7662   format %{ "# cast_checked_II $dst" %}
 7663   ins_encode %{
 7664     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7665   %}
 7666   ins_pipe(pipe_slow);
 7667 %}
 7668 
 7669 instruct castLL(rRegL dst)
 7670 %{
 7671   predicate(VerifyConstraintCasts == 0);
 7672   match(Set dst (CastLL dst));
 7673 
 7674   size(0);
 7675   format %{ "# castLL of $dst" %}
 7676   ins_encode(/* empty encoding */);
 7677   ins_cost(0);
 7678   ins_pipe(empty);
 7679 %}
 7680 
 7681 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7682 %{
 7683   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7684   match(Set dst (CastLL dst));
 7685 
 7686   effect(KILL cr);
 7687   format %{ "# cast_checked_LL $dst" %}
 7688   ins_encode %{
 7689     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7690   %}
 7691   ins_pipe(pipe_slow);
 7692 %}
 7693 
 7694 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7695 %{
 7696   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7697   match(Set dst (CastLL dst));
 7698 
 7699   effect(KILL cr, TEMP tmp);
 7700   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7701   ins_encode %{
 7702     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7703   %}
 7704   ins_pipe(pipe_slow);
 7705 %}
 7706 
 7707 instruct castFF(regF dst)
 7708 %{
 7709   match(Set dst (CastFF dst));
 7710 
 7711   size(0);
 7712   format %{ "# castFF of $dst" %}
 7713   ins_encode(/* empty encoding */);
 7714   ins_cost(0);
 7715   ins_pipe(empty);
 7716 %}
 7717 
 7718 instruct castHH(regF dst)
 7719 %{
 7720   match(Set dst (CastHH dst));
 7721 
 7722   size(0);
 7723   format %{ "# castHH of $dst" %}
 7724   ins_encode(/* empty encoding */);
 7725   ins_cost(0);
 7726   ins_pipe(empty);
 7727 %}
 7728 
 7729 instruct castDD(regD dst)
 7730 %{
 7731   match(Set dst (CastDD dst));
 7732 
 7733   size(0);
 7734   format %{ "# castDD of $dst" %}
 7735   ins_encode(/* empty encoding */);
 7736   ins_cost(0);
 7737   ins_pipe(empty);
 7738 %}
 7739 
 7740 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7741 instruct compareAndSwapP(rRegI res,
 7742                          memory mem_ptr,
 7743                          rax_RegP oldval, rRegP newval,
 7744                          rFlagsReg cr)
 7745 %{
 7746   predicate(n->as_LoadStore()->barrier_data() == 0);
 7747   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7748   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7749   effect(KILL cr, KILL oldval);
 7750 
 7751   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7752             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7753             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7754   ins_encode %{
 7755     __ lock();
 7756     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7757     __ setcc(Assembler::equal, $res$$Register);
 7758   %}
 7759   ins_pipe( pipe_cmpxchg );
 7760 %}
 7761 
 7762 instruct compareAndSwapL(rRegI res,
 7763                          memory mem_ptr,
 7764                          rax_RegL oldval, rRegL newval,
 7765                          rFlagsReg cr)
 7766 %{
 7767   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7768   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7769   effect(KILL cr, KILL oldval);
 7770 
 7771   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7772             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7773             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7774   ins_encode %{
 7775     __ lock();
 7776     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7777     __ setcc(Assembler::equal, $res$$Register);
 7778   %}
 7779   ins_pipe( pipe_cmpxchg );
 7780 %}
 7781 
 7782 instruct compareAndSwapI(rRegI res,
 7783                          memory mem_ptr,
 7784                          rax_RegI oldval, rRegI newval,
 7785                          rFlagsReg cr)
 7786 %{
 7787   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7788   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7789   effect(KILL cr, KILL oldval);
 7790 
 7791   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7792             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7793             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7794   ins_encode %{
 7795     __ lock();
 7796     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7797     __ setcc(Assembler::equal, $res$$Register);
 7798   %}
 7799   ins_pipe( pipe_cmpxchg );
 7800 %}
 7801 
 7802 instruct compareAndSwapB(rRegI res,
 7803                          memory mem_ptr,
 7804                          rax_RegI oldval, rRegI newval,
 7805                          rFlagsReg cr)
 7806 %{
 7807   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7808   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7809   effect(KILL cr, KILL oldval);
 7810 
 7811   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7812             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7813             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7814   ins_encode %{
 7815     __ lock();
 7816     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7817     __ setcc(Assembler::equal, $res$$Register);
 7818   %}
 7819   ins_pipe( pipe_cmpxchg );
 7820 %}
 7821 
 7822 instruct compareAndSwapS(rRegI res,
 7823                          memory mem_ptr,
 7824                          rax_RegI oldval, rRegI newval,
 7825                          rFlagsReg cr)
 7826 %{
 7827   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7828   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7829   effect(KILL cr, KILL oldval);
 7830 
 7831   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7832             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7833             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7834   ins_encode %{
 7835     __ lock();
 7836     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7837     __ setcc(Assembler::equal, $res$$Register);
 7838   %}
 7839   ins_pipe( pipe_cmpxchg );
 7840 %}
 7841 
 7842 instruct compareAndSwapN(rRegI res,
 7843                           memory mem_ptr,
 7844                           rax_RegN oldval, rRegN newval,
 7845                           rFlagsReg cr) %{
 7846   predicate(n->as_LoadStore()->barrier_data() == 0);
 7847   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7848   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7849   effect(KILL cr, KILL oldval);
 7850 
 7851   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7852             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7853             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7854   ins_encode %{
 7855     __ lock();
 7856     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7857     __ setcc(Assembler::equal, $res$$Register);
 7858   %}
 7859   ins_pipe( pipe_cmpxchg );
 7860 %}
 7861 
 7862 instruct compareAndExchangeB(
 7863                          memory mem_ptr,
 7864                          rax_RegI oldval, rRegI newval,
 7865                          rFlagsReg cr)
 7866 %{
 7867   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7868   effect(KILL cr);
 7869 
 7870   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7871             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7872   ins_encode %{
 7873     __ lock();
 7874     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7875   %}
 7876   ins_pipe( pipe_cmpxchg );
 7877 %}
 7878 
 7879 instruct compareAndExchangeS(
 7880                          memory mem_ptr,
 7881                          rax_RegI oldval, rRegI newval,
 7882                          rFlagsReg cr)
 7883 %{
 7884   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7885   effect(KILL cr);
 7886 
 7887   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7888             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7889   ins_encode %{
 7890     __ lock();
 7891     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7892   %}
 7893   ins_pipe( pipe_cmpxchg );
 7894 %}
 7895 
 7896 instruct compareAndExchangeI(
 7897                          memory mem_ptr,
 7898                          rax_RegI oldval, rRegI newval,
 7899                          rFlagsReg cr)
 7900 %{
 7901   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7902   effect(KILL cr);
 7903 
 7904   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7905             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7906   ins_encode %{
 7907     __ lock();
 7908     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7909   %}
 7910   ins_pipe( pipe_cmpxchg );
 7911 %}
 7912 
 7913 instruct compareAndExchangeL(
 7914                          memory mem_ptr,
 7915                          rax_RegL oldval, rRegL newval,
 7916                          rFlagsReg cr)
 7917 %{
 7918   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7919   effect(KILL cr);
 7920 
 7921   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7922             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7923   ins_encode %{
 7924     __ lock();
 7925     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7926   %}
 7927   ins_pipe( pipe_cmpxchg );
 7928 %}
 7929 
 7930 instruct compareAndExchangeN(
 7931                           memory mem_ptr,
 7932                           rax_RegN oldval, rRegN newval,
 7933                           rFlagsReg cr) %{
 7934   predicate(n->as_LoadStore()->barrier_data() == 0);
 7935   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7936   effect(KILL cr);
 7937 
 7938   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7939             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7940   ins_encode %{
 7941     __ lock();
 7942     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7943   %}
 7944   ins_pipe( pipe_cmpxchg );
 7945 %}
 7946 
 7947 instruct compareAndExchangeP(
 7948                          memory mem_ptr,
 7949                          rax_RegP oldval, rRegP newval,
 7950                          rFlagsReg cr)
 7951 %{
 7952   predicate(n->as_LoadStore()->barrier_data() == 0);
 7953   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7954   effect(KILL cr);
 7955 
 7956   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7957             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7958   ins_encode %{
 7959     __ lock();
 7960     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7961   %}
 7962   ins_pipe( pipe_cmpxchg );
 7963 %}
 7964 
 7965 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7966   predicate(n->as_LoadStore()->result_not_used());
 7967   match(Set dummy (GetAndAddB mem add));
 7968   effect(KILL cr);
 7969   format %{ "addb_lock   $mem, $add" %}
 7970   ins_encode %{
 7971     __ lock();
 7972     __ addb($mem$$Address, $add$$Register);
 7973   %}
 7974   ins_pipe(pipe_cmpxchg);
 7975 %}
 7976 
 7977 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7978   predicate(n->as_LoadStore()->result_not_used());
 7979   match(Set dummy (GetAndAddB mem add));
 7980   effect(KILL cr);
 7981   format %{ "addb_lock   $mem, $add" %}
 7982   ins_encode %{
 7983     __ lock();
 7984     __ addb($mem$$Address, $add$$constant);
 7985   %}
 7986   ins_pipe(pipe_cmpxchg);
 7987 %}
 7988 
 7989 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 7990   predicate(!n->as_LoadStore()->result_not_used());
 7991   match(Set newval (GetAndAddB mem newval));
 7992   effect(KILL cr);
 7993   format %{ "xaddb_lock  $mem, $newval" %}
 7994   ins_encode %{
 7995     __ lock();
 7996     __ xaddb($mem$$Address, $newval$$Register);
 7997   %}
 7998   ins_pipe(pipe_cmpxchg);
 7999 %}
 8000 
 8001 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8002   predicate(n->as_LoadStore()->result_not_used());
 8003   match(Set dummy (GetAndAddS mem add));
 8004   effect(KILL cr);
 8005   format %{ "addw_lock   $mem, $add" %}
 8006   ins_encode %{
 8007     __ lock();
 8008     __ addw($mem$$Address, $add$$Register);
 8009   %}
 8010   ins_pipe(pipe_cmpxchg);
 8011 %}
 8012 
 8013 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8014   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 8015   match(Set dummy (GetAndAddS mem add));
 8016   effect(KILL cr);
 8017   format %{ "addw_lock   $mem, $add" %}
 8018   ins_encode %{
 8019     __ lock();
 8020     __ addw($mem$$Address, $add$$constant);
 8021   %}
 8022   ins_pipe(pipe_cmpxchg);
 8023 %}
 8024 
 8025 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8026   predicate(!n->as_LoadStore()->result_not_used());
 8027   match(Set newval (GetAndAddS mem newval));
 8028   effect(KILL cr);
 8029   format %{ "xaddw_lock  $mem, $newval" %}
 8030   ins_encode %{
 8031     __ lock();
 8032     __ xaddw($mem$$Address, $newval$$Register);
 8033   %}
 8034   ins_pipe(pipe_cmpxchg);
 8035 %}
 8036 
 8037 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8038   predicate(n->as_LoadStore()->result_not_used());
 8039   match(Set dummy (GetAndAddI mem add));
 8040   effect(KILL cr);
 8041   format %{ "addl_lock   $mem, $add" %}
 8042   ins_encode %{
 8043     __ lock();
 8044     __ addl($mem$$Address, $add$$Register);
 8045   %}
 8046   ins_pipe(pipe_cmpxchg);
 8047 %}
 8048 
 8049 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8050   predicate(n->as_LoadStore()->result_not_used());
 8051   match(Set dummy (GetAndAddI mem add));
 8052   effect(KILL cr);
 8053   format %{ "addl_lock   $mem, $add" %}
 8054   ins_encode %{
 8055     __ lock();
 8056     __ addl($mem$$Address, $add$$constant);
 8057   %}
 8058   ins_pipe(pipe_cmpxchg);
 8059 %}
 8060 
 8061 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8062   predicate(!n->as_LoadStore()->result_not_used());
 8063   match(Set newval (GetAndAddI mem newval));
 8064   effect(KILL cr);
 8065   format %{ "xaddl_lock  $mem, $newval" %}
 8066   ins_encode %{
 8067     __ lock();
 8068     __ xaddl($mem$$Address, $newval$$Register);
 8069   %}
 8070   ins_pipe(pipe_cmpxchg);
 8071 %}
 8072 
 8073 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8074   predicate(n->as_LoadStore()->result_not_used());
 8075   match(Set dummy (GetAndAddL mem add));
 8076   effect(KILL cr);
 8077   format %{ "addq_lock   $mem, $add" %}
 8078   ins_encode %{
 8079     __ lock();
 8080     __ addq($mem$$Address, $add$$Register);
 8081   %}
 8082   ins_pipe(pipe_cmpxchg);
 8083 %}
 8084 
 8085 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8086   predicate(n->as_LoadStore()->result_not_used());
 8087   match(Set dummy (GetAndAddL mem add));
 8088   effect(KILL cr);
 8089   format %{ "addq_lock   $mem, $add" %}
 8090   ins_encode %{
 8091     __ lock();
 8092     __ addq($mem$$Address, $add$$constant);
 8093   %}
 8094   ins_pipe(pipe_cmpxchg);
 8095 %}
 8096 
 8097 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8098   predicate(!n->as_LoadStore()->result_not_used());
 8099   match(Set newval (GetAndAddL mem newval));
 8100   effect(KILL cr);
 8101   format %{ "xaddq_lock  $mem, $newval" %}
 8102   ins_encode %{
 8103     __ lock();
 8104     __ xaddq($mem$$Address, $newval$$Register);
 8105   %}
 8106   ins_pipe(pipe_cmpxchg);
 8107 %}
 8108 
 8109 instruct xchgB( memory mem, rRegI newval) %{
 8110   match(Set newval (GetAndSetB mem newval));
 8111   format %{ "XCHGB  $newval,[$mem]" %}
 8112   ins_encode %{
 8113     __ xchgb($newval$$Register, $mem$$Address);
 8114   %}
 8115   ins_pipe( pipe_cmpxchg );
 8116 %}
 8117 
 8118 instruct xchgS( memory mem, rRegI newval) %{
 8119   match(Set newval (GetAndSetS mem newval));
 8120   format %{ "XCHGW  $newval,[$mem]" %}
 8121   ins_encode %{
 8122     __ xchgw($newval$$Register, $mem$$Address);
 8123   %}
 8124   ins_pipe( pipe_cmpxchg );
 8125 %}
 8126 
 8127 instruct xchgI( memory mem, rRegI newval) %{
 8128   match(Set newval (GetAndSetI mem newval));
 8129   format %{ "XCHGL  $newval,[$mem]" %}
 8130   ins_encode %{
 8131     __ xchgl($newval$$Register, $mem$$Address);
 8132   %}
 8133   ins_pipe( pipe_cmpxchg );
 8134 %}
 8135 
 8136 instruct xchgL( memory mem, rRegL newval) %{
 8137   match(Set newval (GetAndSetL mem newval));
 8138   format %{ "XCHGL  $newval,[$mem]" %}
 8139   ins_encode %{
 8140     __ xchgq($newval$$Register, $mem$$Address);
 8141   %}
 8142   ins_pipe( pipe_cmpxchg );
 8143 %}
 8144 
 8145 instruct xchgP( memory mem, rRegP newval) %{
 8146   match(Set newval (GetAndSetP mem newval));
 8147   predicate(n->as_LoadStore()->barrier_data() == 0);
 8148   format %{ "XCHGQ  $newval,[$mem]" %}
 8149   ins_encode %{
 8150     __ xchgq($newval$$Register, $mem$$Address);
 8151   %}
 8152   ins_pipe( pipe_cmpxchg );
 8153 %}
 8154 
 8155 instruct xchgN( memory mem, rRegN newval) %{
 8156   predicate(n->as_LoadStore()->barrier_data() == 0);
 8157   match(Set newval (GetAndSetN mem newval));
 8158   format %{ "XCHGL  $newval,$mem]" %}
 8159   ins_encode %{
 8160     __ xchgl($newval$$Register, $mem$$Address);
 8161   %}
 8162   ins_pipe( pipe_cmpxchg );
 8163 %}
 8164 
 8165 //----------Abs Instructions-------------------------------------------
 8166 
 8167 // Integer Absolute Instructions
 8168 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8169 %{
 8170   match(Set dst (AbsI src));
 8171   effect(TEMP dst, KILL cr);
 8172   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8173             "subl    $dst, $src\n\t"
 8174             "cmovll  $dst, $src" %}
 8175   ins_encode %{
 8176     __ xorl($dst$$Register, $dst$$Register);
 8177     __ subl($dst$$Register, $src$$Register);
 8178     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8179   %}
 8180 
 8181   ins_pipe(ialu_reg_reg);
 8182 %}
 8183 
 8184 // Long Absolute Instructions
 8185 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8186 %{
 8187   match(Set dst (AbsL src));
 8188   effect(TEMP dst, KILL cr);
 8189   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8190             "subq    $dst, $src\n\t"
 8191             "cmovlq  $dst, $src" %}
 8192   ins_encode %{
 8193     __ xorl($dst$$Register, $dst$$Register);
 8194     __ subq($dst$$Register, $src$$Register);
 8195     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8196   %}
 8197 
 8198   ins_pipe(ialu_reg_reg);
 8199 %}
 8200 
 8201 //----------Subtraction Instructions-------------------------------------------
 8202 
 8203 // Integer Subtraction Instructions
 8204 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8205 %{
 8206   predicate(!UseAPX);
 8207   match(Set dst (SubI dst src));
 8208   effect(KILL cr);
 8209   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8210 
 8211   format %{ "subl    $dst, $src\t# int" %}
 8212   ins_encode %{
 8213     __ subl($dst$$Register, $src$$Register);
 8214   %}
 8215   ins_pipe(ialu_reg_reg);
 8216 %}
 8217 
 8218 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8219 %{
 8220   predicate(UseAPX);
 8221   match(Set dst (SubI src1 src2));
 8222   effect(KILL cr);
 8223   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8224 
 8225   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8226   ins_encode %{
 8227     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8228   %}
 8229   ins_pipe(ialu_reg_reg);
 8230 %}
 8231 
 8232 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8233 %{
 8234   predicate(UseAPX);
 8235   match(Set dst (SubI src1 src2));
 8236   effect(KILL cr);
 8237   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8238 
 8239   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8240   ins_encode %{
 8241     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8242   %}
 8243   ins_pipe(ialu_reg_reg);
 8244 %}
 8245 
 8246 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8247 %{
 8248   predicate(UseAPX);
 8249   match(Set dst (SubI (LoadI src1) src2));
 8250   effect(KILL cr);
 8251   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8252 
 8253   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8254   ins_encode %{
 8255     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8256   %}
 8257   ins_pipe(ialu_reg_reg);
 8258 %}
 8259 
 8260 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8261 %{
 8262   predicate(!UseAPX);
 8263   match(Set dst (SubI dst (LoadI src)));
 8264   effect(KILL cr);
 8265   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8266 
 8267   ins_cost(150);
 8268   format %{ "subl    $dst, $src\t# int" %}
 8269   ins_encode %{
 8270     __ subl($dst$$Register, $src$$Address);
 8271   %}
 8272   ins_pipe(ialu_reg_mem);
 8273 %}
 8274 
 8275 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8276 %{
 8277   predicate(UseAPX);
 8278   match(Set dst (SubI src1 (LoadI src2)));
 8279   effect(KILL cr);
 8280   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8281 
 8282   ins_cost(150);
 8283   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8284   ins_encode %{
 8285     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8286   %}
 8287   ins_pipe(ialu_reg_mem);
 8288 %}
 8289 
 8290 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8291 %{
 8292   predicate(UseAPX);
 8293   match(Set dst (SubI (LoadI src1) src2));
 8294   effect(KILL cr);
 8295   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8296 
 8297   ins_cost(150);
 8298   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8299   ins_encode %{
 8300     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8301   %}
 8302   ins_pipe(ialu_reg_mem);
 8303 %}
 8304 
 8305 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8306 %{
 8307   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8308   effect(KILL cr);
 8309   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8310 
 8311   ins_cost(150);
 8312   format %{ "subl    $dst, $src\t# int" %}
 8313   ins_encode %{
 8314     __ subl($dst$$Address, $src$$Register);
 8315   %}
 8316   ins_pipe(ialu_mem_reg);
 8317 %}
 8318 
 8319 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8320 %{
 8321   predicate(!UseAPX);
 8322   match(Set dst (SubL dst src));
 8323   effect(KILL cr);
 8324   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8325 
 8326   format %{ "subq    $dst, $src\t# long" %}
 8327   ins_encode %{
 8328     __ subq($dst$$Register, $src$$Register);
 8329   %}
 8330   ins_pipe(ialu_reg_reg);
 8331 %}
 8332 
 8333 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8334 %{
 8335   predicate(UseAPX);
 8336   match(Set dst (SubL src1 src2));
 8337   effect(KILL cr);
 8338   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8339 
 8340   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8341   ins_encode %{
 8342     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8343   %}
 8344   ins_pipe(ialu_reg_reg);
 8345 %}
 8346 
 8347 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8348 %{
 8349   predicate(UseAPX);
 8350   match(Set dst (SubL src1 src2));
 8351   effect(KILL cr);
 8352   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8353 
 8354   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8355   ins_encode %{
 8356     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8357   %}
 8358   ins_pipe(ialu_reg_reg);
 8359 %}
 8360 
 8361 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8362 %{
 8363   predicate(UseAPX);
 8364   match(Set dst (SubL (LoadL src1) src2));
 8365   effect(KILL cr);
 8366   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8367 
 8368   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8369   ins_encode %{
 8370     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8371   %}
 8372   ins_pipe(ialu_reg_reg);
 8373 %}
 8374 
 8375 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8376 %{
 8377   predicate(!UseAPX);
 8378   match(Set dst (SubL dst (LoadL src)));
 8379   effect(KILL cr);
 8380   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8381 
 8382   ins_cost(150);
 8383   format %{ "subq    $dst, $src\t# long" %}
 8384   ins_encode %{
 8385     __ subq($dst$$Register, $src$$Address);
 8386   %}
 8387   ins_pipe(ialu_reg_mem);
 8388 %}
 8389 
 8390 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8391 %{
 8392   predicate(UseAPX);
 8393   match(Set dst (SubL src1 (LoadL src2)));
 8394   effect(KILL cr);
 8395   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8396 
 8397   ins_cost(150);
 8398   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8399   ins_encode %{
 8400     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8401   %}
 8402   ins_pipe(ialu_reg_mem);
 8403 %}
 8404 
 8405 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8406 %{
 8407   predicate(UseAPX);
 8408   match(Set dst (SubL (LoadL src1) src2));
 8409   effect(KILL cr);
 8410   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8411 
 8412   ins_cost(150);
 8413   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8414   ins_encode %{
 8415     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8416   %}
 8417   ins_pipe(ialu_reg_mem);
 8418 %}
 8419 
 8420 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8421 %{
 8422   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8423   effect(KILL cr);
 8424   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8425 
 8426   ins_cost(150);
 8427   format %{ "subq    $dst, $src\t# long" %}
 8428   ins_encode %{
 8429     __ subq($dst$$Address, $src$$Register);
 8430   %}
 8431   ins_pipe(ialu_mem_reg);
 8432 %}
 8433 
 8434 // Subtract from a pointer
 8435 // XXX hmpf???
 8436 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8437 %{
 8438   match(Set dst (AddP dst (SubI zero src)));
 8439   effect(KILL cr);
 8440 
 8441   format %{ "subq    $dst, $src\t# ptr - int" %}
 8442   ins_encode %{
 8443     __ subq($dst$$Register, $src$$Register);
 8444   %}
 8445   ins_pipe(ialu_reg_reg);
 8446 %}
 8447 
 8448 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8449 %{
 8450   predicate(!UseAPX);
 8451   match(Set dst (SubI zero dst));
 8452   effect(KILL cr);
 8453   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8454 
 8455   format %{ "negl    $dst\t# int" %}
 8456   ins_encode %{
 8457     __ negl($dst$$Register);
 8458   %}
 8459   ins_pipe(ialu_reg);
 8460 %}
 8461 
 8462 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8463 %{
 8464   predicate(UseAPX);
 8465   match(Set dst (SubI zero src));
 8466   effect(KILL cr);
 8467   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8468 
 8469   format %{ "enegl    $dst, $src\t# int ndd" %}
 8470   ins_encode %{
 8471     __ enegl($dst$$Register, $src$$Register, false);
 8472   %}
 8473   ins_pipe(ialu_reg);
 8474 %}
 8475 
 8476 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8477 %{
 8478   predicate(!UseAPX);
 8479   match(Set dst (NegI dst));
 8480   effect(KILL cr);
 8481   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8482 
 8483   format %{ "negl    $dst\t# int" %}
 8484   ins_encode %{
 8485     __ negl($dst$$Register);
 8486   %}
 8487   ins_pipe(ialu_reg);
 8488 %}
 8489 
 8490 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8491 %{
 8492   predicate(UseAPX);
 8493   match(Set dst (NegI src));
 8494   effect(KILL cr);
 8495   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8496 
 8497   format %{ "enegl    $dst, $src\t# int ndd" %}
 8498   ins_encode %{
 8499     __ enegl($dst$$Register, $src$$Register, false);
 8500   %}
 8501   ins_pipe(ialu_reg);
 8502 %}
 8503 
 8504 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8505 %{
 8506   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8507   effect(KILL cr);
 8508   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8509 
 8510   format %{ "negl    $dst\t# int" %}
 8511   ins_encode %{
 8512     __ negl($dst$$Address);
 8513   %}
 8514   ins_pipe(ialu_reg);
 8515 %}
 8516 
 8517 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8518 %{
 8519   predicate(!UseAPX);
 8520   match(Set dst (SubL zero dst));
 8521   effect(KILL cr);
 8522   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8523 
 8524   format %{ "negq    $dst\t# long" %}
 8525   ins_encode %{
 8526     __ negq($dst$$Register);
 8527   %}
 8528   ins_pipe(ialu_reg);
 8529 %}
 8530 
 8531 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8532 %{
 8533   predicate(UseAPX);
 8534   match(Set dst (SubL zero src));
 8535   effect(KILL cr);
 8536   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8537 
 8538   format %{ "enegq    $dst, $src\t# long ndd" %}
 8539   ins_encode %{
 8540     __ enegq($dst$$Register, $src$$Register, false);
 8541   %}
 8542   ins_pipe(ialu_reg);
 8543 %}
 8544 
 8545 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8546 %{
 8547   predicate(!UseAPX);
 8548   match(Set dst (NegL dst));
 8549   effect(KILL cr);
 8550   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8551 
 8552   format %{ "negq    $dst\t# int" %}
 8553   ins_encode %{
 8554     __ negq($dst$$Register);
 8555   %}
 8556   ins_pipe(ialu_reg);
 8557 %}
 8558 
 8559 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8560 %{
 8561   predicate(UseAPX);
 8562   match(Set dst (NegL src));
 8563   effect(KILL cr);
 8564   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8565 
 8566   format %{ "enegq    $dst, $src\t# long ndd" %}
 8567   ins_encode %{
 8568     __ enegq($dst$$Register, $src$$Register, false);
 8569   %}
 8570   ins_pipe(ialu_reg);
 8571 %}
 8572 
 8573 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8574 %{
 8575   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8576   effect(KILL cr);
 8577   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8578 
 8579   format %{ "negq    $dst\t# long" %}
 8580   ins_encode %{
 8581     __ negq($dst$$Address);
 8582   %}
 8583   ins_pipe(ialu_reg);
 8584 %}
 8585 
 8586 //----------Multiplication/Division Instructions-------------------------------
 8587 // Integer Multiplication Instructions
 8588 // Multiply Register
 8589 
 8590 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8591 %{
 8592   predicate(!UseAPX);
 8593   match(Set dst (MulI dst src));
 8594   effect(KILL cr);
 8595 
 8596   ins_cost(300);
 8597   format %{ "imull   $dst, $src\t# int" %}
 8598   ins_encode %{
 8599     __ imull($dst$$Register, $src$$Register);
 8600   %}
 8601   ins_pipe(ialu_reg_reg_alu0);
 8602 %}
 8603 
 8604 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8605 %{
 8606   predicate(UseAPX);
 8607   match(Set dst (MulI src1 src2));
 8608   effect(KILL cr);
 8609 
 8610   ins_cost(300);
 8611   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8612   ins_encode %{
 8613     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8614   %}
 8615   ins_pipe(ialu_reg_reg_alu0);
 8616 %}
 8617 
 8618 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8619 %{
 8620   match(Set dst (MulI src imm));
 8621   effect(KILL cr);
 8622 
 8623   ins_cost(300);
 8624   format %{ "imull   $dst, $src, $imm\t# int" %}
 8625   ins_encode %{
 8626     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8627   %}
 8628   ins_pipe(ialu_reg_reg_alu0);
 8629 %}
 8630 
 8631 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8632 %{
 8633   predicate(!UseAPX);
 8634   match(Set dst (MulI dst (LoadI src)));
 8635   effect(KILL cr);
 8636 
 8637   ins_cost(350);
 8638   format %{ "imull   $dst, $src\t# int" %}
 8639   ins_encode %{
 8640     __ imull($dst$$Register, $src$$Address);
 8641   %}
 8642   ins_pipe(ialu_reg_mem_alu0);
 8643 %}
 8644 
 8645 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8646 %{
 8647   predicate(UseAPX);
 8648   match(Set dst (MulI src1 (LoadI src2)));
 8649   effect(KILL cr);
 8650 
 8651   ins_cost(350);
 8652   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8653   ins_encode %{
 8654     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8655   %}
 8656   ins_pipe(ialu_reg_mem_alu0);
 8657 %}
 8658 
 8659 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8660 %{
 8661   match(Set dst (MulI (LoadI src) imm));
 8662   effect(KILL cr);
 8663 
 8664   ins_cost(300);
 8665   format %{ "imull   $dst, $src, $imm\t# int" %}
 8666   ins_encode %{
 8667     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8668   %}
 8669   ins_pipe(ialu_reg_mem_alu0);
 8670 %}
 8671 
 8672 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8673 %{
 8674   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8675   effect(KILL cr, KILL src2);
 8676 
 8677   expand %{ mulI_rReg(dst, src1, cr);
 8678            mulI_rReg(src2, src3, cr);
 8679            addI_rReg(dst, src2, cr); %}
 8680 %}
 8681 
 8682 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8683 %{
 8684   predicate(!UseAPX);
 8685   match(Set dst (MulL dst src));
 8686   effect(KILL cr);
 8687 
 8688   ins_cost(300);
 8689   format %{ "imulq   $dst, $src\t# long" %}
 8690   ins_encode %{
 8691     __ imulq($dst$$Register, $src$$Register);
 8692   %}
 8693   ins_pipe(ialu_reg_reg_alu0);
 8694 %}
 8695 
 8696 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8697 %{
 8698   predicate(UseAPX);
 8699   match(Set dst (MulL src1 src2));
 8700   effect(KILL cr);
 8701 
 8702   ins_cost(300);
 8703   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8704   ins_encode %{
 8705     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8706   %}
 8707   ins_pipe(ialu_reg_reg_alu0);
 8708 %}
 8709 
 8710 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8711 %{
 8712   match(Set dst (MulL src imm));
 8713   effect(KILL cr);
 8714 
 8715   ins_cost(300);
 8716   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8717   ins_encode %{
 8718     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8719   %}
 8720   ins_pipe(ialu_reg_reg_alu0);
 8721 %}
 8722 
 8723 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8724 %{
 8725   predicate(!UseAPX);
 8726   match(Set dst (MulL dst (LoadL src)));
 8727   effect(KILL cr);
 8728 
 8729   ins_cost(350);
 8730   format %{ "imulq   $dst, $src\t# long" %}
 8731   ins_encode %{
 8732     __ imulq($dst$$Register, $src$$Address);
 8733   %}
 8734   ins_pipe(ialu_reg_mem_alu0);
 8735 %}
 8736 
 8737 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8738 %{
 8739   predicate(UseAPX);
 8740   match(Set dst (MulL src1 (LoadL src2)));
 8741   effect(KILL cr);
 8742 
 8743   ins_cost(350);
 8744   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8745   ins_encode %{
 8746     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8747   %}
 8748   ins_pipe(ialu_reg_mem_alu0);
 8749 %}
 8750 
 8751 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8752 %{
 8753   match(Set dst (MulL (LoadL src) imm));
 8754   effect(KILL cr);
 8755 
 8756   ins_cost(300);
 8757   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8758   ins_encode %{
 8759     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8760   %}
 8761   ins_pipe(ialu_reg_mem_alu0);
 8762 %}
 8763 
 8764 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8765 %{
 8766   match(Set dst (MulHiL src rax));
 8767   effect(USE_KILL rax, KILL cr);
 8768 
 8769   ins_cost(300);
 8770   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8771   ins_encode %{
 8772     __ imulq($src$$Register);
 8773   %}
 8774   ins_pipe(ialu_reg_reg_alu0);
 8775 %}
 8776 
 8777 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8778 %{
 8779   match(Set dst (UMulHiL src rax));
 8780   effect(USE_KILL rax, KILL cr);
 8781 
 8782   ins_cost(300);
 8783   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8784   ins_encode %{
 8785     __ mulq($src$$Register);
 8786   %}
 8787   ins_pipe(ialu_reg_reg_alu0);
 8788 %}
 8789 
 8790 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8791                    rFlagsReg cr)
 8792 %{
 8793   match(Set rax (DivI rax div));
 8794   effect(KILL rdx, KILL cr);
 8795 
 8796   ins_cost(30*100+10*100); // XXX
 8797   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8798             "jne,s   normal\n\t"
 8799             "xorl    rdx, rdx\n\t"
 8800             "cmpl    $div, -1\n\t"
 8801             "je,s    done\n"
 8802     "normal: cdql\n\t"
 8803             "idivl   $div\n"
 8804     "done:"        %}
 8805   ins_encode(cdql_enc(div));
 8806   ins_pipe(ialu_reg_reg_alu0);
 8807 %}
 8808 
 8809 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8810                    rFlagsReg cr)
 8811 %{
 8812   match(Set rax (DivL rax div));
 8813   effect(KILL rdx, KILL cr);
 8814 
 8815   ins_cost(30*100+10*100); // XXX
 8816   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8817             "cmpq    rax, rdx\n\t"
 8818             "jne,s   normal\n\t"
 8819             "xorl    rdx, rdx\n\t"
 8820             "cmpq    $div, -1\n\t"
 8821             "je,s    done\n"
 8822     "normal: cdqq\n\t"
 8823             "idivq   $div\n"
 8824     "done:"        %}
 8825   ins_encode(cdqq_enc(div));
 8826   ins_pipe(ialu_reg_reg_alu0);
 8827 %}
 8828 
 8829 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8830 %{
 8831   match(Set rax (UDivI rax div));
 8832   effect(KILL rdx, KILL cr);
 8833 
 8834   ins_cost(300);
 8835   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8836   ins_encode %{
 8837     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8838   %}
 8839   ins_pipe(ialu_reg_reg_alu0);
 8840 %}
 8841 
 8842 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8843 %{
 8844   match(Set rax (UDivL rax div));
 8845   effect(KILL rdx, KILL cr);
 8846 
 8847   ins_cost(300);
 8848   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8849   ins_encode %{
 8850      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8851   %}
 8852   ins_pipe(ialu_reg_reg_alu0);
 8853 %}
 8854 
 8855 // Integer DIVMOD with Register, both quotient and mod results
 8856 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8857                              rFlagsReg cr)
 8858 %{
 8859   match(DivModI rax div);
 8860   effect(KILL cr);
 8861 
 8862   ins_cost(30*100+10*100); // XXX
 8863   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8864             "jne,s   normal\n\t"
 8865             "xorl    rdx, rdx\n\t"
 8866             "cmpl    $div, -1\n\t"
 8867             "je,s    done\n"
 8868     "normal: cdql\n\t"
 8869             "idivl   $div\n"
 8870     "done:"        %}
 8871   ins_encode(cdql_enc(div));
 8872   ins_pipe(pipe_slow);
 8873 %}
 8874 
 8875 // Long DIVMOD with Register, both quotient and mod results
 8876 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8877                              rFlagsReg cr)
 8878 %{
 8879   match(DivModL rax div);
 8880   effect(KILL cr);
 8881 
 8882   ins_cost(30*100+10*100); // XXX
 8883   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8884             "cmpq    rax, rdx\n\t"
 8885             "jne,s   normal\n\t"
 8886             "xorl    rdx, rdx\n\t"
 8887             "cmpq    $div, -1\n\t"
 8888             "je,s    done\n"
 8889     "normal: cdqq\n\t"
 8890             "idivq   $div\n"
 8891     "done:"        %}
 8892   ins_encode(cdqq_enc(div));
 8893   ins_pipe(pipe_slow);
 8894 %}
 8895 
 8896 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8897 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8898                               no_rax_rdx_RegI div, rFlagsReg cr)
 8899 %{
 8900   match(UDivModI rax div);
 8901   effect(TEMP tmp, KILL cr);
 8902 
 8903   ins_cost(300);
 8904   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8905             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8906           %}
 8907   ins_encode %{
 8908     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8909   %}
 8910   ins_pipe(pipe_slow);
 8911 %}
 8912 
 8913 // Unsigned long DIVMOD with Register, both quotient and mod results
 8914 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8915                               no_rax_rdx_RegL div, rFlagsReg cr)
 8916 %{
 8917   match(UDivModL rax div);
 8918   effect(TEMP tmp, KILL cr);
 8919 
 8920   ins_cost(300);
 8921   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8922             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8923           %}
 8924   ins_encode %{
 8925     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8926   %}
 8927   ins_pipe(pipe_slow);
 8928 %}
 8929 
 8930 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8931                    rFlagsReg cr)
 8932 %{
 8933   match(Set rdx (ModI rax div));
 8934   effect(KILL rax, KILL cr);
 8935 
 8936   ins_cost(300); // XXX
 8937   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8938             "jne,s   normal\n\t"
 8939             "xorl    rdx, rdx\n\t"
 8940             "cmpl    $div, -1\n\t"
 8941             "je,s    done\n"
 8942     "normal: cdql\n\t"
 8943             "idivl   $div\n"
 8944     "done:"        %}
 8945   ins_encode(cdql_enc(div));
 8946   ins_pipe(ialu_reg_reg_alu0);
 8947 %}
 8948 
 8949 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8950                    rFlagsReg cr)
 8951 %{
 8952   match(Set rdx (ModL rax div));
 8953   effect(KILL rax, KILL cr);
 8954 
 8955   ins_cost(300); // XXX
 8956   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8957             "cmpq    rax, rdx\n\t"
 8958             "jne,s   normal\n\t"
 8959             "xorl    rdx, rdx\n\t"
 8960             "cmpq    $div, -1\n\t"
 8961             "je,s    done\n"
 8962     "normal: cdqq\n\t"
 8963             "idivq   $div\n"
 8964     "done:"        %}
 8965   ins_encode(cdqq_enc(div));
 8966   ins_pipe(ialu_reg_reg_alu0);
 8967 %}
 8968 
 8969 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8970 %{
 8971   match(Set rdx (UModI rax div));
 8972   effect(KILL rax, KILL cr);
 8973 
 8974   ins_cost(300);
 8975   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8976   ins_encode %{
 8977     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8978   %}
 8979   ins_pipe(ialu_reg_reg_alu0);
 8980 %}
 8981 
 8982 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8983 %{
 8984   match(Set rdx (UModL rax div));
 8985   effect(KILL rax, KILL cr);
 8986 
 8987   ins_cost(300);
 8988   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 8989   ins_encode %{
 8990     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 8991   %}
 8992   ins_pipe(ialu_reg_reg_alu0);
 8993 %}
 8994 
 8995 // Integer Shift Instructions
 8996 // Shift Left by one, two, three
 8997 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 8998 %{
 8999   predicate(!UseAPX);
 9000   match(Set dst (LShiftI dst shift));
 9001   effect(KILL cr);
 9002 
 9003   format %{ "sall    $dst, $shift" %}
 9004   ins_encode %{
 9005     __ sall($dst$$Register, $shift$$constant);
 9006   %}
 9007   ins_pipe(ialu_reg);
 9008 %}
 9009 
 9010 // Shift Left by one, two, three
 9011 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9012 %{
 9013   predicate(UseAPX);
 9014   match(Set dst (LShiftI src shift));
 9015   effect(KILL cr);
 9016 
 9017   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9018   ins_encode %{
 9019     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9020   %}
 9021   ins_pipe(ialu_reg);
 9022 %}
 9023 
 9024 // Shift Left by 8-bit immediate
 9025 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9026 %{
 9027   predicate(!UseAPX);
 9028   match(Set dst (LShiftI dst shift));
 9029   effect(KILL cr);
 9030 
 9031   format %{ "sall    $dst, $shift" %}
 9032   ins_encode %{
 9033     __ sall($dst$$Register, $shift$$constant);
 9034   %}
 9035   ins_pipe(ialu_reg);
 9036 %}
 9037 
 9038 // Shift Left by 8-bit immediate
 9039 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9040 %{
 9041   predicate(UseAPX);
 9042   match(Set dst (LShiftI src shift));
 9043   effect(KILL cr);
 9044 
 9045   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9046   ins_encode %{
 9047     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9048   %}
 9049   ins_pipe(ialu_reg);
 9050 %}
 9051 
 9052 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9053 %{
 9054   predicate(UseAPX);
 9055   match(Set dst (LShiftI (LoadI src) shift));
 9056   effect(KILL cr);
 9057 
 9058   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9059   ins_encode %{
 9060     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9061   %}
 9062   ins_pipe(ialu_reg);
 9063 %}
 9064 
 9065 // Shift Left by 8-bit immediate
 9066 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9067 %{
 9068   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9069   effect(KILL cr);
 9070 
 9071   format %{ "sall    $dst, $shift" %}
 9072   ins_encode %{
 9073     __ sall($dst$$Address, $shift$$constant);
 9074   %}
 9075   ins_pipe(ialu_mem_imm);
 9076 %}
 9077 
 9078 // Shift Left by variable
 9079 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9080 %{
 9081   predicate(!VM_Version::supports_bmi2());
 9082   match(Set dst (LShiftI dst shift));
 9083   effect(KILL cr);
 9084 
 9085   format %{ "sall    $dst, $shift" %}
 9086   ins_encode %{
 9087     __ sall($dst$$Register);
 9088   %}
 9089   ins_pipe(ialu_reg_reg);
 9090 %}
 9091 
 9092 // Shift Left by variable
 9093 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9094 %{
 9095   predicate(!VM_Version::supports_bmi2());
 9096   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9097   effect(KILL cr);
 9098 
 9099   format %{ "sall    $dst, $shift" %}
 9100   ins_encode %{
 9101     __ sall($dst$$Address);
 9102   %}
 9103   ins_pipe(ialu_mem_reg);
 9104 %}
 9105 
 9106 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9107 %{
 9108   predicate(VM_Version::supports_bmi2());
 9109   match(Set dst (LShiftI src shift));
 9110 
 9111   format %{ "shlxl   $dst, $src, $shift" %}
 9112   ins_encode %{
 9113     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9114   %}
 9115   ins_pipe(ialu_reg_reg);
 9116 %}
 9117 
 9118 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9119 %{
 9120   predicate(VM_Version::supports_bmi2());
 9121   match(Set dst (LShiftI (LoadI src) shift));
 9122   ins_cost(175);
 9123   format %{ "shlxl   $dst, $src, $shift" %}
 9124   ins_encode %{
 9125     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9126   %}
 9127   ins_pipe(ialu_reg_mem);
 9128 %}
 9129 
 9130 // Arithmetic Shift Right by 8-bit immediate
 9131 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9132 %{
 9133   predicate(!UseAPX);
 9134   match(Set dst (RShiftI dst shift));
 9135   effect(KILL cr);
 9136 
 9137   format %{ "sarl    $dst, $shift" %}
 9138   ins_encode %{
 9139     __ sarl($dst$$Register, $shift$$constant);
 9140   %}
 9141   ins_pipe(ialu_mem_imm);
 9142 %}
 9143 
 9144 // Arithmetic Shift Right by 8-bit immediate
 9145 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9146 %{
 9147   predicate(UseAPX);
 9148   match(Set dst (RShiftI src shift));
 9149   effect(KILL cr);
 9150 
 9151   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9152   ins_encode %{
 9153     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9154   %}
 9155   ins_pipe(ialu_mem_imm);
 9156 %}
 9157 
 9158 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9159 %{
 9160   predicate(UseAPX);
 9161   match(Set dst (RShiftI (LoadI src) shift));
 9162   effect(KILL cr);
 9163 
 9164   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9165   ins_encode %{
 9166     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9167   %}
 9168   ins_pipe(ialu_mem_imm);
 9169 %}
 9170 
 9171 // Arithmetic Shift Right by 8-bit immediate
 9172 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9173 %{
 9174   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9175   effect(KILL cr);
 9176 
 9177   format %{ "sarl    $dst, $shift" %}
 9178   ins_encode %{
 9179     __ sarl($dst$$Address, $shift$$constant);
 9180   %}
 9181   ins_pipe(ialu_mem_imm);
 9182 %}
 9183 
 9184 // Arithmetic Shift Right by variable
 9185 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9186 %{
 9187   predicate(!VM_Version::supports_bmi2());
 9188   match(Set dst (RShiftI dst shift));
 9189   effect(KILL cr);
 9190 
 9191   format %{ "sarl    $dst, $shift" %}
 9192   ins_encode %{
 9193     __ sarl($dst$$Register);
 9194   %}
 9195   ins_pipe(ialu_reg_reg);
 9196 %}
 9197 
 9198 // Arithmetic Shift Right by variable
 9199 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9200 %{
 9201   predicate(!VM_Version::supports_bmi2());
 9202   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9203   effect(KILL cr);
 9204 
 9205   format %{ "sarl    $dst, $shift" %}
 9206   ins_encode %{
 9207     __ sarl($dst$$Address);
 9208   %}
 9209   ins_pipe(ialu_mem_reg);
 9210 %}
 9211 
 9212 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9213 %{
 9214   predicate(VM_Version::supports_bmi2());
 9215   match(Set dst (RShiftI src shift));
 9216 
 9217   format %{ "sarxl   $dst, $src, $shift" %}
 9218   ins_encode %{
 9219     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9220   %}
 9221   ins_pipe(ialu_reg_reg);
 9222 %}
 9223 
 9224 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9225 %{
 9226   predicate(VM_Version::supports_bmi2());
 9227   match(Set dst (RShiftI (LoadI src) shift));
 9228   ins_cost(175);
 9229   format %{ "sarxl   $dst, $src, $shift" %}
 9230   ins_encode %{
 9231     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9232   %}
 9233   ins_pipe(ialu_reg_mem);
 9234 %}
 9235 
 9236 // Logical Shift Right by 8-bit immediate
 9237 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9238 %{
 9239   predicate(!UseAPX);
 9240   match(Set dst (URShiftI dst shift));
 9241   effect(KILL cr);
 9242 
 9243   format %{ "shrl    $dst, $shift" %}
 9244   ins_encode %{
 9245     __ shrl($dst$$Register, $shift$$constant);
 9246   %}
 9247   ins_pipe(ialu_reg);
 9248 %}
 9249 
 9250 // Logical Shift Right by 8-bit immediate
 9251 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9252 %{
 9253   predicate(UseAPX);
 9254   match(Set dst (URShiftI src shift));
 9255   effect(KILL cr);
 9256 
 9257   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9258   ins_encode %{
 9259     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9260   %}
 9261   ins_pipe(ialu_reg);
 9262 %}
 9263 
 9264 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9265 %{
 9266   predicate(UseAPX);
 9267   match(Set dst (URShiftI (LoadI src) shift));
 9268   effect(KILL cr);
 9269 
 9270   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9271   ins_encode %{
 9272     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9273   %}
 9274   ins_pipe(ialu_reg);
 9275 %}
 9276 
 9277 // Logical Shift Right by 8-bit immediate
 9278 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9279 %{
 9280   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9281   effect(KILL cr);
 9282 
 9283   format %{ "shrl    $dst, $shift" %}
 9284   ins_encode %{
 9285     __ shrl($dst$$Address, $shift$$constant);
 9286   %}
 9287   ins_pipe(ialu_mem_imm);
 9288 %}
 9289 
 9290 // Logical Shift Right by variable
 9291 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9292 %{
 9293   predicate(!VM_Version::supports_bmi2());
 9294   match(Set dst (URShiftI dst shift));
 9295   effect(KILL cr);
 9296 
 9297   format %{ "shrl    $dst, $shift" %}
 9298   ins_encode %{
 9299     __ shrl($dst$$Register);
 9300   %}
 9301   ins_pipe(ialu_reg_reg);
 9302 %}
 9303 
 9304 // Logical Shift Right by variable
 9305 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9306 %{
 9307   predicate(!VM_Version::supports_bmi2());
 9308   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9309   effect(KILL cr);
 9310 
 9311   format %{ "shrl    $dst, $shift" %}
 9312   ins_encode %{
 9313     __ shrl($dst$$Address);
 9314   %}
 9315   ins_pipe(ialu_mem_reg);
 9316 %}
 9317 
 9318 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9319 %{
 9320   predicate(VM_Version::supports_bmi2());
 9321   match(Set dst (URShiftI src shift));
 9322 
 9323   format %{ "shrxl   $dst, $src, $shift" %}
 9324   ins_encode %{
 9325     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9326   %}
 9327   ins_pipe(ialu_reg_reg);
 9328 %}
 9329 
 9330 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9331 %{
 9332   predicate(VM_Version::supports_bmi2());
 9333   match(Set dst (URShiftI (LoadI src) shift));
 9334   ins_cost(175);
 9335   format %{ "shrxl   $dst, $src, $shift" %}
 9336   ins_encode %{
 9337     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9338   %}
 9339   ins_pipe(ialu_reg_mem);
 9340 %}
 9341 
 9342 // Long Shift Instructions
 9343 // Shift Left by one, two, three
 9344 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9345 %{
 9346   predicate(!UseAPX);
 9347   match(Set dst (LShiftL dst shift));
 9348   effect(KILL cr);
 9349 
 9350   format %{ "salq    $dst, $shift" %}
 9351   ins_encode %{
 9352     __ salq($dst$$Register, $shift$$constant);
 9353   %}
 9354   ins_pipe(ialu_reg);
 9355 %}
 9356 
 9357 // Shift Left by one, two, three
 9358 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9359 %{
 9360   predicate(UseAPX);
 9361   match(Set dst (LShiftL src shift));
 9362   effect(KILL cr);
 9363 
 9364   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9365   ins_encode %{
 9366     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9367   %}
 9368   ins_pipe(ialu_reg);
 9369 %}
 9370 
 9371 // Shift Left by 8-bit immediate
 9372 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9373 %{
 9374   predicate(!UseAPX);
 9375   match(Set dst (LShiftL dst shift));
 9376   effect(KILL cr);
 9377 
 9378   format %{ "salq    $dst, $shift" %}
 9379   ins_encode %{
 9380     __ salq($dst$$Register, $shift$$constant);
 9381   %}
 9382   ins_pipe(ialu_reg);
 9383 %}
 9384 
 9385 // Shift Left by 8-bit immediate
 9386 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9387 %{
 9388   predicate(UseAPX);
 9389   match(Set dst (LShiftL src shift));
 9390   effect(KILL cr);
 9391 
 9392   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9393   ins_encode %{
 9394     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9395   %}
 9396   ins_pipe(ialu_reg);
 9397 %}
 9398 
 9399 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9400 %{
 9401   predicate(UseAPX);
 9402   match(Set dst (LShiftL (LoadL src) shift));
 9403   effect(KILL cr);
 9404 
 9405   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9406   ins_encode %{
 9407     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9408   %}
 9409   ins_pipe(ialu_reg);
 9410 %}
 9411 
 9412 // Shift Left by 8-bit immediate
 9413 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9414 %{
 9415   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9416   effect(KILL cr);
 9417 
 9418   format %{ "salq    $dst, $shift" %}
 9419   ins_encode %{
 9420     __ salq($dst$$Address, $shift$$constant);
 9421   %}
 9422   ins_pipe(ialu_mem_imm);
 9423 %}
 9424 
 9425 // Shift Left by variable
 9426 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9427 %{
 9428   predicate(!VM_Version::supports_bmi2());
 9429   match(Set dst (LShiftL dst shift));
 9430   effect(KILL cr);
 9431 
 9432   format %{ "salq    $dst, $shift" %}
 9433   ins_encode %{
 9434     __ salq($dst$$Register);
 9435   %}
 9436   ins_pipe(ialu_reg_reg);
 9437 %}
 9438 
 9439 // Shift Left by variable
 9440 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9441 %{
 9442   predicate(!VM_Version::supports_bmi2());
 9443   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9444   effect(KILL cr);
 9445 
 9446   format %{ "salq    $dst, $shift" %}
 9447   ins_encode %{
 9448     __ salq($dst$$Address);
 9449   %}
 9450   ins_pipe(ialu_mem_reg);
 9451 %}
 9452 
 9453 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9454 %{
 9455   predicate(VM_Version::supports_bmi2());
 9456   match(Set dst (LShiftL src shift));
 9457 
 9458   format %{ "shlxq   $dst, $src, $shift" %}
 9459   ins_encode %{
 9460     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9461   %}
 9462   ins_pipe(ialu_reg_reg);
 9463 %}
 9464 
 9465 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9466 %{
 9467   predicate(VM_Version::supports_bmi2());
 9468   match(Set dst (LShiftL (LoadL src) shift));
 9469   ins_cost(175);
 9470   format %{ "shlxq   $dst, $src, $shift" %}
 9471   ins_encode %{
 9472     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9473   %}
 9474   ins_pipe(ialu_reg_mem);
 9475 %}
 9476 
 9477 // Arithmetic Shift Right by 8-bit immediate
 9478 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9479 %{
 9480   predicate(!UseAPX);
 9481   match(Set dst (RShiftL dst shift));
 9482   effect(KILL cr);
 9483 
 9484   format %{ "sarq    $dst, $shift" %}
 9485   ins_encode %{
 9486     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9487   %}
 9488   ins_pipe(ialu_mem_imm);
 9489 %}
 9490 
 9491 // Arithmetic Shift Right by 8-bit immediate
 9492 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9493 %{
 9494   predicate(UseAPX);
 9495   match(Set dst (RShiftL src shift));
 9496   effect(KILL cr);
 9497 
 9498   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9499   ins_encode %{
 9500     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9501   %}
 9502   ins_pipe(ialu_mem_imm);
 9503 %}
 9504 
 9505 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9506 %{
 9507   predicate(UseAPX);
 9508   match(Set dst (RShiftL (LoadL src) shift));
 9509   effect(KILL cr);
 9510 
 9511   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9512   ins_encode %{
 9513     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9514   %}
 9515   ins_pipe(ialu_mem_imm);
 9516 %}
 9517 
 9518 // Arithmetic Shift Right by 8-bit immediate
 9519 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9520 %{
 9521   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9522   effect(KILL cr);
 9523 
 9524   format %{ "sarq    $dst, $shift" %}
 9525   ins_encode %{
 9526     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9527   %}
 9528   ins_pipe(ialu_mem_imm);
 9529 %}
 9530 
 9531 // Arithmetic Shift Right by variable
 9532 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9533 %{
 9534   predicate(!VM_Version::supports_bmi2());
 9535   match(Set dst (RShiftL dst shift));
 9536   effect(KILL cr);
 9537 
 9538   format %{ "sarq    $dst, $shift" %}
 9539   ins_encode %{
 9540     __ sarq($dst$$Register);
 9541   %}
 9542   ins_pipe(ialu_reg_reg);
 9543 %}
 9544 
 9545 // Arithmetic Shift Right by variable
 9546 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9547 %{
 9548   predicate(!VM_Version::supports_bmi2());
 9549   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9550   effect(KILL cr);
 9551 
 9552   format %{ "sarq    $dst, $shift" %}
 9553   ins_encode %{
 9554     __ sarq($dst$$Address);
 9555   %}
 9556   ins_pipe(ialu_mem_reg);
 9557 %}
 9558 
 9559 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9560 %{
 9561   predicate(VM_Version::supports_bmi2());
 9562   match(Set dst (RShiftL src shift));
 9563 
 9564   format %{ "sarxq   $dst, $src, $shift" %}
 9565   ins_encode %{
 9566     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9567   %}
 9568   ins_pipe(ialu_reg_reg);
 9569 %}
 9570 
 9571 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9572 %{
 9573   predicate(VM_Version::supports_bmi2());
 9574   match(Set dst (RShiftL (LoadL src) shift));
 9575   ins_cost(175);
 9576   format %{ "sarxq   $dst, $src, $shift" %}
 9577   ins_encode %{
 9578     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9579   %}
 9580   ins_pipe(ialu_reg_mem);
 9581 %}
 9582 
 9583 // Logical Shift Right by 8-bit immediate
 9584 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9585 %{
 9586   predicate(!UseAPX);
 9587   match(Set dst (URShiftL dst shift));
 9588   effect(KILL cr);
 9589 
 9590   format %{ "shrq    $dst, $shift" %}
 9591   ins_encode %{
 9592     __ shrq($dst$$Register, $shift$$constant);
 9593   %}
 9594   ins_pipe(ialu_reg);
 9595 %}
 9596 
 9597 // Logical Shift Right by 8-bit immediate
 9598 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9599 %{
 9600   predicate(UseAPX);
 9601   match(Set dst (URShiftL src shift));
 9602   effect(KILL cr);
 9603 
 9604   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9605   ins_encode %{
 9606     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9607   %}
 9608   ins_pipe(ialu_reg);
 9609 %}
 9610 
 9611 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9612 %{
 9613   predicate(UseAPX);
 9614   match(Set dst (URShiftL (LoadL src) shift));
 9615   effect(KILL cr);
 9616 
 9617   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9618   ins_encode %{
 9619     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9620   %}
 9621   ins_pipe(ialu_reg);
 9622 %}
 9623 
 9624 // Logical Shift Right by 8-bit immediate
 9625 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9626 %{
 9627   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9628   effect(KILL cr);
 9629 
 9630   format %{ "shrq    $dst, $shift" %}
 9631   ins_encode %{
 9632     __ shrq($dst$$Address, $shift$$constant);
 9633   %}
 9634   ins_pipe(ialu_mem_imm);
 9635 %}
 9636 
 9637 // Logical Shift Right by variable
 9638 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9639 %{
 9640   predicate(!VM_Version::supports_bmi2());
 9641   match(Set dst (URShiftL dst shift));
 9642   effect(KILL cr);
 9643 
 9644   format %{ "shrq    $dst, $shift" %}
 9645   ins_encode %{
 9646     __ shrq($dst$$Register);
 9647   %}
 9648   ins_pipe(ialu_reg_reg);
 9649 %}
 9650 
 9651 // Logical Shift Right by variable
 9652 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9653 %{
 9654   predicate(!VM_Version::supports_bmi2());
 9655   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9656   effect(KILL cr);
 9657 
 9658   format %{ "shrq    $dst, $shift" %}
 9659   ins_encode %{
 9660     __ shrq($dst$$Address);
 9661   %}
 9662   ins_pipe(ialu_mem_reg);
 9663 %}
 9664 
 9665 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9666 %{
 9667   predicate(VM_Version::supports_bmi2());
 9668   match(Set dst (URShiftL src shift));
 9669 
 9670   format %{ "shrxq   $dst, $src, $shift" %}
 9671   ins_encode %{
 9672     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9673   %}
 9674   ins_pipe(ialu_reg_reg);
 9675 %}
 9676 
 9677 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9678 %{
 9679   predicate(VM_Version::supports_bmi2());
 9680   match(Set dst (URShiftL (LoadL src) shift));
 9681   ins_cost(175);
 9682   format %{ "shrxq   $dst, $src, $shift" %}
 9683   ins_encode %{
 9684     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9685   %}
 9686   ins_pipe(ialu_reg_mem);
 9687 %}
 9688 
 9689 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9690 // This idiom is used by the compiler for the i2b bytecode.
 9691 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9692 %{
 9693   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9694 
 9695   format %{ "movsbl  $dst, $src\t# i2b" %}
 9696   ins_encode %{
 9697     __ movsbl($dst$$Register, $src$$Register);
 9698   %}
 9699   ins_pipe(ialu_reg_reg);
 9700 %}
 9701 
 9702 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9703 // This idiom is used by the compiler the i2s bytecode.
 9704 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9705 %{
 9706   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9707 
 9708   format %{ "movswl  $dst, $src\t# i2s" %}
 9709   ins_encode %{
 9710     __ movswl($dst$$Register, $src$$Register);
 9711   %}
 9712   ins_pipe(ialu_reg_reg);
 9713 %}
 9714 
 9715 // ROL/ROR instructions
 9716 
 9717 // Rotate left by constant.
 9718 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9719 %{
 9720   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9721   match(Set dst (RotateLeft dst shift));
 9722   effect(KILL cr);
 9723   format %{ "roll    $dst, $shift" %}
 9724   ins_encode %{
 9725     __ roll($dst$$Register, $shift$$constant);
 9726   %}
 9727   ins_pipe(ialu_reg);
 9728 %}
 9729 
 9730 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9731 %{
 9732   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9733   match(Set dst (RotateLeft src shift));
 9734   format %{ "rolxl   $dst, $src, $shift" %}
 9735   ins_encode %{
 9736     int shift = 32 - ($shift$$constant & 31);
 9737     __ rorxl($dst$$Register, $src$$Register, shift);
 9738   %}
 9739   ins_pipe(ialu_reg_reg);
 9740 %}
 9741 
 9742 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9743 %{
 9744   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9745   match(Set dst (RotateLeft (LoadI src) shift));
 9746   ins_cost(175);
 9747   format %{ "rolxl   $dst, $src, $shift" %}
 9748   ins_encode %{
 9749     int shift = 32 - ($shift$$constant & 31);
 9750     __ rorxl($dst$$Register, $src$$Address, shift);
 9751   %}
 9752   ins_pipe(ialu_reg_mem);
 9753 %}
 9754 
 9755 // Rotate Left by variable
 9756 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9757 %{
 9758   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9759   match(Set dst (RotateLeft dst shift));
 9760   effect(KILL cr);
 9761   format %{ "roll    $dst, $shift" %}
 9762   ins_encode %{
 9763     __ roll($dst$$Register);
 9764   %}
 9765   ins_pipe(ialu_reg_reg);
 9766 %}
 9767 
 9768 // Rotate Left by variable
 9769 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9770 %{
 9771   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9772   match(Set dst (RotateLeft src shift));
 9773   effect(KILL cr);
 9774 
 9775   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9776   ins_encode %{
 9777     __ eroll($dst$$Register, $src$$Register, false);
 9778   %}
 9779   ins_pipe(ialu_reg_reg);
 9780 %}
 9781 
 9782 // Rotate Right by constant.
 9783 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9784 %{
 9785   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9786   match(Set dst (RotateRight dst shift));
 9787   effect(KILL cr);
 9788   format %{ "rorl    $dst, $shift" %}
 9789   ins_encode %{
 9790     __ rorl($dst$$Register, $shift$$constant);
 9791   %}
 9792   ins_pipe(ialu_reg);
 9793 %}
 9794 
 9795 // Rotate Right by constant.
 9796 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9797 %{
 9798   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9799   match(Set dst (RotateRight src shift));
 9800   format %{ "rorxl   $dst, $src, $shift" %}
 9801   ins_encode %{
 9802     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9803   %}
 9804   ins_pipe(ialu_reg_reg);
 9805 %}
 9806 
 9807 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9808 %{
 9809   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9810   match(Set dst (RotateRight (LoadI src) shift));
 9811   ins_cost(175);
 9812   format %{ "rorxl   $dst, $src, $shift" %}
 9813   ins_encode %{
 9814     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9815   %}
 9816   ins_pipe(ialu_reg_mem);
 9817 %}
 9818 
 9819 // Rotate Right by variable
 9820 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9821 %{
 9822   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9823   match(Set dst (RotateRight dst shift));
 9824   effect(KILL cr);
 9825   format %{ "rorl    $dst, $shift" %}
 9826   ins_encode %{
 9827     __ rorl($dst$$Register);
 9828   %}
 9829   ins_pipe(ialu_reg_reg);
 9830 %}
 9831 
 9832 // Rotate Right by variable
 9833 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9834 %{
 9835   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9836   match(Set dst (RotateRight src shift));
 9837   effect(KILL cr);
 9838 
 9839   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9840   ins_encode %{
 9841     __ erorl($dst$$Register, $src$$Register, false);
 9842   %}
 9843   ins_pipe(ialu_reg_reg);
 9844 %}
 9845 
 9846 // Rotate Left by constant.
 9847 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9848 %{
 9849   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9850   match(Set dst (RotateLeft dst shift));
 9851   effect(KILL cr);
 9852   format %{ "rolq    $dst, $shift" %}
 9853   ins_encode %{
 9854     __ rolq($dst$$Register, $shift$$constant);
 9855   %}
 9856   ins_pipe(ialu_reg);
 9857 %}
 9858 
 9859 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9860 %{
 9861   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9862   match(Set dst (RotateLeft src shift));
 9863   format %{ "rolxq   $dst, $src, $shift" %}
 9864   ins_encode %{
 9865     int shift = 64 - ($shift$$constant & 63);
 9866     __ rorxq($dst$$Register, $src$$Register, shift);
 9867   %}
 9868   ins_pipe(ialu_reg_reg);
 9869 %}
 9870 
 9871 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9872 %{
 9873   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9874   match(Set dst (RotateLeft (LoadL src) shift));
 9875   ins_cost(175);
 9876   format %{ "rolxq   $dst, $src, $shift" %}
 9877   ins_encode %{
 9878     int shift = 64 - ($shift$$constant & 63);
 9879     __ rorxq($dst$$Register, $src$$Address, shift);
 9880   %}
 9881   ins_pipe(ialu_reg_mem);
 9882 %}
 9883 
 9884 // Rotate Left by variable
 9885 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9886 %{
 9887   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9888   match(Set dst (RotateLeft dst shift));
 9889   effect(KILL cr);
 9890   format %{ "rolq    $dst, $shift" %}
 9891   ins_encode %{
 9892     __ rolq($dst$$Register);
 9893   %}
 9894   ins_pipe(ialu_reg_reg);
 9895 %}
 9896 
 9897 // Rotate Left by variable
 9898 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9899 %{
 9900   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9901   match(Set dst (RotateLeft src shift));
 9902   effect(KILL cr);
 9903 
 9904   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9905   ins_encode %{
 9906     __ erolq($dst$$Register, $src$$Register, false);
 9907   %}
 9908   ins_pipe(ialu_reg_reg);
 9909 %}
 9910 
 9911 // Rotate Right by constant.
 9912 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9913 %{
 9914   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9915   match(Set dst (RotateRight dst shift));
 9916   effect(KILL cr);
 9917   format %{ "rorq    $dst, $shift" %}
 9918   ins_encode %{
 9919     __ rorq($dst$$Register, $shift$$constant);
 9920   %}
 9921   ins_pipe(ialu_reg);
 9922 %}
 9923 
 9924 // Rotate Right by constant
 9925 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9926 %{
 9927   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9928   match(Set dst (RotateRight src shift));
 9929   format %{ "rorxq   $dst, $src, $shift" %}
 9930   ins_encode %{
 9931     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9932   %}
 9933   ins_pipe(ialu_reg_reg);
 9934 %}
 9935 
 9936 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9937 %{
 9938   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9939   match(Set dst (RotateRight (LoadL src) shift));
 9940   ins_cost(175);
 9941   format %{ "rorxq   $dst, $src, $shift" %}
 9942   ins_encode %{
 9943     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9944   %}
 9945   ins_pipe(ialu_reg_mem);
 9946 %}
 9947 
 9948 // Rotate Right by variable
 9949 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9950 %{
 9951   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9952   match(Set dst (RotateRight dst shift));
 9953   effect(KILL cr);
 9954   format %{ "rorq    $dst, $shift" %}
 9955   ins_encode %{
 9956     __ rorq($dst$$Register);
 9957   %}
 9958   ins_pipe(ialu_reg_reg);
 9959 %}
 9960 
 9961 // Rotate Right by variable
 9962 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9963 %{
 9964   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9965   match(Set dst (RotateRight src shift));
 9966   effect(KILL cr);
 9967 
 9968   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
 9969   ins_encode %{
 9970     __ erorq($dst$$Register, $src$$Register, false);
 9971   %}
 9972   ins_pipe(ialu_reg_reg);
 9973 %}
 9974 
 9975 //----------------------------- CompressBits/ExpandBits ------------------------
 9976 
 9977 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9978   predicate(n->bottom_type()->isa_long());
 9979   match(Set dst (CompressBits src mask));
 9980   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9981   ins_encode %{
 9982     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9983   %}
 9984   ins_pipe( pipe_slow );
 9985 %}
 9986 
 9987 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9988   predicate(n->bottom_type()->isa_long());
 9989   match(Set dst (ExpandBits src mask));
 9990   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
 9991   ins_encode %{
 9992     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
 9993   %}
 9994   ins_pipe( pipe_slow );
 9995 %}
 9996 
 9997 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
 9998   predicate(n->bottom_type()->isa_long());
 9999   match(Set dst (CompressBits src (LoadL mask)));
10000   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10001   ins_encode %{
10002     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10003   %}
10004   ins_pipe( pipe_slow );
10005 %}
10006 
10007 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10008   predicate(n->bottom_type()->isa_long());
10009   match(Set dst (ExpandBits src (LoadL mask)));
10010   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10011   ins_encode %{
10012     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10013   %}
10014   ins_pipe( pipe_slow );
10015 %}
10016 
10017 
10018 // Logical Instructions
10019 
10020 // Integer Logical Instructions
10021 
10022 // And Instructions
10023 // And Register with Register
10024 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10025 %{
10026   predicate(!UseAPX);
10027   match(Set dst (AndI dst src));
10028   effect(KILL cr);
10029   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10030 
10031   format %{ "andl    $dst, $src\t# int" %}
10032   ins_encode %{
10033     __ andl($dst$$Register, $src$$Register);
10034   %}
10035   ins_pipe(ialu_reg_reg);
10036 %}
10037 
10038 // And Register with Register using New Data Destination (NDD)
10039 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10040 %{
10041   predicate(UseAPX);
10042   match(Set dst (AndI src1 src2));
10043   effect(KILL cr);
10044   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10045 
10046   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10047   ins_encode %{
10048     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10049 
10050   %}
10051   ins_pipe(ialu_reg_reg);
10052 %}
10053 
10054 // And Register with Immediate 255
10055 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10056 %{
10057   match(Set dst (AndI src mask));
10058 
10059   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10060   ins_encode %{
10061     __ movzbl($dst$$Register, $src$$Register);
10062   %}
10063   ins_pipe(ialu_reg);
10064 %}
10065 
10066 // And Register with Immediate 255 and promote to long
10067 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10068 %{
10069   match(Set dst (ConvI2L (AndI src mask)));
10070 
10071   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10072   ins_encode %{
10073     __ movzbl($dst$$Register, $src$$Register);
10074   %}
10075   ins_pipe(ialu_reg);
10076 %}
10077 
10078 // And Register with Immediate 65535
10079 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10080 %{
10081   match(Set dst (AndI src mask));
10082 
10083   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10084   ins_encode %{
10085     __ movzwl($dst$$Register, $src$$Register);
10086   %}
10087   ins_pipe(ialu_reg);
10088 %}
10089 
10090 // And Register with Immediate 65535 and promote to long
10091 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10092 %{
10093   match(Set dst (ConvI2L (AndI src mask)));
10094 
10095   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10096   ins_encode %{
10097     __ movzwl($dst$$Register, $src$$Register);
10098   %}
10099   ins_pipe(ialu_reg);
10100 %}
10101 
10102 // Can skip int2long conversions after AND with small bitmask
10103 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10104 %{
10105   predicate(VM_Version::supports_bmi2());
10106   ins_cost(125);
10107   effect(TEMP tmp, KILL cr);
10108   match(Set dst (ConvI2L (AndI src mask)));
10109   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10110   ins_encode %{
10111     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10112     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10113   %}
10114   ins_pipe(ialu_reg_reg);
10115 %}
10116 
10117 // And Register with Immediate
10118 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10119 %{
10120   predicate(!UseAPX);
10121   match(Set dst (AndI dst src));
10122   effect(KILL cr);
10123   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10124 
10125   format %{ "andl    $dst, $src\t# int" %}
10126   ins_encode %{
10127     __ andl($dst$$Register, $src$$constant);
10128   %}
10129   ins_pipe(ialu_reg);
10130 %}
10131 
10132 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10133 %{
10134   predicate(UseAPX);
10135   match(Set dst (AndI src1 src2));
10136   effect(KILL cr);
10137   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10138 
10139   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10140   ins_encode %{
10141     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10142   %}
10143   ins_pipe(ialu_reg);
10144 %}
10145 
10146 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10147 %{
10148   predicate(UseAPX);
10149   match(Set dst (AndI (LoadI src1) src2));
10150   effect(KILL cr);
10151   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10152 
10153   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10154   ins_encode %{
10155     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10156   %}
10157   ins_pipe(ialu_reg);
10158 %}
10159 
10160 // And Register with Memory
10161 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10162 %{
10163   predicate(!UseAPX);
10164   match(Set dst (AndI dst (LoadI src)));
10165   effect(KILL cr);
10166   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10167 
10168   ins_cost(150);
10169   format %{ "andl    $dst, $src\t# int" %}
10170   ins_encode %{
10171     __ andl($dst$$Register, $src$$Address);
10172   %}
10173   ins_pipe(ialu_reg_mem);
10174 %}
10175 
10176 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10177 %{
10178   predicate(UseAPX);
10179   match(Set dst (AndI src1 (LoadI src2)));
10180   effect(KILL cr);
10181   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10182 
10183   ins_cost(150);
10184   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10185   ins_encode %{
10186     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10187   %}
10188   ins_pipe(ialu_reg_mem);
10189 %}
10190 
10191 // And Memory with Register
10192 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10193 %{
10194   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10195   effect(KILL cr);
10196   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10197 
10198   ins_cost(150);
10199   format %{ "andb    $dst, $src\t# byte" %}
10200   ins_encode %{
10201     __ andb($dst$$Address, $src$$Register);
10202   %}
10203   ins_pipe(ialu_mem_reg);
10204 %}
10205 
10206 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10207 %{
10208   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10209   effect(KILL cr);
10210   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10211 
10212   ins_cost(150);
10213   format %{ "andl    $dst, $src\t# int" %}
10214   ins_encode %{
10215     __ andl($dst$$Address, $src$$Register);
10216   %}
10217   ins_pipe(ialu_mem_reg);
10218 %}
10219 
10220 // And Memory with Immediate
10221 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10222 %{
10223   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10224   effect(KILL cr);
10225   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10226 
10227   ins_cost(125);
10228   format %{ "andl    $dst, $src\t# int" %}
10229   ins_encode %{
10230     __ andl($dst$$Address, $src$$constant);
10231   %}
10232   ins_pipe(ialu_mem_imm);
10233 %}
10234 
10235 // BMI1 instructions
10236 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10237   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10238   predicate(UseBMI1Instructions);
10239   effect(KILL cr);
10240   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10241 
10242   ins_cost(125);
10243   format %{ "andnl  $dst, $src1, $src2" %}
10244 
10245   ins_encode %{
10246     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10247   %}
10248   ins_pipe(ialu_reg_mem);
10249 %}
10250 
10251 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10252   match(Set dst (AndI (XorI src1 minus_1) src2));
10253   predicate(UseBMI1Instructions);
10254   effect(KILL cr);
10255   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10256 
10257   format %{ "andnl  $dst, $src1, $src2" %}
10258 
10259   ins_encode %{
10260     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10261   %}
10262   ins_pipe(ialu_reg);
10263 %}
10264 
10265 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10266   match(Set dst (AndI (SubI imm_zero src) src));
10267   predicate(UseBMI1Instructions);
10268   effect(KILL cr);
10269   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10270 
10271   format %{ "blsil  $dst, $src" %}
10272 
10273   ins_encode %{
10274     __ blsil($dst$$Register, $src$$Register);
10275   %}
10276   ins_pipe(ialu_reg);
10277 %}
10278 
10279 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10280   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10281   predicate(UseBMI1Instructions);
10282   effect(KILL cr);
10283   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10284 
10285   ins_cost(125);
10286   format %{ "blsil  $dst, $src" %}
10287 
10288   ins_encode %{
10289     __ blsil($dst$$Register, $src$$Address);
10290   %}
10291   ins_pipe(ialu_reg_mem);
10292 %}
10293 
10294 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10295 %{
10296   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10297   predicate(UseBMI1Instructions);
10298   effect(KILL cr);
10299   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10300 
10301   ins_cost(125);
10302   format %{ "blsmskl $dst, $src" %}
10303 
10304   ins_encode %{
10305     __ blsmskl($dst$$Register, $src$$Address);
10306   %}
10307   ins_pipe(ialu_reg_mem);
10308 %}
10309 
10310 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10311 %{
10312   match(Set dst (XorI (AddI src minus_1) src));
10313   predicate(UseBMI1Instructions);
10314   effect(KILL cr);
10315   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10316 
10317   format %{ "blsmskl $dst, $src" %}
10318 
10319   ins_encode %{
10320     __ blsmskl($dst$$Register, $src$$Register);
10321   %}
10322 
10323   ins_pipe(ialu_reg);
10324 %}
10325 
10326 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10327 %{
10328   match(Set dst (AndI (AddI src minus_1) src) );
10329   predicate(UseBMI1Instructions);
10330   effect(KILL cr);
10331   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10332 
10333   format %{ "blsrl  $dst, $src" %}
10334 
10335   ins_encode %{
10336     __ blsrl($dst$$Register, $src$$Register);
10337   %}
10338 
10339   ins_pipe(ialu_reg_mem);
10340 %}
10341 
10342 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10343 %{
10344   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10345   predicate(UseBMI1Instructions);
10346   effect(KILL cr);
10347   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10348 
10349   ins_cost(125);
10350   format %{ "blsrl  $dst, $src" %}
10351 
10352   ins_encode %{
10353     __ blsrl($dst$$Register, $src$$Address);
10354   %}
10355 
10356   ins_pipe(ialu_reg);
10357 %}
10358 
10359 // Or Instructions
10360 // Or Register with Register
10361 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10362 %{
10363   predicate(!UseAPX);
10364   match(Set dst (OrI dst src));
10365   effect(KILL cr);
10366   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10367 
10368   format %{ "orl     $dst, $src\t# int" %}
10369   ins_encode %{
10370     __ orl($dst$$Register, $src$$Register);
10371   %}
10372   ins_pipe(ialu_reg_reg);
10373 %}
10374 
10375 // Or Register with Register using New Data Destination (NDD)
10376 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10377 %{
10378   predicate(UseAPX);
10379   match(Set dst (OrI src1 src2));
10380   effect(KILL cr);
10381   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10382 
10383   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10384   ins_encode %{
10385     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10386   %}
10387   ins_pipe(ialu_reg_reg);
10388 %}
10389 
10390 // Or Register with Immediate
10391 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10392 %{
10393   predicate(!UseAPX);
10394   match(Set dst (OrI dst src));
10395   effect(KILL cr);
10396   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10397 
10398   format %{ "orl     $dst, $src\t# int" %}
10399   ins_encode %{
10400     __ orl($dst$$Register, $src$$constant);
10401   %}
10402   ins_pipe(ialu_reg);
10403 %}
10404 
10405 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10406 %{
10407   predicate(UseAPX);
10408   match(Set dst (OrI src1 src2));
10409   effect(KILL cr);
10410   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10411 
10412   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10413   ins_encode %{
10414     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10415   %}
10416   ins_pipe(ialu_reg);
10417 %}
10418 
10419 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10420 %{
10421   predicate(UseAPX);
10422   match(Set dst (OrI src1 src2));
10423   effect(KILL cr);
10424   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10425 
10426   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10427   ins_encode %{
10428     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10429   %}
10430   ins_pipe(ialu_reg);
10431 %}
10432 
10433 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10434 %{
10435   predicate(UseAPX);
10436   match(Set dst (OrI (LoadI src1) src2));
10437   effect(KILL cr);
10438   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10439 
10440   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10441   ins_encode %{
10442     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10443   %}
10444   ins_pipe(ialu_reg);
10445 %}
10446 
10447 // Or Register with Memory
10448 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10449 %{
10450   predicate(!UseAPX);
10451   match(Set dst (OrI dst (LoadI src)));
10452   effect(KILL cr);
10453   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10454 
10455   ins_cost(150);
10456   format %{ "orl     $dst, $src\t# int" %}
10457   ins_encode %{
10458     __ orl($dst$$Register, $src$$Address);
10459   %}
10460   ins_pipe(ialu_reg_mem);
10461 %}
10462 
10463 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10464 %{
10465   predicate(UseAPX);
10466   match(Set dst (OrI src1 (LoadI src2)));
10467   effect(KILL cr);
10468   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10469 
10470   ins_cost(150);
10471   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10472   ins_encode %{
10473     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10474   %}
10475   ins_pipe(ialu_reg_mem);
10476 %}
10477 
10478 // Or Memory with Register
10479 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10480 %{
10481   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10482   effect(KILL cr);
10483   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10484 
10485   ins_cost(150);
10486   format %{ "orb    $dst, $src\t# byte" %}
10487   ins_encode %{
10488     __ orb($dst$$Address, $src$$Register);
10489   %}
10490   ins_pipe(ialu_mem_reg);
10491 %}
10492 
10493 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10494 %{
10495   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10496   effect(KILL cr);
10497   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10498 
10499   ins_cost(150);
10500   format %{ "orl     $dst, $src\t# int" %}
10501   ins_encode %{
10502     __ orl($dst$$Address, $src$$Register);
10503   %}
10504   ins_pipe(ialu_mem_reg);
10505 %}
10506 
10507 // Or Memory with Immediate
10508 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10509 %{
10510   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10511   effect(KILL cr);
10512   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10513 
10514   ins_cost(125);
10515   format %{ "orl     $dst, $src\t# int" %}
10516   ins_encode %{
10517     __ orl($dst$$Address, $src$$constant);
10518   %}
10519   ins_pipe(ialu_mem_imm);
10520 %}
10521 
10522 // Xor Instructions
10523 // Xor Register with Register
10524 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10525 %{
10526   predicate(!UseAPX);
10527   match(Set dst (XorI dst src));
10528   effect(KILL cr);
10529   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10530 
10531   format %{ "xorl    $dst, $src\t# int" %}
10532   ins_encode %{
10533     __ xorl($dst$$Register, $src$$Register);
10534   %}
10535   ins_pipe(ialu_reg_reg);
10536 %}
10537 
10538 // Xor Register with Register using New Data Destination (NDD)
10539 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10540 %{
10541   predicate(UseAPX);
10542   match(Set dst (XorI src1 src2));
10543   effect(KILL cr);
10544   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10545 
10546   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10547   ins_encode %{
10548     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10549   %}
10550   ins_pipe(ialu_reg_reg);
10551 %}
10552 
10553 // Xor Register with Immediate -1
10554 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10555 %{
10556   predicate(!UseAPX);
10557   match(Set dst (XorI dst imm));
10558 
10559   format %{ "notl    $dst" %}
10560   ins_encode %{
10561      __ notl($dst$$Register);
10562   %}
10563   ins_pipe(ialu_reg);
10564 %}
10565 
10566 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10567 %{
10568   match(Set dst (XorI src imm));
10569   predicate(UseAPX);
10570 
10571   format %{ "enotl    $dst, $src" %}
10572   ins_encode %{
10573      __ enotl($dst$$Register, $src$$Register);
10574   %}
10575   ins_pipe(ialu_reg);
10576 %}
10577 
10578 // Xor Register with Immediate
10579 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10580 %{
10581   // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
10582   predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10583   match(Set dst (XorI dst src));
10584   effect(KILL cr);
10585   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10586 
10587   format %{ "xorl    $dst, $src\t# int" %}
10588   ins_encode %{
10589     __ xorl($dst$$Register, $src$$constant);
10590   %}
10591   ins_pipe(ialu_reg);
10592 %}
10593 
10594 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10595 %{
10596   // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
10597   predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10598   match(Set dst (XorI src1 src2));
10599   effect(KILL cr);
10600   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10601 
10602   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10603   ins_encode %{
10604     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10605   %}
10606   ins_pipe(ialu_reg);
10607 %}
10608 
10609 // Xor Memory with Immediate
10610 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10611 %{
10612   predicate(UseAPX);
10613   match(Set dst (XorI (LoadI src1) src2));
10614   effect(KILL cr);
10615   ins_cost(150);
10616   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10617 
10618   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10619   ins_encode %{
10620     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10621   %}
10622   ins_pipe(ialu_reg);
10623 %}
10624 
10625 // Xor Register with Memory
10626 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10627 %{
10628   predicate(!UseAPX);
10629   match(Set dst (XorI dst (LoadI src)));
10630   effect(KILL cr);
10631   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10632 
10633   ins_cost(150);
10634   format %{ "xorl    $dst, $src\t# int" %}
10635   ins_encode %{
10636     __ xorl($dst$$Register, $src$$Address);
10637   %}
10638   ins_pipe(ialu_reg_mem);
10639 %}
10640 
10641 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10642 %{
10643   predicate(UseAPX);
10644   match(Set dst (XorI src1 (LoadI src2)));
10645   effect(KILL cr);
10646   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10647 
10648   ins_cost(150);
10649   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10650   ins_encode %{
10651     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10652   %}
10653   ins_pipe(ialu_reg_mem);
10654 %}
10655 
10656 // Xor Memory with Register
10657 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10658 %{
10659   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10660   effect(KILL cr);
10661   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10662 
10663   ins_cost(150);
10664   format %{ "xorb    $dst, $src\t# byte" %}
10665   ins_encode %{
10666     __ xorb($dst$$Address, $src$$Register);
10667   %}
10668   ins_pipe(ialu_mem_reg);
10669 %}
10670 
10671 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10672 %{
10673   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10674   effect(KILL cr);
10675   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10676 
10677   ins_cost(150);
10678   format %{ "xorl    $dst, $src\t# int" %}
10679   ins_encode %{
10680     __ xorl($dst$$Address, $src$$Register);
10681   %}
10682   ins_pipe(ialu_mem_reg);
10683 %}
10684 
10685 // Xor Memory with Immediate
10686 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10687 %{
10688   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10689   effect(KILL cr);
10690   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10691 
10692   ins_cost(125);
10693   format %{ "xorl    $dst, $src\t# int" %}
10694   ins_encode %{
10695     __ xorl($dst$$Address, $src$$constant);
10696   %}
10697   ins_pipe(ialu_mem_imm);
10698 %}
10699 
10700 
10701 // Long Logical Instructions
10702 
10703 // And Instructions
10704 // And Register with Register
10705 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10706 %{
10707   predicate(!UseAPX);
10708   match(Set dst (AndL dst src));
10709   effect(KILL cr);
10710   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10711 
10712   format %{ "andq    $dst, $src\t# long" %}
10713   ins_encode %{
10714     __ andq($dst$$Register, $src$$Register);
10715   %}
10716   ins_pipe(ialu_reg_reg);
10717 %}
10718 
10719 // And Register with Register using New Data Destination (NDD)
10720 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10721 %{
10722   predicate(UseAPX);
10723   match(Set dst (AndL src1 src2));
10724   effect(KILL cr);
10725   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10726 
10727   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10728   ins_encode %{
10729     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10730 
10731   %}
10732   ins_pipe(ialu_reg_reg);
10733 %}
10734 
10735 // And Register with Immediate 255
10736 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10737 %{
10738   match(Set dst (AndL src mask));
10739 
10740   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10741   ins_encode %{
10742     // movzbl zeroes out the upper 32-bit and does not need REX.W
10743     __ movzbl($dst$$Register, $src$$Register);
10744   %}
10745   ins_pipe(ialu_reg);
10746 %}
10747 
10748 // And Register with Immediate 65535
10749 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10750 %{
10751   match(Set dst (AndL src mask));
10752 
10753   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10754   ins_encode %{
10755     // movzwl zeroes out the upper 32-bit and does not need REX.W
10756     __ movzwl($dst$$Register, $src$$Register);
10757   %}
10758   ins_pipe(ialu_reg);
10759 %}
10760 
10761 // And Register with Immediate
10762 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10763 %{
10764   predicate(!UseAPX);
10765   match(Set dst (AndL dst src));
10766   effect(KILL cr);
10767   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10768 
10769   format %{ "andq    $dst, $src\t# long" %}
10770   ins_encode %{
10771     __ andq($dst$$Register, $src$$constant);
10772   %}
10773   ins_pipe(ialu_reg);
10774 %}
10775 
10776 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10777 %{
10778   predicate(UseAPX);
10779   match(Set dst (AndL src1 src2));
10780   effect(KILL cr);
10781   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10782 
10783   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10784   ins_encode %{
10785     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10786   %}
10787   ins_pipe(ialu_reg);
10788 %}
10789 
10790 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10791 %{
10792   predicate(UseAPX);
10793   match(Set dst (AndL (LoadL src1) src2));
10794   effect(KILL cr);
10795   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10796 
10797   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10798   ins_encode %{
10799     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10800   %}
10801   ins_pipe(ialu_reg);
10802 %}
10803 
10804 // And Register with Memory
10805 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10806 %{
10807   predicate(!UseAPX);
10808   match(Set dst (AndL dst (LoadL src)));
10809   effect(KILL cr);
10810   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10811 
10812   ins_cost(150);
10813   format %{ "andq    $dst, $src\t# long" %}
10814   ins_encode %{
10815     __ andq($dst$$Register, $src$$Address);
10816   %}
10817   ins_pipe(ialu_reg_mem);
10818 %}
10819 
10820 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10821 %{
10822   predicate(UseAPX);
10823   match(Set dst (AndL src1 (LoadL src2)));
10824   effect(KILL cr);
10825   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10826 
10827   ins_cost(150);
10828   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10829   ins_encode %{
10830     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10831   %}
10832   ins_pipe(ialu_reg_mem);
10833 %}
10834 
10835 // And Memory with Register
10836 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10837 %{
10838   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10839   effect(KILL cr);
10840   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10841 
10842   ins_cost(150);
10843   format %{ "andq    $dst, $src\t# long" %}
10844   ins_encode %{
10845     __ andq($dst$$Address, $src$$Register);
10846   %}
10847   ins_pipe(ialu_mem_reg);
10848 %}
10849 
10850 // And Memory with Immediate
10851 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10852 %{
10853   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10854   effect(KILL cr);
10855   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10856 
10857   ins_cost(125);
10858   format %{ "andq    $dst, $src\t# long" %}
10859   ins_encode %{
10860     __ andq($dst$$Address, $src$$constant);
10861   %}
10862   ins_pipe(ialu_mem_imm);
10863 %}
10864 
10865 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10866 %{
10867   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10868   // because AND/OR works well enough for 8/32-bit values.
10869   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10870 
10871   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10872   effect(KILL cr);
10873 
10874   ins_cost(125);
10875   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10876   ins_encode %{
10877     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10878   %}
10879   ins_pipe(ialu_mem_imm);
10880 %}
10881 
10882 // BMI1 instructions
10883 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10884   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10885   predicate(UseBMI1Instructions);
10886   effect(KILL cr);
10887   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10888 
10889   ins_cost(125);
10890   format %{ "andnq  $dst, $src1, $src2" %}
10891 
10892   ins_encode %{
10893     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10894   %}
10895   ins_pipe(ialu_reg_mem);
10896 %}
10897 
10898 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10899   match(Set dst (AndL (XorL src1 minus_1) src2));
10900   predicate(UseBMI1Instructions);
10901   effect(KILL cr);
10902   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10903 
10904   format %{ "andnq  $dst, $src1, $src2" %}
10905 
10906   ins_encode %{
10907   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10908   %}
10909   ins_pipe(ialu_reg_mem);
10910 %}
10911 
10912 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10913   match(Set dst (AndL (SubL imm_zero src) src));
10914   predicate(UseBMI1Instructions);
10915   effect(KILL cr);
10916   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10917 
10918   format %{ "blsiq  $dst, $src" %}
10919 
10920   ins_encode %{
10921     __ blsiq($dst$$Register, $src$$Register);
10922   %}
10923   ins_pipe(ialu_reg);
10924 %}
10925 
10926 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10927   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10928   predicate(UseBMI1Instructions);
10929   effect(KILL cr);
10930   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10931 
10932   ins_cost(125);
10933   format %{ "blsiq  $dst, $src" %}
10934 
10935   ins_encode %{
10936     __ blsiq($dst$$Register, $src$$Address);
10937   %}
10938   ins_pipe(ialu_reg_mem);
10939 %}
10940 
10941 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10942 %{
10943   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10944   predicate(UseBMI1Instructions);
10945   effect(KILL cr);
10946   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10947 
10948   ins_cost(125);
10949   format %{ "blsmskq $dst, $src" %}
10950 
10951   ins_encode %{
10952     __ blsmskq($dst$$Register, $src$$Address);
10953   %}
10954   ins_pipe(ialu_reg_mem);
10955 %}
10956 
10957 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10958 %{
10959   match(Set dst (XorL (AddL src minus_1) src));
10960   predicate(UseBMI1Instructions);
10961   effect(KILL cr);
10962   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10963 
10964   format %{ "blsmskq $dst, $src" %}
10965 
10966   ins_encode %{
10967     __ blsmskq($dst$$Register, $src$$Register);
10968   %}
10969 
10970   ins_pipe(ialu_reg);
10971 %}
10972 
10973 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10974 %{
10975   match(Set dst (AndL (AddL src minus_1) src) );
10976   predicate(UseBMI1Instructions);
10977   effect(KILL cr);
10978   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10979 
10980   format %{ "blsrq  $dst, $src" %}
10981 
10982   ins_encode %{
10983     __ blsrq($dst$$Register, $src$$Register);
10984   %}
10985 
10986   ins_pipe(ialu_reg);
10987 %}
10988 
10989 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10990 %{
10991   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10992   predicate(UseBMI1Instructions);
10993   effect(KILL cr);
10994   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10995 
10996   ins_cost(125);
10997   format %{ "blsrq  $dst, $src" %}
10998 
10999   ins_encode %{
11000     __ blsrq($dst$$Register, $src$$Address);
11001   %}
11002 
11003   ins_pipe(ialu_reg);
11004 %}
11005 
11006 // Or Instructions
11007 // Or Register with Register
11008 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11009 %{
11010   predicate(!UseAPX);
11011   match(Set dst (OrL dst src));
11012   effect(KILL cr);
11013   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11014 
11015   format %{ "orq     $dst, $src\t# long" %}
11016   ins_encode %{
11017     __ orq($dst$$Register, $src$$Register);
11018   %}
11019   ins_pipe(ialu_reg_reg);
11020 %}
11021 
11022 // Or Register with Register using New Data Destination (NDD)
11023 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11024 %{
11025   predicate(UseAPX);
11026   match(Set dst (OrL src1 src2));
11027   effect(KILL cr);
11028   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11029 
11030   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11031   ins_encode %{
11032     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11033 
11034   %}
11035   ins_pipe(ialu_reg_reg);
11036 %}
11037 
11038 // Use any_RegP to match R15 (TLS register) without spilling.
11039 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11040   match(Set dst (OrL dst (CastP2X src)));
11041   effect(KILL cr);
11042   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11043 
11044   format %{ "orq     $dst, $src\t# long" %}
11045   ins_encode %{
11046     __ orq($dst$$Register, $src$$Register);
11047   %}
11048   ins_pipe(ialu_reg_reg);
11049 %}
11050 
11051 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11052   match(Set dst (OrL src1 (CastP2X src2)));
11053   effect(KILL cr);
11054   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11055 
11056   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11057   ins_encode %{
11058     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11059   %}
11060   ins_pipe(ialu_reg_reg);
11061 %}
11062 
11063 // Or Register with Immediate
11064 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11065 %{
11066   predicate(!UseAPX);
11067   match(Set dst (OrL dst src));
11068   effect(KILL cr);
11069   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11070 
11071   format %{ "orq     $dst, $src\t# long" %}
11072   ins_encode %{
11073     __ orq($dst$$Register, $src$$constant);
11074   %}
11075   ins_pipe(ialu_reg);
11076 %}
11077 
11078 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11079 %{
11080   predicate(UseAPX);
11081   match(Set dst (OrL src1 src2));
11082   effect(KILL cr);
11083   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11084 
11085   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11086   ins_encode %{
11087     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11088   %}
11089   ins_pipe(ialu_reg);
11090 %}
11091 
11092 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11093 %{
11094   predicate(UseAPX);
11095   match(Set dst (OrL src1 src2));
11096   effect(KILL cr);
11097   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11098 
11099   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11100   ins_encode %{
11101     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11102   %}
11103   ins_pipe(ialu_reg);
11104 %}
11105 
11106 // Or Memory with Immediate
11107 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11108 %{
11109   predicate(UseAPX);
11110   match(Set dst (OrL (LoadL src1) src2));
11111   effect(KILL cr);
11112   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11113 
11114   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11115   ins_encode %{
11116     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11117   %}
11118   ins_pipe(ialu_reg);
11119 %}
11120 
11121 // Or Register with Memory
11122 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11123 %{
11124   predicate(!UseAPX);
11125   match(Set dst (OrL dst (LoadL src)));
11126   effect(KILL cr);
11127   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11128 
11129   ins_cost(150);
11130   format %{ "orq     $dst, $src\t# long" %}
11131   ins_encode %{
11132     __ orq($dst$$Register, $src$$Address);
11133   %}
11134   ins_pipe(ialu_reg_mem);
11135 %}
11136 
11137 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11138 %{
11139   predicate(UseAPX);
11140   match(Set dst (OrL src1 (LoadL src2)));
11141   effect(KILL cr);
11142   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11143 
11144   ins_cost(150);
11145   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11146   ins_encode %{
11147     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11148   %}
11149   ins_pipe(ialu_reg_mem);
11150 %}
11151 
11152 // Or Memory with Register
11153 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11154 %{
11155   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11156   effect(KILL cr);
11157   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11158 
11159   ins_cost(150);
11160   format %{ "orq     $dst, $src\t# long" %}
11161   ins_encode %{
11162     __ orq($dst$$Address, $src$$Register);
11163   %}
11164   ins_pipe(ialu_mem_reg);
11165 %}
11166 
11167 // Or Memory with Immediate
11168 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11169 %{
11170   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11171   effect(KILL cr);
11172   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11173 
11174   ins_cost(125);
11175   format %{ "orq     $dst, $src\t# long" %}
11176   ins_encode %{
11177     __ orq($dst$$Address, $src$$constant);
11178   %}
11179   ins_pipe(ialu_mem_imm);
11180 %}
11181 
11182 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11183 %{
11184   // con should be a pure 64-bit power of 2 immediate
11185   // because AND/OR works well enough for 8/32-bit values.
11186   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11187 
11188   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11189   effect(KILL cr);
11190 
11191   ins_cost(125);
11192   format %{ "btsq    $dst, log2($con)\t# long" %}
11193   ins_encode %{
11194     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11195   %}
11196   ins_pipe(ialu_mem_imm);
11197 %}
11198 
11199 // Xor Instructions
11200 // Xor Register with Register
11201 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11202 %{
11203   predicate(!UseAPX);
11204   match(Set dst (XorL dst src));
11205   effect(KILL cr);
11206   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11207 
11208   format %{ "xorq    $dst, $src\t# long" %}
11209   ins_encode %{
11210     __ xorq($dst$$Register, $src$$Register);
11211   %}
11212   ins_pipe(ialu_reg_reg);
11213 %}
11214 
11215 // Xor Register with Register using New Data Destination (NDD)
11216 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11217 %{
11218   predicate(UseAPX);
11219   match(Set dst (XorL src1 src2));
11220   effect(KILL cr);
11221   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11222 
11223   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11224   ins_encode %{
11225     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11226   %}
11227   ins_pipe(ialu_reg_reg);
11228 %}
11229 
11230 // Xor Register with Immediate -1
11231 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11232 %{
11233   predicate(!UseAPX);
11234   match(Set dst (XorL dst imm));
11235 
11236   format %{ "notq   $dst" %}
11237   ins_encode %{
11238      __ notq($dst$$Register);
11239   %}
11240   ins_pipe(ialu_reg);
11241 %}
11242 
11243 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11244 %{
11245   predicate(UseAPX);
11246   match(Set dst (XorL src imm));
11247 
11248   format %{ "enotq   $dst, $src" %}
11249   ins_encode %{
11250     __ enotq($dst$$Register, $src$$Register);
11251   %}
11252   ins_pipe(ialu_reg);
11253 %}
11254 
11255 // Xor Register with Immediate
11256 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11257 %{
11258   // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
11259   predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11260   match(Set dst (XorL dst src));
11261   effect(KILL cr);
11262   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11263 
11264   format %{ "xorq    $dst, $src\t# long" %}
11265   ins_encode %{
11266     __ xorq($dst$$Register, $src$$constant);
11267   %}
11268   ins_pipe(ialu_reg);
11269 %}
11270 
11271 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11272 %{
11273   // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
11274   predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11275   match(Set dst (XorL src1 src2));
11276   effect(KILL cr);
11277   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11278 
11279   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11280   ins_encode %{
11281     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11282   %}
11283   ins_pipe(ialu_reg);
11284 %}
11285 
11286 // Xor Memory with Immediate
11287 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11288 %{
11289   predicate(UseAPX);
11290   match(Set dst (XorL (LoadL src1) src2));
11291   effect(KILL cr);
11292   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11293   ins_cost(150);
11294 
11295   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11296   ins_encode %{
11297     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11298   %}
11299   ins_pipe(ialu_reg);
11300 %}
11301 
11302 // Xor Register with Memory
11303 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11304 %{
11305   predicate(!UseAPX);
11306   match(Set dst (XorL dst (LoadL src)));
11307   effect(KILL cr);
11308   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11309 
11310   ins_cost(150);
11311   format %{ "xorq    $dst, $src\t# long" %}
11312   ins_encode %{
11313     __ xorq($dst$$Register, $src$$Address);
11314   %}
11315   ins_pipe(ialu_reg_mem);
11316 %}
11317 
11318 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11319 %{
11320   predicate(UseAPX);
11321   match(Set dst (XorL src1 (LoadL src2)));
11322   effect(KILL cr);
11323   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11324 
11325   ins_cost(150);
11326   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11327   ins_encode %{
11328     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11329   %}
11330   ins_pipe(ialu_reg_mem);
11331 %}
11332 
11333 // Xor Memory with Register
11334 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11335 %{
11336   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11337   effect(KILL cr);
11338   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11339 
11340   ins_cost(150);
11341   format %{ "xorq    $dst, $src\t# long" %}
11342   ins_encode %{
11343     __ xorq($dst$$Address, $src$$Register);
11344   %}
11345   ins_pipe(ialu_mem_reg);
11346 %}
11347 
11348 // Xor Memory with Immediate
11349 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11350 %{
11351   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11352   effect(KILL cr);
11353   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11354 
11355   ins_cost(125);
11356   format %{ "xorq    $dst, $src\t# long" %}
11357   ins_encode %{
11358     __ xorq($dst$$Address, $src$$constant);
11359   %}
11360   ins_pipe(ialu_mem_imm);
11361 %}
11362 
11363 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11364 %{
11365   match(Set dst (CmpLTMask p q));
11366   effect(KILL cr);
11367 
11368   ins_cost(400);
11369   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11370             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11371             "negl    $dst" %}
11372   ins_encode %{
11373     __ cmpl($p$$Register, $q$$Register);
11374     __ setcc(Assembler::less, $dst$$Register);
11375     __ negl($dst$$Register);
11376   %}
11377   ins_pipe(pipe_slow);
11378 %}
11379 
11380 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11381 %{
11382   match(Set dst (CmpLTMask dst zero));
11383   effect(KILL cr);
11384 
11385   ins_cost(100);
11386   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11387   ins_encode %{
11388     __ sarl($dst$$Register, 31);
11389   %}
11390   ins_pipe(ialu_reg);
11391 %}
11392 
11393 /* Better to save a register than avoid a branch */
11394 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11395 %{
11396   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11397   effect(KILL cr);
11398   ins_cost(300);
11399   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11400             "jge     done\n\t"
11401             "addl    $p,$y\n"
11402             "done:   " %}
11403   ins_encode %{
11404     Register Rp = $p$$Register;
11405     Register Rq = $q$$Register;
11406     Register Ry = $y$$Register;
11407     Label done;
11408     __ subl(Rp, Rq);
11409     __ jccb(Assembler::greaterEqual, done);
11410     __ addl(Rp, Ry);
11411     __ bind(done);
11412   %}
11413   ins_pipe(pipe_cmplt);
11414 %}
11415 
11416 /* Better to save a register than avoid a branch */
11417 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11418 %{
11419   match(Set y (AndI (CmpLTMask p q) y));
11420   effect(KILL cr);
11421 
11422   ins_cost(300);
11423 
11424   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11425             "jlt     done\n\t"
11426             "xorl    $y, $y\n"
11427             "done:   " %}
11428   ins_encode %{
11429     Register Rp = $p$$Register;
11430     Register Rq = $q$$Register;
11431     Register Ry = $y$$Register;
11432     Label done;
11433     __ cmpl(Rp, Rq);
11434     __ jccb(Assembler::less, done);
11435     __ xorl(Ry, Ry);
11436     __ bind(done);
11437   %}
11438   ins_pipe(pipe_cmplt);
11439 %}
11440 
11441 
11442 //---------- FP Instructions------------------------------------------------
11443 
11444 // Really expensive, avoid
11445 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11446 %{
11447   match(Set cr (CmpF src1 src2));
11448 
11449   ins_cost(500);
11450   format %{ "ucomiss $src1, $src2\n\t"
11451             "jnp,s   exit\n\t"
11452             "pushfq\t# saw NaN, set CF\n\t"
11453             "andq    [rsp], #0xffffff2b\n\t"
11454             "popfq\n"
11455     "exit:" %}
11456   ins_encode %{
11457     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11458     emit_cmpfp_fixup(masm);
11459   %}
11460   ins_pipe(pipe_slow);
11461 %}
11462 
11463 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11464   match(Set cr (CmpF src1 src2));
11465 
11466   ins_cost(100);
11467   format %{ "ucomiss $src1, $src2" %}
11468   ins_encode %{
11469     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11470   %}
11471   ins_pipe(pipe_slow);
11472 %}
11473 
11474 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11475   match(Set cr (CmpF src1 (LoadF src2)));
11476 
11477   ins_cost(100);
11478   format %{ "ucomiss $src1, $src2" %}
11479   ins_encode %{
11480     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11481   %}
11482   ins_pipe(pipe_slow);
11483 %}
11484 
11485 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11486   match(Set cr (CmpF src con));
11487   ins_cost(100);
11488   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11489   ins_encode %{
11490     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11491   %}
11492   ins_pipe(pipe_slow);
11493 %}
11494 
11495 // Really expensive, avoid
11496 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11497 %{
11498   match(Set cr (CmpD src1 src2));
11499 
11500   ins_cost(500);
11501   format %{ "ucomisd $src1, $src2\n\t"
11502             "jnp,s   exit\n\t"
11503             "pushfq\t# saw NaN, set CF\n\t"
11504             "andq    [rsp], #0xffffff2b\n\t"
11505             "popfq\n"
11506     "exit:" %}
11507   ins_encode %{
11508     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11509     emit_cmpfp_fixup(masm);
11510   %}
11511   ins_pipe(pipe_slow);
11512 %}
11513 
11514 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11515   match(Set cr (CmpD src1 src2));
11516 
11517   ins_cost(100);
11518   format %{ "ucomisd $src1, $src2 test" %}
11519   ins_encode %{
11520     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11521   %}
11522   ins_pipe(pipe_slow);
11523 %}
11524 
11525 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11526   match(Set cr (CmpD src1 (LoadD src2)));
11527 
11528   ins_cost(100);
11529   format %{ "ucomisd $src1, $src2" %}
11530   ins_encode %{
11531     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11532   %}
11533   ins_pipe(pipe_slow);
11534 %}
11535 
11536 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11537   match(Set cr (CmpD src con));
11538   ins_cost(100);
11539   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11540   ins_encode %{
11541     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11542   %}
11543   ins_pipe(pipe_slow);
11544 %}
11545 
11546 // Compare into -1,0,1
11547 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11548 %{
11549   match(Set dst (CmpF3 src1 src2));
11550   effect(KILL cr);
11551 
11552   ins_cost(275);
11553   format %{ "ucomiss $src1, $src2\n\t"
11554             "movl    $dst, #-1\n\t"
11555             "jp,s    done\n\t"
11556             "jb,s    done\n\t"
11557             "setne   $dst\n\t"
11558             "movzbl  $dst, $dst\n"
11559     "done:" %}
11560   ins_encode %{
11561     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11562     emit_cmpfp3(masm, $dst$$Register);
11563   %}
11564   ins_pipe(pipe_slow);
11565 %}
11566 
11567 // Compare into -1,0,1
11568 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11569 %{
11570   match(Set dst (CmpF3 src1 (LoadF src2)));
11571   effect(KILL cr);
11572 
11573   ins_cost(275);
11574   format %{ "ucomiss $src1, $src2\n\t"
11575             "movl    $dst, #-1\n\t"
11576             "jp,s    done\n\t"
11577             "jb,s    done\n\t"
11578             "setne   $dst\n\t"
11579             "movzbl  $dst, $dst\n"
11580     "done:" %}
11581   ins_encode %{
11582     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11583     emit_cmpfp3(masm, $dst$$Register);
11584   %}
11585   ins_pipe(pipe_slow);
11586 %}
11587 
11588 // Compare into -1,0,1
11589 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11590   match(Set dst (CmpF3 src con));
11591   effect(KILL cr);
11592 
11593   ins_cost(275);
11594   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11595             "movl    $dst, #-1\n\t"
11596             "jp,s    done\n\t"
11597             "jb,s    done\n\t"
11598             "setne   $dst\n\t"
11599             "movzbl  $dst, $dst\n"
11600     "done:" %}
11601   ins_encode %{
11602     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11603     emit_cmpfp3(masm, $dst$$Register);
11604   %}
11605   ins_pipe(pipe_slow);
11606 %}
11607 
11608 // Compare into -1,0,1
11609 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11610 %{
11611   match(Set dst (CmpD3 src1 src2));
11612   effect(KILL cr);
11613 
11614   ins_cost(275);
11615   format %{ "ucomisd $src1, $src2\n\t"
11616             "movl    $dst, #-1\n\t"
11617             "jp,s    done\n\t"
11618             "jb,s    done\n\t"
11619             "setne   $dst\n\t"
11620             "movzbl  $dst, $dst\n"
11621     "done:" %}
11622   ins_encode %{
11623     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11624     emit_cmpfp3(masm, $dst$$Register);
11625   %}
11626   ins_pipe(pipe_slow);
11627 %}
11628 
11629 // Compare into -1,0,1
11630 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11631 %{
11632   match(Set dst (CmpD3 src1 (LoadD src2)));
11633   effect(KILL cr);
11634 
11635   ins_cost(275);
11636   format %{ "ucomisd $src1, $src2\n\t"
11637             "movl    $dst, #-1\n\t"
11638             "jp,s    done\n\t"
11639             "jb,s    done\n\t"
11640             "setne   $dst\n\t"
11641             "movzbl  $dst, $dst\n"
11642     "done:" %}
11643   ins_encode %{
11644     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11645     emit_cmpfp3(masm, $dst$$Register);
11646   %}
11647   ins_pipe(pipe_slow);
11648 %}
11649 
11650 // Compare into -1,0,1
11651 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11652   match(Set dst (CmpD3 src con));
11653   effect(KILL cr);
11654 
11655   ins_cost(275);
11656   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11657             "movl    $dst, #-1\n\t"
11658             "jp,s    done\n\t"
11659             "jb,s    done\n\t"
11660             "setne   $dst\n\t"
11661             "movzbl  $dst, $dst\n"
11662     "done:" %}
11663   ins_encode %{
11664     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11665     emit_cmpfp3(masm, $dst$$Register);
11666   %}
11667   ins_pipe(pipe_slow);
11668 %}
11669 
11670 //----------Arithmetic Conversion Instructions---------------------------------
11671 
11672 instruct convF2D_reg_reg(regD dst, regF src)
11673 %{
11674   match(Set dst (ConvF2D src));
11675 
11676   format %{ "cvtss2sd $dst, $src" %}
11677   ins_encode %{
11678     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11679   %}
11680   ins_pipe(pipe_slow); // XXX
11681 %}
11682 
11683 instruct convF2D_reg_mem(regD dst, memory src)
11684 %{
11685   predicate(UseAVX == 0);
11686   match(Set dst (ConvF2D (LoadF src)));
11687 
11688   format %{ "cvtss2sd $dst, $src" %}
11689   ins_encode %{
11690     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11691   %}
11692   ins_pipe(pipe_slow); // XXX
11693 %}
11694 
11695 instruct convD2F_reg_reg(regF dst, regD src)
11696 %{
11697   match(Set dst (ConvD2F src));
11698 
11699   format %{ "cvtsd2ss $dst, $src" %}
11700   ins_encode %{
11701     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11702   %}
11703   ins_pipe(pipe_slow); // XXX
11704 %}
11705 
11706 instruct convD2F_reg_mem(regF dst, memory src)
11707 %{
11708   predicate(UseAVX == 0);
11709   match(Set dst (ConvD2F (LoadD src)));
11710 
11711   format %{ "cvtsd2ss $dst, $src" %}
11712   ins_encode %{
11713     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11714   %}
11715   ins_pipe(pipe_slow); // XXX
11716 %}
11717 
11718 // XXX do mem variants
11719 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11720 %{
11721   match(Set dst (ConvF2I src));
11722   effect(KILL cr);
11723   format %{ "convert_f2i $dst, $src" %}
11724   ins_encode %{
11725     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11726   %}
11727   ins_pipe(pipe_slow);
11728 %}
11729 
11730 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11731 %{
11732   match(Set dst (ConvF2L src));
11733   effect(KILL cr);
11734   format %{ "convert_f2l $dst, $src"%}
11735   ins_encode %{
11736     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11737   %}
11738   ins_pipe(pipe_slow);
11739 %}
11740 
11741 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11742 %{
11743   match(Set dst (ConvD2I src));
11744   effect(KILL cr);
11745   format %{ "convert_d2i $dst, $src"%}
11746   ins_encode %{
11747     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11748   %}
11749   ins_pipe(pipe_slow);
11750 %}
11751 
11752 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11753 %{
11754   match(Set dst (ConvD2L src));
11755   effect(KILL cr);
11756   format %{ "convert_d2l $dst, $src"%}
11757   ins_encode %{
11758     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11759   %}
11760   ins_pipe(pipe_slow);
11761 %}
11762 
11763 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11764 %{
11765   match(Set dst (RoundD src));
11766   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11767   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11768   ins_encode %{
11769     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11770   %}
11771   ins_pipe(pipe_slow);
11772 %}
11773 
11774 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11775 %{
11776   match(Set dst (RoundF src));
11777   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11778   format %{ "round_float $dst,$src" %}
11779   ins_encode %{
11780     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11781   %}
11782   ins_pipe(pipe_slow);
11783 %}
11784 
11785 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11786 %{
11787   predicate(!UseXmmI2F);
11788   match(Set dst (ConvI2F src));
11789 
11790   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11791   ins_encode %{
11792     if (UseAVX > 0) {
11793       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11794     }
11795     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11796   %}
11797   ins_pipe(pipe_slow); // XXX
11798 %}
11799 
11800 instruct convI2F_reg_mem(regF dst, memory src)
11801 %{
11802   predicate(UseAVX == 0);
11803   match(Set dst (ConvI2F (LoadI src)));
11804 
11805   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11806   ins_encode %{
11807     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11808   %}
11809   ins_pipe(pipe_slow); // XXX
11810 %}
11811 
11812 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11813 %{
11814   predicate(!UseXmmI2D);
11815   match(Set dst (ConvI2D src));
11816 
11817   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11818   ins_encode %{
11819     if (UseAVX > 0) {
11820       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11821     }
11822     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11823   %}
11824   ins_pipe(pipe_slow); // XXX
11825 %}
11826 
11827 instruct convI2D_reg_mem(regD dst, memory src)
11828 %{
11829   predicate(UseAVX == 0);
11830   match(Set dst (ConvI2D (LoadI src)));
11831 
11832   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11833   ins_encode %{
11834     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11835   %}
11836   ins_pipe(pipe_slow); // XXX
11837 %}
11838 
11839 instruct convXI2F_reg(regF dst, rRegI src)
11840 %{
11841   predicate(UseXmmI2F);
11842   match(Set dst (ConvI2F src));
11843 
11844   format %{ "movdl $dst, $src\n\t"
11845             "cvtdq2psl $dst, $dst\t# i2f" %}
11846   ins_encode %{
11847     __ movdl($dst$$XMMRegister, $src$$Register);
11848     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11849   %}
11850   ins_pipe(pipe_slow); // XXX
11851 %}
11852 
11853 instruct convXI2D_reg(regD dst, rRegI src)
11854 %{
11855   predicate(UseXmmI2D);
11856   match(Set dst (ConvI2D src));
11857 
11858   format %{ "movdl $dst, $src\n\t"
11859             "cvtdq2pdl $dst, $dst\t# i2d" %}
11860   ins_encode %{
11861     __ movdl($dst$$XMMRegister, $src$$Register);
11862     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11863   %}
11864   ins_pipe(pipe_slow); // XXX
11865 %}
11866 
11867 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11868 %{
11869   match(Set dst (ConvL2F src));
11870 
11871   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11872   ins_encode %{
11873     if (UseAVX > 0) {
11874       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11875     }
11876     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11877   %}
11878   ins_pipe(pipe_slow); // XXX
11879 %}
11880 
11881 instruct convL2F_reg_mem(regF dst, memory src)
11882 %{
11883   predicate(UseAVX == 0);
11884   match(Set dst (ConvL2F (LoadL src)));
11885 
11886   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11887   ins_encode %{
11888     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11889   %}
11890   ins_pipe(pipe_slow); // XXX
11891 %}
11892 
11893 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11894 %{
11895   match(Set dst (ConvL2D src));
11896 
11897   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11898   ins_encode %{
11899     if (UseAVX > 0) {
11900       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11901     }
11902     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11903   %}
11904   ins_pipe(pipe_slow); // XXX
11905 %}
11906 
11907 instruct convL2D_reg_mem(regD dst, memory src)
11908 %{
11909   predicate(UseAVX == 0);
11910   match(Set dst (ConvL2D (LoadL src)));
11911 
11912   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11913   ins_encode %{
11914     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11915   %}
11916   ins_pipe(pipe_slow); // XXX
11917 %}
11918 
11919 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11920 %{
11921   match(Set dst (ConvI2L src));
11922 
11923   ins_cost(125);
11924   format %{ "movslq  $dst, $src\t# i2l" %}
11925   ins_encode %{
11926     __ movslq($dst$$Register, $src$$Register);
11927   %}
11928   ins_pipe(ialu_reg_reg);
11929 %}
11930 
11931 // Zero-extend convert int to long
11932 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11933 %{
11934   match(Set dst (AndL (ConvI2L src) mask));
11935 
11936   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11937   ins_encode %{
11938     if ($dst$$reg != $src$$reg) {
11939       __ movl($dst$$Register, $src$$Register);
11940     }
11941   %}
11942   ins_pipe(ialu_reg_reg);
11943 %}
11944 
11945 // Zero-extend convert int to long
11946 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11947 %{
11948   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11949 
11950   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11951   ins_encode %{
11952     __ movl($dst$$Register, $src$$Address);
11953   %}
11954   ins_pipe(ialu_reg_mem);
11955 %}
11956 
11957 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11958 %{
11959   match(Set dst (AndL src mask));
11960 
11961   format %{ "movl    $dst, $src\t# zero-extend long" %}
11962   ins_encode %{
11963     __ movl($dst$$Register, $src$$Register);
11964   %}
11965   ins_pipe(ialu_reg_reg);
11966 %}
11967 
11968 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11969 %{
11970   match(Set dst (ConvL2I src));
11971 
11972   format %{ "movl    $dst, $src\t# l2i" %}
11973   ins_encode %{
11974     __ movl($dst$$Register, $src$$Register);
11975   %}
11976   ins_pipe(ialu_reg_reg);
11977 %}
11978 
11979 
11980 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11981   match(Set dst (MoveF2I src));
11982   effect(DEF dst, USE src);
11983 
11984   ins_cost(125);
11985   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11986   ins_encode %{
11987     __ movl($dst$$Register, Address(rsp, $src$$disp));
11988   %}
11989   ins_pipe(ialu_reg_mem);
11990 %}
11991 
11992 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11993   match(Set dst (MoveI2F src));
11994   effect(DEF dst, USE src);
11995 
11996   ins_cost(125);
11997   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11998   ins_encode %{
11999     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
12000   %}
12001   ins_pipe(pipe_slow);
12002 %}
12003 
12004 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12005   match(Set dst (MoveD2L src));
12006   effect(DEF dst, USE src);
12007 
12008   ins_cost(125);
12009   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12010   ins_encode %{
12011     __ movq($dst$$Register, Address(rsp, $src$$disp));
12012   %}
12013   ins_pipe(ialu_reg_mem);
12014 %}
12015 
12016 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12017   predicate(!UseXmmLoadAndClearUpper);
12018   match(Set dst (MoveL2D src));
12019   effect(DEF dst, USE src);
12020 
12021   ins_cost(125);
12022   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12023   ins_encode %{
12024     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12025   %}
12026   ins_pipe(pipe_slow);
12027 %}
12028 
12029 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12030   predicate(UseXmmLoadAndClearUpper);
12031   match(Set dst (MoveL2D src));
12032   effect(DEF dst, USE src);
12033 
12034   ins_cost(125);
12035   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12036   ins_encode %{
12037     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12038   %}
12039   ins_pipe(pipe_slow);
12040 %}
12041 
12042 
12043 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12044   match(Set dst (MoveF2I src));
12045   effect(DEF dst, USE src);
12046 
12047   ins_cost(95); // XXX
12048   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12049   ins_encode %{
12050     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12051   %}
12052   ins_pipe(pipe_slow);
12053 %}
12054 
12055 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12056   match(Set dst (MoveI2F src));
12057   effect(DEF dst, USE src);
12058 
12059   ins_cost(100);
12060   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12061   ins_encode %{
12062     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12063   %}
12064   ins_pipe( ialu_mem_reg );
12065 %}
12066 
12067 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12068   match(Set dst (MoveD2L src));
12069   effect(DEF dst, USE src);
12070 
12071   ins_cost(95); // XXX
12072   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12073   ins_encode %{
12074     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12075   %}
12076   ins_pipe(pipe_slow);
12077 %}
12078 
12079 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12080   match(Set dst (MoveL2D src));
12081   effect(DEF dst, USE src);
12082 
12083   ins_cost(100);
12084   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12085   ins_encode %{
12086     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12087   %}
12088   ins_pipe(ialu_mem_reg);
12089 %}
12090 
12091 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12092   match(Set dst (MoveF2I src));
12093   effect(DEF dst, USE src);
12094   ins_cost(85);
12095   format %{ "movd    $dst,$src\t# MoveF2I" %}
12096   ins_encode %{
12097     __ movdl($dst$$Register, $src$$XMMRegister);
12098   %}
12099   ins_pipe( pipe_slow );
12100 %}
12101 
12102 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12103   match(Set dst (MoveD2L src));
12104   effect(DEF dst, USE src);
12105   ins_cost(85);
12106   format %{ "movd    $dst,$src\t# MoveD2L" %}
12107   ins_encode %{
12108     __ movdq($dst$$Register, $src$$XMMRegister);
12109   %}
12110   ins_pipe( pipe_slow );
12111 %}
12112 
12113 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12114   match(Set dst (MoveI2F src));
12115   effect(DEF dst, USE src);
12116   ins_cost(100);
12117   format %{ "movd    $dst,$src\t# MoveI2F" %}
12118   ins_encode %{
12119     __ movdl($dst$$XMMRegister, $src$$Register);
12120   %}
12121   ins_pipe( pipe_slow );
12122 %}
12123 
12124 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12125   match(Set dst (MoveL2D src));
12126   effect(DEF dst, USE src);
12127   ins_cost(100);
12128   format %{ "movd    $dst,$src\t# MoveL2D" %}
12129   ins_encode %{
12130      __ movdq($dst$$XMMRegister, $src$$Register);
12131   %}
12132   ins_pipe( pipe_slow );
12133 %}
12134 
12135 // Fast clearing of an array
12136 // Small non-constant lenght ClearArray for non-AVX512 targets.
12137 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12138                   Universe dummy, rFlagsReg cr)
12139 %{
12140   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12141   match(Set dummy (ClearArray cnt base));
12142   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12143 
12144   format %{ $$template
12145     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12146     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12147     $$emit$$"jg      LARGE\n\t"
12148     $$emit$$"dec     rcx\n\t"
12149     $$emit$$"js      DONE\t# Zero length\n\t"
12150     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12151     $$emit$$"dec     rcx\n\t"
12152     $$emit$$"jge     LOOP\n\t"
12153     $$emit$$"jmp     DONE\n\t"
12154     $$emit$$"# LARGE:\n\t"
12155     if (UseFastStosb) {
12156        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12157        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12158     } else if (UseXMMForObjInit) {
12159        $$emit$$"mov     rdi,rax\n\t"
12160        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12161        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12162        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12163        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12164        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12165        $$emit$$"add     0x40,rax\n\t"
12166        $$emit$$"# L_zero_64_bytes:\n\t"
12167        $$emit$$"sub     0x8,rcx\n\t"
12168        $$emit$$"jge     L_loop\n\t"
12169        $$emit$$"add     0x4,rcx\n\t"
12170        $$emit$$"jl      L_tail\n\t"
12171        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12172        $$emit$$"add     0x20,rax\n\t"
12173        $$emit$$"sub     0x4,rcx\n\t"
12174        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12175        $$emit$$"add     0x4,rcx\n\t"
12176        $$emit$$"jle     L_end\n\t"
12177        $$emit$$"dec     rcx\n\t"
12178        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12179        $$emit$$"vmovq   xmm0,(rax)\n\t"
12180        $$emit$$"add     0x8,rax\n\t"
12181        $$emit$$"dec     rcx\n\t"
12182        $$emit$$"jge     L_sloop\n\t"
12183        $$emit$$"# L_end:\n\t"
12184     } else {
12185        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12186     }
12187     $$emit$$"# DONE"
12188   %}
12189   ins_encode %{
12190     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12191                  $tmp$$XMMRegister, false, knoreg);
12192   %}
12193   ins_pipe(pipe_slow);
12194 %}
12195 
12196 // Small non-constant length ClearArray for AVX512 targets.
12197 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12198                        Universe dummy, rFlagsReg cr)
12199 %{
12200   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12201   match(Set dummy (ClearArray cnt base));
12202   ins_cost(125);
12203   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12204 
12205   format %{ $$template
12206     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12207     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12208     $$emit$$"jg      LARGE\n\t"
12209     $$emit$$"dec     rcx\n\t"
12210     $$emit$$"js      DONE\t# Zero length\n\t"
12211     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12212     $$emit$$"dec     rcx\n\t"
12213     $$emit$$"jge     LOOP\n\t"
12214     $$emit$$"jmp     DONE\n\t"
12215     $$emit$$"# LARGE:\n\t"
12216     if (UseFastStosb) {
12217        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12218        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12219     } else if (UseXMMForObjInit) {
12220        $$emit$$"mov     rdi,rax\n\t"
12221        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12222        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12223        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12224        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12225        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12226        $$emit$$"add     0x40,rax\n\t"
12227        $$emit$$"# L_zero_64_bytes:\n\t"
12228        $$emit$$"sub     0x8,rcx\n\t"
12229        $$emit$$"jge     L_loop\n\t"
12230        $$emit$$"add     0x4,rcx\n\t"
12231        $$emit$$"jl      L_tail\n\t"
12232        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12233        $$emit$$"add     0x20,rax\n\t"
12234        $$emit$$"sub     0x4,rcx\n\t"
12235        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12236        $$emit$$"add     0x4,rcx\n\t"
12237        $$emit$$"jle     L_end\n\t"
12238        $$emit$$"dec     rcx\n\t"
12239        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12240        $$emit$$"vmovq   xmm0,(rax)\n\t"
12241        $$emit$$"add     0x8,rax\n\t"
12242        $$emit$$"dec     rcx\n\t"
12243        $$emit$$"jge     L_sloop\n\t"
12244        $$emit$$"# L_end:\n\t"
12245     } else {
12246        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12247     }
12248     $$emit$$"# DONE"
12249   %}
12250   ins_encode %{
12251     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12252                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12253   %}
12254   ins_pipe(pipe_slow);
12255 %}
12256 
12257 // Large non-constant length ClearArray for non-AVX512 targets.
12258 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12259                         Universe dummy, rFlagsReg cr)
12260 %{
12261   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12262   match(Set dummy (ClearArray cnt base));
12263   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12264 
12265   format %{ $$template
12266     if (UseFastStosb) {
12267        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12268        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12269        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12270     } else if (UseXMMForObjInit) {
12271        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12272        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12273        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12274        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12275        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12276        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12277        $$emit$$"add     0x40,rax\n\t"
12278        $$emit$$"# L_zero_64_bytes:\n\t"
12279        $$emit$$"sub     0x8,rcx\n\t"
12280        $$emit$$"jge     L_loop\n\t"
12281        $$emit$$"add     0x4,rcx\n\t"
12282        $$emit$$"jl      L_tail\n\t"
12283        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12284        $$emit$$"add     0x20,rax\n\t"
12285        $$emit$$"sub     0x4,rcx\n\t"
12286        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12287        $$emit$$"add     0x4,rcx\n\t"
12288        $$emit$$"jle     L_end\n\t"
12289        $$emit$$"dec     rcx\n\t"
12290        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12291        $$emit$$"vmovq   xmm0,(rax)\n\t"
12292        $$emit$$"add     0x8,rax\n\t"
12293        $$emit$$"dec     rcx\n\t"
12294        $$emit$$"jge     L_sloop\n\t"
12295        $$emit$$"# L_end:\n\t"
12296     } else {
12297        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12298        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12299     }
12300   %}
12301   ins_encode %{
12302     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12303                  $tmp$$XMMRegister, true, knoreg);
12304   %}
12305   ins_pipe(pipe_slow);
12306 %}
12307 
12308 // Large non-constant length ClearArray for AVX512 targets.
12309 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12310                              Universe dummy, rFlagsReg cr)
12311 %{
12312   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12313   match(Set dummy (ClearArray cnt base));
12314   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12315 
12316   format %{ $$template
12317     if (UseFastStosb) {
12318        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12319        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12320        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12321     } else if (UseXMMForObjInit) {
12322        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12323        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12324        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12325        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12326        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12327        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12328        $$emit$$"add     0x40,rax\n\t"
12329        $$emit$$"# L_zero_64_bytes:\n\t"
12330        $$emit$$"sub     0x8,rcx\n\t"
12331        $$emit$$"jge     L_loop\n\t"
12332        $$emit$$"add     0x4,rcx\n\t"
12333        $$emit$$"jl      L_tail\n\t"
12334        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12335        $$emit$$"add     0x20,rax\n\t"
12336        $$emit$$"sub     0x4,rcx\n\t"
12337        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12338        $$emit$$"add     0x4,rcx\n\t"
12339        $$emit$$"jle     L_end\n\t"
12340        $$emit$$"dec     rcx\n\t"
12341        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12342        $$emit$$"vmovq   xmm0,(rax)\n\t"
12343        $$emit$$"add     0x8,rax\n\t"
12344        $$emit$$"dec     rcx\n\t"
12345        $$emit$$"jge     L_sloop\n\t"
12346        $$emit$$"# L_end:\n\t"
12347     } else {
12348        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12349        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12350     }
12351   %}
12352   ins_encode %{
12353     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12354                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12355   %}
12356   ins_pipe(pipe_slow);
12357 %}
12358 
12359 // Small constant length ClearArray for AVX512 targets.
12360 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12361 %{
12362   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12363   match(Set dummy (ClearArray cnt base));
12364   ins_cost(100);
12365   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12366   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12367   ins_encode %{
12368    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12369   %}
12370   ins_pipe(pipe_slow);
12371 %}
12372 
12373 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12374                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12375 %{
12376   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12377   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12378   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12379 
12380   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12381   ins_encode %{
12382     __ string_compare($str1$$Register, $str2$$Register,
12383                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12384                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12385   %}
12386   ins_pipe( pipe_slow );
12387 %}
12388 
12389 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12390                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12391 %{
12392   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12393   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12394   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12395 
12396   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12397   ins_encode %{
12398     __ string_compare($str1$$Register, $str2$$Register,
12399                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12400                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12401   %}
12402   ins_pipe( pipe_slow );
12403 %}
12404 
12405 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12406                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12407 %{
12408   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12409   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12410   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12411 
12412   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12413   ins_encode %{
12414     __ string_compare($str1$$Register, $str2$$Register,
12415                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12416                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12417   %}
12418   ins_pipe( pipe_slow );
12419 %}
12420 
12421 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12422                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12423 %{
12424   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12425   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12426   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12427 
12428   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12429   ins_encode %{
12430     __ string_compare($str1$$Register, $str2$$Register,
12431                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12432                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12433   %}
12434   ins_pipe( pipe_slow );
12435 %}
12436 
12437 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12438                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12439 %{
12440   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12441   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12442   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12443 
12444   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12445   ins_encode %{
12446     __ string_compare($str1$$Register, $str2$$Register,
12447                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12448                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12449   %}
12450   ins_pipe( pipe_slow );
12451 %}
12452 
12453 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12454                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12455 %{
12456   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12457   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12458   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12459 
12460   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12461   ins_encode %{
12462     __ string_compare($str1$$Register, $str2$$Register,
12463                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12464                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12465   %}
12466   ins_pipe( pipe_slow );
12467 %}
12468 
12469 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12470                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12471 %{
12472   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12473   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12474   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12475 
12476   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12477   ins_encode %{
12478     __ string_compare($str2$$Register, $str1$$Register,
12479                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12480                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12481   %}
12482   ins_pipe( pipe_slow );
12483 %}
12484 
12485 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12486                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12487 %{
12488   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12489   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12490   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12491 
12492   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12493   ins_encode %{
12494     __ string_compare($str2$$Register, $str1$$Register,
12495                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12496                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12497   %}
12498   ins_pipe( pipe_slow );
12499 %}
12500 
12501 // fast search of substring with known size.
12502 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12503                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12504 %{
12505   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12506   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12507   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12508 
12509   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12510   ins_encode %{
12511     int icnt2 = (int)$int_cnt2$$constant;
12512     if (icnt2 >= 16) {
12513       // IndexOf for constant substrings with size >= 16 elements
12514       // which don't need to be loaded through stack.
12515       __ string_indexofC8($str1$$Register, $str2$$Register,
12516                           $cnt1$$Register, $cnt2$$Register,
12517                           icnt2, $result$$Register,
12518                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12519     } else {
12520       // Small strings are loaded through stack if they cross page boundary.
12521       __ string_indexof($str1$$Register, $str2$$Register,
12522                         $cnt1$$Register, $cnt2$$Register,
12523                         icnt2, $result$$Register,
12524                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12525     }
12526   %}
12527   ins_pipe( pipe_slow );
12528 %}
12529 
12530 // fast search of substring with known size.
12531 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12532                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12533 %{
12534   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12535   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12536   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12537 
12538   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12539   ins_encode %{
12540     int icnt2 = (int)$int_cnt2$$constant;
12541     if (icnt2 >= 8) {
12542       // IndexOf for constant substrings with size >= 8 elements
12543       // which don't need to be loaded through stack.
12544       __ string_indexofC8($str1$$Register, $str2$$Register,
12545                           $cnt1$$Register, $cnt2$$Register,
12546                           icnt2, $result$$Register,
12547                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12548     } else {
12549       // Small strings are loaded through stack if they cross page boundary.
12550       __ string_indexof($str1$$Register, $str2$$Register,
12551                         $cnt1$$Register, $cnt2$$Register,
12552                         icnt2, $result$$Register,
12553                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12554     }
12555   %}
12556   ins_pipe( pipe_slow );
12557 %}
12558 
12559 // fast search of substring with known size.
12560 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12561                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12562 %{
12563   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12564   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12565   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12566 
12567   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12568   ins_encode %{
12569     int icnt2 = (int)$int_cnt2$$constant;
12570     if (icnt2 >= 8) {
12571       // IndexOf for constant substrings with size >= 8 elements
12572       // which don't need to be loaded through stack.
12573       __ string_indexofC8($str1$$Register, $str2$$Register,
12574                           $cnt1$$Register, $cnt2$$Register,
12575                           icnt2, $result$$Register,
12576                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12577     } else {
12578       // Small strings are loaded through stack if they cross page boundary.
12579       __ string_indexof($str1$$Register, $str2$$Register,
12580                         $cnt1$$Register, $cnt2$$Register,
12581                         icnt2, $result$$Register,
12582                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12583     }
12584   %}
12585   ins_pipe( pipe_slow );
12586 %}
12587 
12588 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12589                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12590 %{
12591   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12592   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12593   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12594 
12595   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12596   ins_encode %{
12597     __ string_indexof($str1$$Register, $str2$$Register,
12598                       $cnt1$$Register, $cnt2$$Register,
12599                       (-1), $result$$Register,
12600                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12601   %}
12602   ins_pipe( pipe_slow );
12603 %}
12604 
12605 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12606                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12607 %{
12608   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12609   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12610   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12611 
12612   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12613   ins_encode %{
12614     __ string_indexof($str1$$Register, $str2$$Register,
12615                       $cnt1$$Register, $cnt2$$Register,
12616                       (-1), $result$$Register,
12617                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12618   %}
12619   ins_pipe( pipe_slow );
12620 %}
12621 
12622 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12623                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12624 %{
12625   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12626   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12627   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12628 
12629   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12630   ins_encode %{
12631     __ string_indexof($str1$$Register, $str2$$Register,
12632                       $cnt1$$Register, $cnt2$$Register,
12633                       (-1), $result$$Register,
12634                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12635   %}
12636   ins_pipe( pipe_slow );
12637 %}
12638 
12639 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12640                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12641 %{
12642   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12643   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12644   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12645   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12646   ins_encode %{
12647     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12648                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12649   %}
12650   ins_pipe( pipe_slow );
12651 %}
12652 
12653 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12654                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12655 %{
12656   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12657   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12658   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12659   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12660   ins_encode %{
12661     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12662                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12663   %}
12664   ins_pipe( pipe_slow );
12665 %}
12666 
12667 // fast string equals
12668 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12669                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12670 %{
12671   predicate(!VM_Version::supports_avx512vlbw());
12672   match(Set result (StrEquals (Binary str1 str2) cnt));
12673   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12674 
12675   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12676   ins_encode %{
12677     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12678                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12679                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12680   %}
12681   ins_pipe( pipe_slow );
12682 %}
12683 
12684 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12685                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12686 %{
12687   predicate(VM_Version::supports_avx512vlbw());
12688   match(Set result (StrEquals (Binary str1 str2) cnt));
12689   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12690 
12691   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12692   ins_encode %{
12693     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12694                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12695                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12696   %}
12697   ins_pipe( pipe_slow );
12698 %}
12699 
12700 // fast array equals
12701 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12702                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12703 %{
12704   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12705   match(Set result (AryEq ary1 ary2));
12706   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12707 
12708   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12709   ins_encode %{
12710     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12711                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12712                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12713   %}
12714   ins_pipe( pipe_slow );
12715 %}
12716 
12717 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12718                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12719 %{
12720   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12721   match(Set result (AryEq ary1 ary2));
12722   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12723 
12724   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12725   ins_encode %{
12726     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12727                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12728                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12729   %}
12730   ins_pipe( pipe_slow );
12731 %}
12732 
12733 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12734                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12735 %{
12736   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12737   match(Set result (AryEq ary1 ary2));
12738   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12739 
12740   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12741   ins_encode %{
12742     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12743                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12744                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12745   %}
12746   ins_pipe( pipe_slow );
12747 %}
12748 
12749 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12750                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12751 %{
12752   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12753   match(Set result (AryEq ary1 ary2));
12754   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12755 
12756   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12757   ins_encode %{
12758     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12759                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12760                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12761   %}
12762   ins_pipe( pipe_slow );
12763 %}
12764 
12765 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12766                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12767                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12768                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12769                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12770 %{
12771   predicate(UseAVX >= 2);
12772   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12773   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12774          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12775          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12776          USE basic_type, KILL cr);
12777 
12778   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12779   ins_encode %{
12780     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12781                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12782                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12783                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12784                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12785                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12786                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12787   %}
12788   ins_pipe( pipe_slow );
12789 %}
12790 
12791 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12792                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12793 %{
12794   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12795   match(Set result (CountPositives ary1 len));
12796   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12797 
12798   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12799   ins_encode %{
12800     __ count_positives($ary1$$Register, $len$$Register,
12801                        $result$$Register, $tmp3$$Register,
12802                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12803   %}
12804   ins_pipe( pipe_slow );
12805 %}
12806 
12807 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12808                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12809 %{
12810   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12811   match(Set result (CountPositives ary1 len));
12812   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12813 
12814   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12815   ins_encode %{
12816     __ count_positives($ary1$$Register, $len$$Register,
12817                        $result$$Register, $tmp3$$Register,
12818                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12819   %}
12820   ins_pipe( pipe_slow );
12821 %}
12822 
12823 // fast char[] to byte[] compression
12824 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12825                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12826   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12827   match(Set result (StrCompressedCopy src (Binary dst len)));
12828   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12829          USE_KILL len, KILL tmp5, KILL cr);
12830 
12831   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12832   ins_encode %{
12833     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12834                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12835                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12836                            knoreg, knoreg);
12837   %}
12838   ins_pipe( pipe_slow );
12839 %}
12840 
12841 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12842                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12843   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12844   match(Set result (StrCompressedCopy src (Binary dst len)));
12845   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12846          USE_KILL len, KILL tmp5, KILL cr);
12847 
12848   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12849   ins_encode %{
12850     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12851                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12852                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12853                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12854   %}
12855   ins_pipe( pipe_slow );
12856 %}
12857 // fast byte[] to char[] inflation
12858 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12859                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12860   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12861   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12862   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12863 
12864   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12865   ins_encode %{
12866     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12867                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12868   %}
12869   ins_pipe( pipe_slow );
12870 %}
12871 
12872 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12873                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12874   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12875   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12876   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12877 
12878   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12879   ins_encode %{
12880     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12881                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12882   %}
12883   ins_pipe( pipe_slow );
12884 %}
12885 
12886 // encode char[] to byte[] in ISO_8859_1
12887 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12888                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12889                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12890   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12891   match(Set result (EncodeISOArray src (Binary dst len)));
12892   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12893 
12894   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12895   ins_encode %{
12896     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12897                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12898                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
12899   %}
12900   ins_pipe( pipe_slow );
12901 %}
12902 
12903 // encode char[] to byte[] in ASCII
12904 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12905                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12906                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12907   predicate(((EncodeISOArrayNode*)n)->is_ascii());
12908   match(Set result (EncodeISOArray src (Binary dst len)));
12909   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12910 
12911   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
12912   ins_encode %{
12913     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
12914                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12915                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
12916   %}
12917   ins_pipe( pipe_slow );
12918 %}
12919 
12920 //----------Overflow Math Instructions-----------------------------------------
12921 
12922 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
12923 %{
12924   match(Set cr (OverflowAddI op1 op2));
12925   effect(DEF cr, USE_KILL op1, USE op2);
12926 
12927   format %{ "addl    $op1, $op2\t# overflow check int" %}
12928 
12929   ins_encode %{
12930     __ addl($op1$$Register, $op2$$Register);
12931   %}
12932   ins_pipe(ialu_reg_reg);
12933 %}
12934 
12935 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
12936 %{
12937   match(Set cr (OverflowAddI op1 op2));
12938   effect(DEF cr, USE_KILL op1, USE op2);
12939 
12940   format %{ "addl    $op1, $op2\t# overflow check int" %}
12941 
12942   ins_encode %{
12943     __ addl($op1$$Register, $op2$$constant);
12944   %}
12945   ins_pipe(ialu_reg_reg);
12946 %}
12947 
12948 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
12949 %{
12950   match(Set cr (OverflowAddL op1 op2));
12951   effect(DEF cr, USE_KILL op1, USE op2);
12952 
12953   format %{ "addq    $op1, $op2\t# overflow check long" %}
12954   ins_encode %{
12955     __ addq($op1$$Register, $op2$$Register);
12956   %}
12957   ins_pipe(ialu_reg_reg);
12958 %}
12959 
12960 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
12961 %{
12962   match(Set cr (OverflowAddL op1 op2));
12963   effect(DEF cr, USE_KILL op1, USE op2);
12964 
12965   format %{ "addq    $op1, $op2\t# overflow check long" %}
12966   ins_encode %{
12967     __ addq($op1$$Register, $op2$$constant);
12968   %}
12969   ins_pipe(ialu_reg_reg);
12970 %}
12971 
12972 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12973 %{
12974   match(Set cr (OverflowSubI op1 op2));
12975 
12976   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12977   ins_encode %{
12978     __ cmpl($op1$$Register, $op2$$Register);
12979   %}
12980   ins_pipe(ialu_reg_reg);
12981 %}
12982 
12983 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12984 %{
12985   match(Set cr (OverflowSubI op1 op2));
12986 
12987   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
12988   ins_encode %{
12989     __ cmpl($op1$$Register, $op2$$constant);
12990   %}
12991   ins_pipe(ialu_reg_reg);
12992 %}
12993 
12994 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12995 %{
12996   match(Set cr (OverflowSubL op1 op2));
12997 
12998   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
12999   ins_encode %{
13000     __ cmpq($op1$$Register, $op2$$Register);
13001   %}
13002   ins_pipe(ialu_reg_reg);
13003 %}
13004 
13005 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13006 %{
13007   match(Set cr (OverflowSubL op1 op2));
13008 
13009   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13010   ins_encode %{
13011     __ cmpq($op1$$Register, $op2$$constant);
13012   %}
13013   ins_pipe(ialu_reg_reg);
13014 %}
13015 
13016 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13017 %{
13018   match(Set cr (OverflowSubI zero op2));
13019   effect(DEF cr, USE_KILL op2);
13020 
13021   format %{ "negl    $op2\t# overflow check int" %}
13022   ins_encode %{
13023     __ negl($op2$$Register);
13024   %}
13025   ins_pipe(ialu_reg_reg);
13026 %}
13027 
13028 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13029 %{
13030   match(Set cr (OverflowSubL zero op2));
13031   effect(DEF cr, USE_KILL op2);
13032 
13033   format %{ "negq    $op2\t# overflow check long" %}
13034   ins_encode %{
13035     __ negq($op2$$Register);
13036   %}
13037   ins_pipe(ialu_reg_reg);
13038 %}
13039 
13040 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13041 %{
13042   match(Set cr (OverflowMulI op1 op2));
13043   effect(DEF cr, USE_KILL op1, USE op2);
13044 
13045   format %{ "imull    $op1, $op2\t# overflow check int" %}
13046   ins_encode %{
13047     __ imull($op1$$Register, $op2$$Register);
13048   %}
13049   ins_pipe(ialu_reg_reg_alu0);
13050 %}
13051 
13052 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13053 %{
13054   match(Set cr (OverflowMulI op1 op2));
13055   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13056 
13057   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13058   ins_encode %{
13059     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13060   %}
13061   ins_pipe(ialu_reg_reg_alu0);
13062 %}
13063 
13064 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13065 %{
13066   match(Set cr (OverflowMulL op1 op2));
13067   effect(DEF cr, USE_KILL op1, USE op2);
13068 
13069   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13070   ins_encode %{
13071     __ imulq($op1$$Register, $op2$$Register);
13072   %}
13073   ins_pipe(ialu_reg_reg_alu0);
13074 %}
13075 
13076 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13077 %{
13078   match(Set cr (OverflowMulL op1 op2));
13079   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13080 
13081   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13082   ins_encode %{
13083     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13084   %}
13085   ins_pipe(ialu_reg_reg_alu0);
13086 %}
13087 
13088 
13089 //----------Control Flow Instructions------------------------------------------
13090 // Signed compare Instructions
13091 
13092 // XXX more variants!!
13093 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13094 %{
13095   match(Set cr (CmpI op1 op2));
13096   effect(DEF cr, USE op1, USE op2);
13097 
13098   format %{ "cmpl    $op1, $op2" %}
13099   ins_encode %{
13100     __ cmpl($op1$$Register, $op2$$Register);
13101   %}
13102   ins_pipe(ialu_cr_reg_reg);
13103 %}
13104 
13105 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13106 %{
13107   match(Set cr (CmpI op1 op2));
13108 
13109   format %{ "cmpl    $op1, $op2" %}
13110   ins_encode %{
13111     __ cmpl($op1$$Register, $op2$$constant);
13112   %}
13113   ins_pipe(ialu_cr_reg_imm);
13114 %}
13115 
13116 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13117 %{
13118   match(Set cr (CmpI op1 (LoadI op2)));
13119 
13120   ins_cost(500); // XXX
13121   format %{ "cmpl    $op1, $op2" %}
13122   ins_encode %{
13123     __ cmpl($op1$$Register, $op2$$Address);
13124   %}
13125   ins_pipe(ialu_cr_reg_mem);
13126 %}
13127 
13128 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13129 %{
13130   match(Set cr (CmpI src zero));
13131 
13132   format %{ "testl   $src, $src" %}
13133   ins_encode %{
13134     __ testl($src$$Register, $src$$Register);
13135   %}
13136   ins_pipe(ialu_cr_reg_imm);
13137 %}
13138 
13139 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13140 %{
13141   match(Set cr (CmpI (AndI src con) zero));
13142 
13143   format %{ "testl   $src, $con" %}
13144   ins_encode %{
13145     __ testl($src$$Register, $con$$constant);
13146   %}
13147   ins_pipe(ialu_cr_reg_imm);
13148 %}
13149 
13150 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13151 %{
13152   match(Set cr (CmpI (AndI src1 src2) zero));
13153 
13154   format %{ "testl   $src1, $src2" %}
13155   ins_encode %{
13156     __ testl($src1$$Register, $src2$$Register);
13157   %}
13158   ins_pipe(ialu_cr_reg_imm);
13159 %}
13160 
13161 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13162 %{
13163   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13164 
13165   format %{ "testl   $src, $mem" %}
13166   ins_encode %{
13167     __ testl($src$$Register, $mem$$Address);
13168   %}
13169   ins_pipe(ialu_cr_reg_mem);
13170 %}
13171 
13172 // Unsigned compare Instructions; really, same as signed except they
13173 // produce an rFlagsRegU instead of rFlagsReg.
13174 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13175 %{
13176   match(Set cr (CmpU op1 op2));
13177 
13178   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13179   ins_encode %{
13180     __ cmpl($op1$$Register, $op2$$Register);
13181   %}
13182   ins_pipe(ialu_cr_reg_reg);
13183 %}
13184 
13185 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13186 %{
13187   match(Set cr (CmpU op1 op2));
13188 
13189   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13190   ins_encode %{
13191     __ cmpl($op1$$Register, $op2$$constant);
13192   %}
13193   ins_pipe(ialu_cr_reg_imm);
13194 %}
13195 
13196 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13197 %{
13198   match(Set cr (CmpU op1 (LoadI op2)));
13199 
13200   ins_cost(500); // XXX
13201   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13202   ins_encode %{
13203     __ cmpl($op1$$Register, $op2$$Address);
13204   %}
13205   ins_pipe(ialu_cr_reg_mem);
13206 %}
13207 
13208 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13209 %{
13210   match(Set cr (CmpU src zero));
13211 
13212   format %{ "testl   $src, $src\t# unsigned" %}
13213   ins_encode %{
13214     __ testl($src$$Register, $src$$Register);
13215   %}
13216   ins_pipe(ialu_cr_reg_imm);
13217 %}
13218 
13219 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13220 %{
13221   match(Set cr (CmpP op1 op2));
13222 
13223   format %{ "cmpq    $op1, $op2\t# ptr" %}
13224   ins_encode %{
13225     __ cmpq($op1$$Register, $op2$$Register);
13226   %}
13227   ins_pipe(ialu_cr_reg_reg);
13228 %}
13229 
13230 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13231 %{
13232   match(Set cr (CmpP op1 (LoadP op2)));
13233   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13234 
13235   ins_cost(500); // XXX
13236   format %{ "cmpq    $op1, $op2\t# ptr" %}
13237   ins_encode %{
13238     __ cmpq($op1$$Register, $op2$$Address);
13239   %}
13240   ins_pipe(ialu_cr_reg_mem);
13241 %}
13242 
13243 // XXX this is generalized by compP_rReg_mem???
13244 // Compare raw pointer (used in out-of-heap check).
13245 // Only works because non-oop pointers must be raw pointers
13246 // and raw pointers have no anti-dependencies.
13247 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13248 %{
13249   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13250             n->in(2)->as_Load()->barrier_data() == 0);
13251   match(Set cr (CmpP op1 (LoadP op2)));
13252 
13253   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13254   ins_encode %{
13255     __ cmpq($op1$$Register, $op2$$Address);
13256   %}
13257   ins_pipe(ialu_cr_reg_mem);
13258 %}
13259 
13260 // This will generate a signed flags result. This should be OK since
13261 // any compare to a zero should be eq/neq.
13262 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13263 %{
13264   match(Set cr (CmpP src zero));
13265 
13266   format %{ "testq   $src, $src\t# ptr" %}
13267   ins_encode %{
13268     __ testq($src$$Register, $src$$Register);
13269   %}
13270   ins_pipe(ialu_cr_reg_imm);
13271 %}
13272 
13273 // This will generate a signed flags result. This should be OK since
13274 // any compare to a zero should be eq/neq.
13275 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13276 %{
13277   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13278             n->in(1)->as_Load()->barrier_data() == 0);
13279   match(Set cr (CmpP (LoadP op) zero));
13280 
13281   ins_cost(500); // XXX
13282   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13283   ins_encode %{
13284     __ testq($op$$Address, 0xFFFFFFFF);
13285   %}
13286   ins_pipe(ialu_cr_reg_imm);
13287 %}
13288 
13289 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13290 %{
13291   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13292             n->in(1)->as_Load()->barrier_data() == 0);
13293   match(Set cr (CmpP (LoadP mem) zero));
13294 
13295   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13296   ins_encode %{
13297     __ cmpq(r12, $mem$$Address);
13298   %}
13299   ins_pipe(ialu_cr_reg_mem);
13300 %}
13301 
13302 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13303 %{
13304   match(Set cr (CmpN op1 op2));
13305 
13306   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13307   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13308   ins_pipe(ialu_cr_reg_reg);
13309 %}
13310 
13311 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13312 %{
13313   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13314   match(Set cr (CmpN src (LoadN mem)));
13315 
13316   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13317   ins_encode %{
13318     __ cmpl($src$$Register, $mem$$Address);
13319   %}
13320   ins_pipe(ialu_cr_reg_mem);
13321 %}
13322 
13323 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13324   match(Set cr (CmpN op1 op2));
13325 
13326   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13327   ins_encode %{
13328     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13329   %}
13330   ins_pipe(ialu_cr_reg_imm);
13331 %}
13332 
13333 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13334 %{
13335   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13336   match(Set cr (CmpN src (LoadN mem)));
13337 
13338   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13339   ins_encode %{
13340     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13341   %}
13342   ins_pipe(ialu_cr_reg_mem);
13343 %}
13344 
13345 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13346   match(Set cr (CmpN op1 op2));
13347 
13348   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13349   ins_encode %{
13350     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13351   %}
13352   ins_pipe(ialu_cr_reg_imm);
13353 %}
13354 
13355 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13356 %{
13357   predicate(!UseCompactObjectHeaders);
13358   match(Set cr (CmpN src (LoadNKlass mem)));
13359 
13360   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13361   ins_encode %{
13362     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13363   %}
13364   ins_pipe(ialu_cr_reg_mem);
13365 %}
13366 
13367 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13368   match(Set cr (CmpN src zero));
13369 
13370   format %{ "testl   $src, $src\t# compressed ptr" %}
13371   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13372   ins_pipe(ialu_cr_reg_imm);
13373 %}
13374 
13375 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13376 %{
13377   predicate(CompressedOops::base() != nullptr &&
13378             n->in(1)->as_Load()->barrier_data() == 0);
13379   match(Set cr (CmpN (LoadN mem) zero));
13380 
13381   ins_cost(500); // XXX
13382   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13383   ins_encode %{
13384     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13385   %}
13386   ins_pipe(ialu_cr_reg_mem);
13387 %}
13388 
13389 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13390 %{
13391   predicate(CompressedOops::base() == nullptr &&
13392             n->in(1)->as_Load()->barrier_data() == 0);
13393   match(Set cr (CmpN (LoadN mem) zero));
13394 
13395   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13396   ins_encode %{
13397     __ cmpl(r12, $mem$$Address);
13398   %}
13399   ins_pipe(ialu_cr_reg_mem);
13400 %}
13401 
13402 // Yanked all unsigned pointer compare operations.
13403 // Pointer compares are done with CmpP which is already unsigned.
13404 
13405 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13406 %{
13407   match(Set cr (CmpL op1 op2));
13408 
13409   format %{ "cmpq    $op1, $op2" %}
13410   ins_encode %{
13411     __ cmpq($op1$$Register, $op2$$Register);
13412   %}
13413   ins_pipe(ialu_cr_reg_reg);
13414 %}
13415 
13416 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13417 %{
13418   match(Set cr (CmpL op1 op2));
13419 
13420   format %{ "cmpq    $op1, $op2" %}
13421   ins_encode %{
13422     __ cmpq($op1$$Register, $op2$$constant);
13423   %}
13424   ins_pipe(ialu_cr_reg_imm);
13425 %}
13426 
13427 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13428 %{
13429   match(Set cr (CmpL op1 (LoadL op2)));
13430 
13431   format %{ "cmpq    $op1, $op2" %}
13432   ins_encode %{
13433     __ cmpq($op1$$Register, $op2$$Address);
13434   %}
13435   ins_pipe(ialu_cr_reg_mem);
13436 %}
13437 
13438 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13439 %{
13440   match(Set cr (CmpL src zero));
13441 
13442   format %{ "testq   $src, $src" %}
13443   ins_encode %{
13444     __ testq($src$$Register, $src$$Register);
13445   %}
13446   ins_pipe(ialu_cr_reg_imm);
13447 %}
13448 
13449 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13450 %{
13451   match(Set cr (CmpL (AndL src con) zero));
13452 
13453   format %{ "testq   $src, $con\t# long" %}
13454   ins_encode %{
13455     __ testq($src$$Register, $con$$constant);
13456   %}
13457   ins_pipe(ialu_cr_reg_imm);
13458 %}
13459 
13460 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13461 %{
13462   match(Set cr (CmpL (AndL src1 src2) zero));
13463 
13464   format %{ "testq   $src1, $src2\t# long" %}
13465   ins_encode %{
13466     __ testq($src1$$Register, $src2$$Register);
13467   %}
13468   ins_pipe(ialu_cr_reg_imm);
13469 %}
13470 
13471 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13472 %{
13473   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13474 
13475   format %{ "testq   $src, $mem" %}
13476   ins_encode %{
13477     __ testq($src$$Register, $mem$$Address);
13478   %}
13479   ins_pipe(ialu_cr_reg_mem);
13480 %}
13481 
13482 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13483 %{
13484   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13485 
13486   format %{ "testq   $src, $mem" %}
13487   ins_encode %{
13488     __ testq($src$$Register, $mem$$Address);
13489   %}
13490   ins_pipe(ialu_cr_reg_mem);
13491 %}
13492 
13493 // Manifest a CmpU result in an integer register.  Very painful.
13494 // This is the test to avoid.
13495 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13496 %{
13497   match(Set dst (CmpU3 src1 src2));
13498   effect(KILL flags);
13499 
13500   ins_cost(275); // XXX
13501   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13502             "movl    $dst, -1\n\t"
13503             "jb,u    done\n\t"
13504             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13505     "done:" %}
13506   ins_encode %{
13507     Label done;
13508     __ cmpl($src1$$Register, $src2$$Register);
13509     __ movl($dst$$Register, -1);
13510     __ jccb(Assembler::below, done);
13511     __ setcc(Assembler::notZero, $dst$$Register);
13512     __ bind(done);
13513   %}
13514   ins_pipe(pipe_slow);
13515 %}
13516 
13517 // Manifest a CmpL result in an integer register.  Very painful.
13518 // This is the test to avoid.
13519 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13520 %{
13521   match(Set dst (CmpL3 src1 src2));
13522   effect(KILL flags);
13523 
13524   ins_cost(275); // XXX
13525   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13526             "movl    $dst, -1\n\t"
13527             "jl,s    done\n\t"
13528             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13529     "done:" %}
13530   ins_encode %{
13531     Label done;
13532     __ cmpq($src1$$Register, $src2$$Register);
13533     __ movl($dst$$Register, -1);
13534     __ jccb(Assembler::less, done);
13535     __ setcc(Assembler::notZero, $dst$$Register);
13536     __ bind(done);
13537   %}
13538   ins_pipe(pipe_slow);
13539 %}
13540 
13541 // Manifest a CmpUL result in an integer register.  Very painful.
13542 // This is the test to avoid.
13543 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13544 %{
13545   match(Set dst (CmpUL3 src1 src2));
13546   effect(KILL flags);
13547 
13548   ins_cost(275); // XXX
13549   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13550             "movl    $dst, -1\n\t"
13551             "jb,u    done\n\t"
13552             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13553     "done:" %}
13554   ins_encode %{
13555     Label done;
13556     __ cmpq($src1$$Register, $src2$$Register);
13557     __ movl($dst$$Register, -1);
13558     __ jccb(Assembler::below, done);
13559     __ setcc(Assembler::notZero, $dst$$Register);
13560     __ bind(done);
13561   %}
13562   ins_pipe(pipe_slow);
13563 %}
13564 
13565 // Unsigned long compare Instructions; really, same as signed long except they
13566 // produce an rFlagsRegU instead of rFlagsReg.
13567 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13568 %{
13569   match(Set cr (CmpUL op1 op2));
13570 
13571   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13572   ins_encode %{
13573     __ cmpq($op1$$Register, $op2$$Register);
13574   %}
13575   ins_pipe(ialu_cr_reg_reg);
13576 %}
13577 
13578 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13579 %{
13580   match(Set cr (CmpUL op1 op2));
13581 
13582   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13583   ins_encode %{
13584     __ cmpq($op1$$Register, $op2$$constant);
13585   %}
13586   ins_pipe(ialu_cr_reg_imm);
13587 %}
13588 
13589 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13590 %{
13591   match(Set cr (CmpUL op1 (LoadL op2)));
13592 
13593   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13594   ins_encode %{
13595     __ cmpq($op1$$Register, $op2$$Address);
13596   %}
13597   ins_pipe(ialu_cr_reg_mem);
13598 %}
13599 
13600 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13601 %{
13602   match(Set cr (CmpUL src zero));
13603 
13604   format %{ "testq   $src, $src\t# unsigned" %}
13605   ins_encode %{
13606     __ testq($src$$Register, $src$$Register);
13607   %}
13608   ins_pipe(ialu_cr_reg_imm);
13609 %}
13610 
13611 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13612 %{
13613   match(Set cr (CmpI (LoadB mem) imm));
13614 
13615   ins_cost(125);
13616   format %{ "cmpb    $mem, $imm" %}
13617   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13618   ins_pipe(ialu_cr_reg_mem);
13619 %}
13620 
13621 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13622 %{
13623   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13624 
13625   ins_cost(125);
13626   format %{ "testb   $mem, $imm\t# ubyte" %}
13627   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13628   ins_pipe(ialu_cr_reg_mem);
13629 %}
13630 
13631 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13632 %{
13633   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13634 
13635   ins_cost(125);
13636   format %{ "testb   $mem, $imm\t# byte" %}
13637   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13638   ins_pipe(ialu_cr_reg_mem);
13639 %}
13640 
13641 //----------Max and Min--------------------------------------------------------
13642 // Min Instructions
13643 
13644 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13645 %{
13646   predicate(!UseAPX);
13647   effect(USE_DEF dst, USE src, USE cr);
13648 
13649   format %{ "cmovlgt $dst, $src\t# min" %}
13650   ins_encode %{
13651     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13652   %}
13653   ins_pipe(pipe_cmov_reg);
13654 %}
13655 
13656 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13657 %{
13658   predicate(UseAPX);
13659   effect(DEF dst, USE src1, USE src2, USE cr);
13660 
13661   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13662   ins_encode %{
13663     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13664   %}
13665   ins_pipe(pipe_cmov_reg);
13666 %}
13667 
13668 instruct minI_rReg(rRegI dst, rRegI src)
13669 %{
13670   predicate(!UseAPX);
13671   match(Set dst (MinI dst src));
13672 
13673   ins_cost(200);
13674   expand %{
13675     rFlagsReg cr;
13676     compI_rReg(cr, dst, src);
13677     cmovI_reg_g(dst, src, cr);
13678   %}
13679 %}
13680 
13681 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13682 %{
13683   predicate(UseAPX);
13684   match(Set dst (MinI src1 src2));
13685   effect(DEF dst, USE src1, USE src2);
13686 
13687   ins_cost(200);
13688   expand %{
13689     rFlagsReg cr;
13690     compI_rReg(cr, src1, src2);
13691     cmovI_reg_g_ndd(dst, src1, src2, cr);
13692   %}
13693 %}
13694 
13695 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13696 %{
13697   predicate(!UseAPX);
13698   effect(USE_DEF dst, USE src, USE cr);
13699 
13700   format %{ "cmovllt $dst, $src\t# max" %}
13701   ins_encode %{
13702     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13703   %}
13704   ins_pipe(pipe_cmov_reg);
13705 %}
13706 
13707 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13708 %{
13709   predicate(UseAPX);
13710   effect(DEF dst, USE src1, USE src2, USE cr);
13711 
13712   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13713   ins_encode %{
13714     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13715   %}
13716   ins_pipe(pipe_cmov_reg);
13717 %}
13718 
13719 instruct maxI_rReg(rRegI dst, rRegI src)
13720 %{
13721   predicate(!UseAPX);
13722   match(Set dst (MaxI dst src));
13723 
13724   ins_cost(200);
13725   expand %{
13726     rFlagsReg cr;
13727     compI_rReg(cr, dst, src);
13728     cmovI_reg_l(dst, src, cr);
13729   %}
13730 %}
13731 
13732 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13733 %{
13734   predicate(UseAPX);
13735   match(Set dst (MaxI src1 src2));
13736   effect(DEF dst, USE src1, USE src2);
13737 
13738   ins_cost(200);
13739   expand %{
13740     rFlagsReg cr;
13741     compI_rReg(cr, src1, src2);
13742     cmovI_reg_l_ndd(dst, src1, src2, cr);
13743   %}
13744 %}
13745 
13746 // ============================================================================
13747 // Branch Instructions
13748 
13749 // Jump Direct - Label defines a relative address from JMP+1
13750 instruct jmpDir(label labl)
13751 %{
13752   match(Goto);
13753   effect(USE labl);
13754 
13755   ins_cost(300);
13756   format %{ "jmp     $labl" %}
13757   size(5);
13758   ins_encode %{
13759     Label* L = $labl$$label;
13760     __ jmp(*L, false); // Always long jump
13761   %}
13762   ins_pipe(pipe_jmp);
13763 %}
13764 
13765 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13766 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13767 %{
13768   match(If cop cr);
13769   effect(USE labl);
13770 
13771   ins_cost(300);
13772   format %{ "j$cop     $labl" %}
13773   size(6);
13774   ins_encode %{
13775     Label* L = $labl$$label;
13776     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13777   %}
13778   ins_pipe(pipe_jcc);
13779 %}
13780 
13781 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13782 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13783 %{
13784   match(CountedLoopEnd cop cr);
13785   effect(USE labl);
13786 
13787   ins_cost(300);
13788   format %{ "j$cop     $labl\t# loop end" %}
13789   size(6);
13790   ins_encode %{
13791     Label* L = $labl$$label;
13792     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13793   %}
13794   ins_pipe(pipe_jcc);
13795 %}
13796 
13797 // Jump Direct Conditional - using unsigned comparison
13798 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13799   match(If cop cmp);
13800   effect(USE labl);
13801 
13802   ins_cost(300);
13803   format %{ "j$cop,u   $labl" %}
13804   size(6);
13805   ins_encode %{
13806     Label* L = $labl$$label;
13807     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13808   %}
13809   ins_pipe(pipe_jcc);
13810 %}
13811 
13812 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13813   match(If cop cmp);
13814   effect(USE labl);
13815 
13816   ins_cost(200);
13817   format %{ "j$cop,u   $labl" %}
13818   size(6);
13819   ins_encode %{
13820     Label* L = $labl$$label;
13821     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13822   %}
13823   ins_pipe(pipe_jcc);
13824 %}
13825 
13826 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13827   match(If cop cmp);
13828   effect(USE labl);
13829 
13830   ins_cost(200);
13831   format %{ $$template
13832     if ($cop$$cmpcode == Assembler::notEqual) {
13833       $$emit$$"jp,u    $labl\n\t"
13834       $$emit$$"j$cop,u   $labl"
13835     } else {
13836       $$emit$$"jp,u    done\n\t"
13837       $$emit$$"j$cop,u   $labl\n\t"
13838       $$emit$$"done:"
13839     }
13840   %}
13841   ins_encode %{
13842     Label* l = $labl$$label;
13843     if ($cop$$cmpcode == Assembler::notEqual) {
13844       __ jcc(Assembler::parity, *l, false);
13845       __ jcc(Assembler::notEqual, *l, false);
13846     } else if ($cop$$cmpcode == Assembler::equal) {
13847       Label done;
13848       __ jccb(Assembler::parity, done);
13849       __ jcc(Assembler::equal, *l, false);
13850       __ bind(done);
13851     } else {
13852        ShouldNotReachHere();
13853     }
13854   %}
13855   ins_pipe(pipe_jcc);
13856 %}
13857 
13858 // ============================================================================
13859 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13860 // superklass array for an instance of the superklass.  Set a hidden
13861 // internal cache on a hit (cache is checked with exposed code in
13862 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13863 // encoding ALSO sets flags.
13864 
13865 instruct partialSubtypeCheck(rdi_RegP result,
13866                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13867                              rFlagsReg cr)
13868 %{
13869   match(Set result (PartialSubtypeCheck sub super));
13870   predicate(!UseSecondarySupersTable);
13871   effect(KILL rcx, KILL cr);
13872 
13873   ins_cost(1100);  // slightly larger than the next version
13874   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13875             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13876             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13877             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13878             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13879             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13880             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13881     "miss:\t" %}
13882 
13883   ins_encode %{
13884     Label miss;
13885     // NB: Callers may assume that, when $result is a valid register,
13886     // check_klass_subtype_slow_path_linear sets it to a nonzero
13887     // value.
13888     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13889                                             $rcx$$Register, $result$$Register,
13890                                             nullptr, &miss,
13891                                             /*set_cond_codes:*/ true);
13892     __ xorptr($result$$Register, $result$$Register);
13893     __ bind(miss);
13894   %}
13895 
13896   ins_pipe(pipe_slow);
13897 %}
13898 
13899 // ============================================================================
13900 // Two versions of hashtable-based partialSubtypeCheck, both used when
13901 // we need to search for a super class in the secondary supers array.
13902 // The first is used when we don't know _a priori_ the class being
13903 // searched for. The second, far more common, is used when we do know:
13904 // this is used for instanceof, checkcast, and any case where C2 can
13905 // determine it by constant propagation.
13906 
13907 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
13908                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13909                                        rFlagsReg cr)
13910 %{
13911   match(Set result (PartialSubtypeCheck sub super));
13912   predicate(UseSecondarySupersTable);
13913   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13914 
13915   ins_cost(1000);
13916   format %{ "partialSubtypeCheck $result, $sub, $super" %}
13917 
13918   ins_encode %{
13919     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
13920 					 $temp3$$Register, $temp4$$Register, $result$$Register);
13921   %}
13922 
13923   ins_pipe(pipe_slow);
13924 %}
13925 
13926 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
13927                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
13928                                        rFlagsReg cr)
13929 %{
13930   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
13931   predicate(UseSecondarySupersTable);
13932   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
13933 
13934   ins_cost(700);  // smaller than the next version
13935   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
13936 
13937   ins_encode %{
13938     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
13939     if (InlineSecondarySupersTest) {
13940       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
13941                                        $temp3$$Register, $temp4$$Register, $result$$Register,
13942                                        super_klass_slot);
13943     } else {
13944       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
13945     }
13946   %}
13947 
13948   ins_pipe(pipe_slow);
13949 %}
13950 
13951 // ============================================================================
13952 // Branch Instructions -- short offset versions
13953 //
13954 // These instructions are used to replace jumps of a long offset (the default
13955 // match) with jumps of a shorter offset.  These instructions are all tagged
13956 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13957 // match rules in general matching.  Instead, the ADLC generates a conversion
13958 // method in the MachNode which can be used to do in-place replacement of the
13959 // long variant with the shorter variant.  The compiler will determine if a
13960 // branch can be taken by the is_short_branch_offset() predicate in the machine
13961 // specific code section of the file.
13962 
13963 // Jump Direct - Label defines a relative address from JMP+1
13964 instruct jmpDir_short(label labl) %{
13965   match(Goto);
13966   effect(USE labl);
13967 
13968   ins_cost(300);
13969   format %{ "jmp,s   $labl" %}
13970   size(2);
13971   ins_encode %{
13972     Label* L = $labl$$label;
13973     __ jmpb(*L);
13974   %}
13975   ins_pipe(pipe_jmp);
13976   ins_short_branch(1);
13977 %}
13978 
13979 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13980 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
13981   match(If cop cr);
13982   effect(USE labl);
13983 
13984   ins_cost(300);
13985   format %{ "j$cop,s   $labl" %}
13986   size(2);
13987   ins_encode %{
13988     Label* L = $labl$$label;
13989     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13990   %}
13991   ins_pipe(pipe_jcc);
13992   ins_short_branch(1);
13993 %}
13994 
13995 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13996 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
13997   match(CountedLoopEnd cop cr);
13998   effect(USE labl);
13999 
14000   ins_cost(300);
14001   format %{ "j$cop,s   $labl\t# loop end" %}
14002   size(2);
14003   ins_encode %{
14004     Label* L = $labl$$label;
14005     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14006   %}
14007   ins_pipe(pipe_jcc);
14008   ins_short_branch(1);
14009 %}
14010 
14011 // Jump Direct Conditional - using unsigned comparison
14012 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14013   match(If cop cmp);
14014   effect(USE labl);
14015 
14016   ins_cost(300);
14017   format %{ "j$cop,us  $labl" %}
14018   size(2);
14019   ins_encode %{
14020     Label* L = $labl$$label;
14021     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14022   %}
14023   ins_pipe(pipe_jcc);
14024   ins_short_branch(1);
14025 %}
14026 
14027 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14028   match(If cop cmp);
14029   effect(USE labl);
14030 
14031   ins_cost(300);
14032   format %{ "j$cop,us  $labl" %}
14033   size(2);
14034   ins_encode %{
14035     Label* L = $labl$$label;
14036     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14037   %}
14038   ins_pipe(pipe_jcc);
14039   ins_short_branch(1);
14040 %}
14041 
14042 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14043   match(If cop cmp);
14044   effect(USE labl);
14045 
14046   ins_cost(300);
14047   format %{ $$template
14048     if ($cop$$cmpcode == Assembler::notEqual) {
14049       $$emit$$"jp,u,s  $labl\n\t"
14050       $$emit$$"j$cop,u,s  $labl"
14051     } else {
14052       $$emit$$"jp,u,s  done\n\t"
14053       $$emit$$"j$cop,u,s  $labl\n\t"
14054       $$emit$$"done:"
14055     }
14056   %}
14057   size(4);
14058   ins_encode %{
14059     Label* l = $labl$$label;
14060     if ($cop$$cmpcode == Assembler::notEqual) {
14061       __ jccb(Assembler::parity, *l);
14062       __ jccb(Assembler::notEqual, *l);
14063     } else if ($cop$$cmpcode == Assembler::equal) {
14064       Label done;
14065       __ jccb(Assembler::parity, done);
14066       __ jccb(Assembler::equal, *l);
14067       __ bind(done);
14068     } else {
14069        ShouldNotReachHere();
14070     }
14071   %}
14072   ins_pipe(pipe_jcc);
14073   ins_short_branch(1);
14074 %}
14075 
14076 // ============================================================================
14077 // inlined locking and unlocking
14078 
14079 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
14080   predicate(LockingMode != LM_LIGHTWEIGHT);
14081   match(Set cr (FastLock object box));
14082   effect(TEMP tmp, TEMP scr, USE_KILL box);
14083   ins_cost(300);
14084   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
14085   ins_encode %{
14086     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
14087                  $scr$$Register, noreg, noreg, r15_thread, nullptr);
14088   %}
14089   ins_pipe(pipe_slow);
14090 %}
14091 
14092 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
14093   predicate(LockingMode != LM_LIGHTWEIGHT);
14094   match(Set cr (FastUnlock object box));
14095   effect(TEMP tmp, USE_KILL box);
14096   ins_cost(300);
14097   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
14098   ins_encode %{
14099     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
14100   %}
14101   ins_pipe(pipe_slow);
14102 %}
14103 
14104 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14105   predicate(LockingMode == LM_LIGHTWEIGHT);
14106   match(Set cr (FastLock object box));
14107   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14108   ins_cost(300);
14109   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14110   ins_encode %{
14111     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14112   %}
14113   ins_pipe(pipe_slow);
14114 %}
14115 
14116 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14117   predicate(LockingMode == LM_LIGHTWEIGHT);
14118   match(Set cr (FastUnlock object rax_reg));
14119   effect(TEMP tmp, USE_KILL rax_reg);
14120   ins_cost(300);
14121   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14122   ins_encode %{
14123     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14124   %}
14125   ins_pipe(pipe_slow);
14126 %}
14127 
14128 
14129 // ============================================================================
14130 // Safepoint Instructions
14131 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14132 %{
14133   match(SafePoint poll);
14134   effect(KILL cr, USE poll);
14135 
14136   format %{ "testl   rax, [$poll]\t"
14137             "# Safepoint: poll for GC" %}
14138   ins_cost(125);
14139   ins_encode %{
14140     __ relocate(relocInfo::poll_type);
14141     address pre_pc = __ pc();
14142     __ testl(rax, Address($poll$$Register, 0));
14143     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14144   %}
14145   ins_pipe(ialu_reg_mem);
14146 %}
14147 
14148 instruct mask_all_evexL(kReg dst, rRegL src) %{
14149   match(Set dst (MaskAll src));
14150   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14151   ins_encode %{
14152     int mask_len = Matcher::vector_length(this);
14153     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14154   %}
14155   ins_pipe( pipe_slow );
14156 %}
14157 
14158 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14159   predicate(Matcher::vector_length(n) > 32);
14160   match(Set dst (MaskAll src));
14161   effect(TEMP tmp);
14162   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14163   ins_encode %{
14164     int mask_len = Matcher::vector_length(this);
14165     __ movslq($tmp$$Register, $src$$Register);
14166     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14167   %}
14168   ins_pipe( pipe_slow );
14169 %}
14170 
14171 // ============================================================================
14172 // Procedure Call/Return Instructions
14173 // Call Java Static Instruction
14174 // Note: If this code changes, the corresponding ret_addr_offset() and
14175 //       compute_padding() functions will have to be adjusted.
14176 instruct CallStaticJavaDirect(method meth) %{
14177   match(CallStaticJava);
14178   effect(USE meth);
14179 
14180   ins_cost(300);
14181   format %{ "call,static " %}
14182   opcode(0xE8); /* E8 cd */
14183   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14184   ins_pipe(pipe_slow);
14185   ins_alignment(4);
14186 %}
14187 
14188 // Call Java Dynamic Instruction
14189 // Note: If this code changes, the corresponding ret_addr_offset() and
14190 //       compute_padding() functions will have to be adjusted.
14191 instruct CallDynamicJavaDirect(method meth)
14192 %{
14193   match(CallDynamicJava);
14194   effect(USE meth);
14195 
14196   ins_cost(300);
14197   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14198             "call,dynamic " %}
14199   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14200   ins_pipe(pipe_slow);
14201   ins_alignment(4);
14202 %}
14203 
14204 // Call Runtime Instruction
14205 instruct CallRuntimeDirect(method meth)
14206 %{
14207   match(CallRuntime);
14208   effect(USE meth);
14209 
14210   ins_cost(300);
14211   format %{ "call,runtime " %}
14212   ins_encode(clear_avx, Java_To_Runtime(meth));
14213   ins_pipe(pipe_slow);
14214 %}
14215 
14216 // Call runtime without safepoint
14217 instruct CallLeafDirect(method meth)
14218 %{
14219   match(CallLeaf);
14220   effect(USE meth);
14221 
14222   ins_cost(300);
14223   format %{ "call_leaf,runtime " %}
14224   ins_encode(clear_avx, Java_To_Runtime(meth));
14225   ins_pipe(pipe_slow);
14226 %}
14227 
14228 // Call runtime without safepoint and with vector arguments
14229 instruct CallLeafDirectVector(method meth)
14230 %{
14231   match(CallLeafVector);
14232   effect(USE meth);
14233 
14234   ins_cost(300);
14235   format %{ "call_leaf,vector " %}
14236   ins_encode(Java_To_Runtime(meth));
14237   ins_pipe(pipe_slow);
14238 %}
14239 
14240 // Call runtime without safepoint
14241 instruct CallLeafNoFPDirect(method meth)
14242 %{
14243   match(CallLeafNoFP);
14244   effect(USE meth);
14245 
14246   ins_cost(300);
14247   format %{ "call_leaf_nofp,runtime " %}
14248   ins_encode(clear_avx, Java_To_Runtime(meth));
14249   ins_pipe(pipe_slow);
14250 %}
14251 
14252 // Return Instruction
14253 // Remove the return address & jump to it.
14254 // Notice: We always emit a nop after a ret to make sure there is room
14255 // for safepoint patching
14256 instruct Ret()
14257 %{
14258   match(Return);
14259 
14260   format %{ "ret" %}
14261   ins_encode %{
14262     __ ret(0);
14263   %}
14264   ins_pipe(pipe_jmp);
14265 %}
14266 
14267 // Tail Call; Jump from runtime stub to Java code.
14268 // Also known as an 'interprocedural jump'.
14269 // Target of jump will eventually return to caller.
14270 // TailJump below removes the return address.
14271 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14272 // emitted just above the TailCall which has reset rbp to the caller state.
14273 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14274 %{
14275   match(TailCall jump_target method_ptr);
14276 
14277   ins_cost(300);
14278   format %{ "jmp     $jump_target\t# rbx holds method" %}
14279   ins_encode %{
14280     __ jmp($jump_target$$Register);
14281   %}
14282   ins_pipe(pipe_jmp);
14283 %}
14284 
14285 // Tail Jump; remove the return address; jump to target.
14286 // TailCall above leaves the return address around.
14287 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14288 %{
14289   match(TailJump jump_target ex_oop);
14290 
14291   ins_cost(300);
14292   format %{ "popq    rdx\t# pop return address\n\t"
14293             "jmp     $jump_target" %}
14294   ins_encode %{
14295     __ popq(as_Register(RDX_enc));
14296     __ jmp($jump_target$$Register);
14297   %}
14298   ins_pipe(pipe_jmp);
14299 %}
14300 
14301 // Forward exception.
14302 instruct ForwardExceptionjmp()
14303 %{
14304   match(ForwardException);
14305 
14306   format %{ "jmp     forward_exception_stub" %}
14307   ins_encode %{
14308     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14309   %}
14310   ins_pipe(pipe_jmp);
14311 %}
14312 
14313 // Create exception oop: created by stack-crawling runtime code.
14314 // Created exception is now available to this handler, and is setup
14315 // just prior to jumping to this handler.  No code emitted.
14316 instruct CreateException(rax_RegP ex_oop)
14317 %{
14318   match(Set ex_oop (CreateEx));
14319 
14320   size(0);
14321   // use the following format syntax
14322   format %{ "# exception oop is in rax; no code emitted" %}
14323   ins_encode();
14324   ins_pipe(empty);
14325 %}
14326 
14327 // Rethrow exception:
14328 // The exception oop will come in the first argument position.
14329 // Then JUMP (not call) to the rethrow stub code.
14330 instruct RethrowException()
14331 %{
14332   match(Rethrow);
14333 
14334   // use the following format syntax
14335   format %{ "jmp     rethrow_stub" %}
14336   ins_encode %{
14337     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14338   %}
14339   ins_pipe(pipe_jmp);
14340 %}
14341 
14342 // ============================================================================
14343 // This name is KNOWN by the ADLC and cannot be changed.
14344 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14345 // for this guy.
14346 instruct tlsLoadP(r15_RegP dst) %{
14347   match(Set dst (ThreadLocal));
14348   effect(DEF dst);
14349 
14350   size(0);
14351   format %{ "# TLS is in R15" %}
14352   ins_encode( /*empty encoding*/ );
14353   ins_pipe(ialu_reg_reg);
14354 %}
14355 
14356 
14357 //----------PEEPHOLE RULES-----------------------------------------------------
14358 // These must follow all instruction definitions as they use the names
14359 // defined in the instructions definitions.
14360 //
14361 // peeppredicate ( rule_predicate );
14362 // // the predicate unless which the peephole rule will be ignored
14363 //
14364 // peepmatch ( root_instr_name [preceding_instruction]* );
14365 //
14366 // peepprocedure ( procedure_name );
14367 // // provide a procedure name to perform the optimization, the procedure should
14368 // // reside in the architecture dependent peephole file, the method has the
14369 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14370 // // with the arguments being the basic block, the current node index inside the
14371 // // block, the register allocator, the functions upon invoked return a new node
14372 // // defined in peepreplace, and the rules of the nodes appearing in the
14373 // // corresponding peepmatch, the function return true if successful, else
14374 // // return false
14375 //
14376 // peepconstraint %{
14377 // (instruction_number.operand_name relational_op instruction_number.operand_name
14378 //  [, ...] );
14379 // // instruction numbers are zero-based using left to right order in peepmatch
14380 //
14381 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14382 // // provide an instruction_number.operand_name for each operand that appears
14383 // // in the replacement instruction's match rule
14384 //
14385 // ---------VM FLAGS---------------------------------------------------------
14386 //
14387 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14388 //
14389 // Each peephole rule is given an identifying number starting with zero and
14390 // increasing by one in the order seen by the parser.  An individual peephole
14391 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14392 // on the command-line.
14393 //
14394 // ---------CURRENT LIMITATIONS----------------------------------------------
14395 //
14396 // Only transformations inside a basic block (do we need more for peephole)
14397 //
14398 // ---------EXAMPLE----------------------------------------------------------
14399 //
14400 // // pertinent parts of existing instructions in architecture description
14401 // instruct movI(rRegI dst, rRegI src)
14402 // %{
14403 //   match(Set dst (CopyI src));
14404 // %}
14405 //
14406 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14407 // %{
14408 //   match(Set dst (AddI dst src));
14409 //   effect(KILL cr);
14410 // %}
14411 //
14412 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14413 // %{
14414 //   match(Set dst (AddI dst src));
14415 // %}
14416 //
14417 // 1. Simple replacement
14418 // - Only match adjacent instructions in same basic block
14419 // - Only equality constraints
14420 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14421 // - Only one replacement instruction
14422 //
14423 // // Change (inc mov) to lea
14424 // peephole %{
14425 //   // lea should only be emitted when beneficial
14426 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14427 //   // increment preceded by register-register move
14428 //   peepmatch ( incI_rReg movI );
14429 //   // require that the destination register of the increment
14430 //   // match the destination register of the move
14431 //   peepconstraint ( 0.dst == 1.dst );
14432 //   // construct a replacement instruction that sets
14433 //   // the destination to ( move's source register + one )
14434 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14435 // %}
14436 //
14437 // 2. Procedural replacement
14438 // - More flexible finding relevent nodes
14439 // - More flexible constraints
14440 // - More flexible transformations
14441 // - May utilise architecture-dependent API more effectively
14442 // - Currently only one replacement instruction due to adlc parsing capabilities
14443 //
14444 // // Change (inc mov) to lea
14445 // peephole %{
14446 //   // lea should only be emitted when beneficial
14447 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14448 //   // the rule numbers of these nodes inside are passed into the function below
14449 //   peepmatch ( incI_rReg movI );
14450 //   // the method that takes the responsibility of transformation
14451 //   peepprocedure ( inc_mov_to_lea );
14452 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14453 //   // node is passed into the function above
14454 //   peepreplace ( leaI_rReg_immI() );
14455 // %}
14456 
14457 // These instructions is not matched by the matcher but used by the peephole
14458 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14459 %{
14460   predicate(false);
14461   match(Set dst (AddI src1 src2));
14462   format %{ "leal    $dst, [$src1 + $src2]" %}
14463   ins_encode %{
14464     Register dst = $dst$$Register;
14465     Register src1 = $src1$$Register;
14466     Register src2 = $src2$$Register;
14467     if (src1 != rbp && src1 != r13) {
14468       __ leal(dst, Address(src1, src2, Address::times_1));
14469     } else {
14470       assert(src2 != rbp && src2 != r13, "");
14471       __ leal(dst, Address(src2, src1, Address::times_1));
14472     }
14473   %}
14474   ins_pipe(ialu_reg_reg);
14475 %}
14476 
14477 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14478 %{
14479   predicate(false);
14480   match(Set dst (AddI src1 src2));
14481   format %{ "leal    $dst, [$src1 + $src2]" %}
14482   ins_encode %{
14483     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14484   %}
14485   ins_pipe(ialu_reg_reg);
14486 %}
14487 
14488 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14489 %{
14490   predicate(false);
14491   match(Set dst (LShiftI src shift));
14492   format %{ "leal    $dst, [$src << $shift]" %}
14493   ins_encode %{
14494     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14495     Register src = $src$$Register;
14496     if (scale == Address::times_2 && src != rbp && src != r13) {
14497       __ leal($dst$$Register, Address(src, src, Address::times_1));
14498     } else {
14499       __ leal($dst$$Register, Address(noreg, src, scale));
14500     }
14501   %}
14502   ins_pipe(ialu_reg_reg);
14503 %}
14504 
14505 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14506 %{
14507   predicate(false);
14508   match(Set dst (AddL src1 src2));
14509   format %{ "leaq    $dst, [$src1 + $src2]" %}
14510   ins_encode %{
14511     Register dst = $dst$$Register;
14512     Register src1 = $src1$$Register;
14513     Register src2 = $src2$$Register;
14514     if (src1 != rbp && src1 != r13) {
14515       __ leaq(dst, Address(src1, src2, Address::times_1));
14516     } else {
14517       assert(src2 != rbp && src2 != r13, "");
14518       __ leaq(dst, Address(src2, src1, Address::times_1));
14519     }
14520   %}
14521   ins_pipe(ialu_reg_reg);
14522 %}
14523 
14524 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14525 %{
14526   predicate(false);
14527   match(Set dst (AddL src1 src2));
14528   format %{ "leaq    $dst, [$src1 + $src2]" %}
14529   ins_encode %{
14530     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14531   %}
14532   ins_pipe(ialu_reg_reg);
14533 %}
14534 
14535 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14536 %{
14537   predicate(false);
14538   match(Set dst (LShiftL src shift));
14539   format %{ "leaq    $dst, [$src << $shift]" %}
14540   ins_encode %{
14541     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14542     Register src = $src$$Register;
14543     if (scale == Address::times_2 && src != rbp && src != r13) {
14544       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14545     } else {
14546       __ leaq($dst$$Register, Address(noreg, src, scale));
14547     }
14548   %}
14549   ins_pipe(ialu_reg_reg);
14550 %}
14551 
14552 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14553 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14554 // processors with at least partial ALU support for lea
14555 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14556 // beneficial for processors with full ALU support
14557 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14558 
14559 peephole
14560 %{
14561   peeppredicate(VM_Version::supports_fast_2op_lea());
14562   peepmatch (addI_rReg);
14563   peepprocedure (lea_coalesce_reg);
14564   peepreplace (leaI_rReg_rReg_peep());
14565 %}
14566 
14567 peephole
14568 %{
14569   peeppredicate(VM_Version::supports_fast_2op_lea());
14570   peepmatch (addI_rReg_imm);
14571   peepprocedure (lea_coalesce_imm);
14572   peepreplace (leaI_rReg_immI_peep());
14573 %}
14574 
14575 peephole
14576 %{
14577   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14578                 VM_Version::is_intel_cascade_lake());
14579   peepmatch (incI_rReg);
14580   peepprocedure (lea_coalesce_imm);
14581   peepreplace (leaI_rReg_immI_peep());
14582 %}
14583 
14584 peephole
14585 %{
14586   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14587                 VM_Version::is_intel_cascade_lake());
14588   peepmatch (decI_rReg);
14589   peepprocedure (lea_coalesce_imm);
14590   peepreplace (leaI_rReg_immI_peep());
14591 %}
14592 
14593 peephole
14594 %{
14595   peeppredicate(VM_Version::supports_fast_2op_lea());
14596   peepmatch (salI_rReg_immI2);
14597   peepprocedure (lea_coalesce_imm);
14598   peepreplace (leaI_rReg_immI2_peep());
14599 %}
14600 
14601 peephole
14602 %{
14603   peeppredicate(VM_Version::supports_fast_2op_lea());
14604   peepmatch (addL_rReg);
14605   peepprocedure (lea_coalesce_reg);
14606   peepreplace (leaL_rReg_rReg_peep());
14607 %}
14608 
14609 peephole
14610 %{
14611   peeppredicate(VM_Version::supports_fast_2op_lea());
14612   peepmatch (addL_rReg_imm);
14613   peepprocedure (lea_coalesce_imm);
14614   peepreplace (leaL_rReg_immL32_peep());
14615 %}
14616 
14617 peephole
14618 %{
14619   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14620                 VM_Version::is_intel_cascade_lake());
14621   peepmatch (incL_rReg);
14622   peepprocedure (lea_coalesce_imm);
14623   peepreplace (leaL_rReg_immL32_peep());
14624 %}
14625 
14626 peephole
14627 %{
14628   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14629                 VM_Version::is_intel_cascade_lake());
14630   peepmatch (decL_rReg);
14631   peepprocedure (lea_coalesce_imm);
14632   peepreplace (leaL_rReg_immL32_peep());
14633 %}
14634 
14635 peephole
14636 %{
14637   peeppredicate(VM_Version::supports_fast_2op_lea());
14638   peepmatch (salL_rReg_immI2);
14639   peepprocedure (lea_coalesce_imm);
14640   peepreplace (leaL_rReg_immI2_peep());
14641 %}
14642 
14643 peephole
14644 %{
14645   peepmatch (leaPCompressedOopOffset);
14646   peepprocedure (lea_remove_redundant);
14647 %}
14648 
14649 peephole
14650 %{
14651   peepmatch (leaP8Narrow);
14652   peepprocedure (lea_remove_redundant);
14653 %}
14654 
14655 peephole
14656 %{
14657   peepmatch (leaP32Narrow);
14658   peepprocedure (lea_remove_redundant);
14659 %}
14660 
14661 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14662 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14663 
14664 //int variant
14665 peephole
14666 %{
14667   peepmatch (testI_reg);
14668   peepprocedure (test_may_remove);
14669 %}
14670 
14671 //long variant
14672 peephole
14673 %{
14674   peepmatch (testL_reg);
14675   peepprocedure (test_may_remove);
14676 %}
14677 
14678 
14679 //----------SMARTSPILL RULES---------------------------------------------------
14680 // These must follow all instruction definitions as they use the names
14681 // defined in the instructions definitions.