1 //
    2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
    3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4 //
    5 // This code is free software; you can redistribute it and/or modify it
    6 // under the terms of the GNU General Public License version 2 only, as
    7 // published by the Free Software Foundation.
    8 //
    9 // This code is distributed in the hope that it will be useful, but WITHOUT
   10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12 // version 2 for more details (a copy is included in the LICENSE file that
   13 // accompanied this code).
   14 //
   15 // You should have received a copy of the GNU General Public License version
   16 // 2 along with this work; if not, write to the Free Software Foundation,
   17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18 //
   19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20 // or visit www.oracle.com if you need additional information or have any
   21 // questions.
   22 //
   23 //
   24 
   25 // AMD64 Architecture Description File
   26 
   27 //----------REGISTER DEFINITION BLOCK------------------------------------------
   28 // This information is used by the matcher and the register allocator to
   29 // describe individual registers and classes of registers within the target
   30 // architecture.
   31 
   32 register %{
   33 //----------Architecture Description Register Definitions----------------------
   34 // General Registers
   35 // "reg_def"  name ( register save type, C convention save type,
   36 //                   ideal register type, encoding );
   37 // Register Save Types:
   38 //
   39 // NS  = No-Save:       The register allocator assumes that these registers
   40 //                      can be used without saving upon entry to the method, &
   41 //                      that they do not need to be saved at call sites.
   42 //
   43 // SOC = Save-On-Call:  The register allocator assumes that these registers
   44 //                      can be used without saving upon entry to the method,
   45 //                      but that they must be saved at call sites.
   46 //
   47 // SOE = Save-On-Entry: The register allocator assumes that these registers
   48 //                      must be saved before using them upon entry to the
   49 //                      method, but they do not need to be saved at call
   50 //                      sites.
   51 //
   52 // AS  = Always-Save:   The register allocator assumes that these registers
   53 //                      must be saved before using them upon entry to the
   54 //                      method, & that they must be saved at call sites.
   55 //
   56 // Ideal Register Type is used to determine how to save & restore a
   57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
   58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
   59 //
   60 // The encoding number is the actual bit-pattern placed into the opcodes.
   61 
   62 // General Registers
   63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
   64 // used as byte registers)
   65 
   66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
   67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
   68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
   69 
   70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
   71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
   72 
   73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
   74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
   75 
   76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
   77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
   78 
   79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
   80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
   81 
   82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
   83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
   84 
   85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
   86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
   87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
   88 
   89 #ifdef _WIN64
   90 
   91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
   92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
   93 
   94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
   95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
   96 
   97 #else
   98 
   99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
  100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
  101 
  102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
  103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
  104 
  105 #endif
  106 
  107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
  108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
  109 
  110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
  111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
  112 
  113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
  114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
  115 
  116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
  117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
  118 
  119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
  120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
  121 
  122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
  123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
  124 
  125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
  126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
  127 
  128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
  129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
  130 
  131 reg_def R16  (SOC, SOC, Op_RegI, 16, r16->as_VMReg());
  132 reg_def R16_H(SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
  133 
  134 reg_def R17  (SOC, SOC, Op_RegI, 17, r17->as_VMReg());
  135 reg_def R17_H(SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
  136 
  137 reg_def R18  (SOC, SOC, Op_RegI, 18, r18->as_VMReg());
  138 reg_def R18_H(SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
  139 
  140 reg_def R19  (SOC, SOC, Op_RegI, 19, r19->as_VMReg());
  141 reg_def R19_H(SOC, SOC, Op_RegI, 19, r19->as_VMReg()->next());
  142 
  143 reg_def R20  (SOC, SOC, Op_RegI, 20, r20->as_VMReg());
  144 reg_def R20_H(SOC, SOC, Op_RegI, 20, r20->as_VMReg()->next());
  145 
  146 reg_def R21  (SOC, SOC, Op_RegI, 21, r21->as_VMReg());
  147 reg_def R21_H(SOC, SOC, Op_RegI, 21, r21->as_VMReg()->next());
  148 
  149 reg_def R22  (SOC, SOC, Op_RegI, 22, r22->as_VMReg());
  150 reg_def R22_H(SOC, SOC, Op_RegI, 22, r22->as_VMReg()->next());
  151 
  152 reg_def R23  (SOC, SOC, Op_RegI, 23, r23->as_VMReg());
  153 reg_def R23_H(SOC, SOC, Op_RegI, 23, r23->as_VMReg()->next());
  154 
  155 reg_def R24  (SOC, SOC, Op_RegI, 24, r24->as_VMReg());
  156 reg_def R24_H(SOC, SOC, Op_RegI, 24, r24->as_VMReg()->next());
  157 
  158 reg_def R25  (SOC, SOC, Op_RegI, 25, r25->as_VMReg());
  159 reg_def R25_H(SOC, SOC, Op_RegI, 25, r25->as_VMReg()->next());
  160 
  161 reg_def R26  (SOC, SOC, Op_RegI, 26, r26->as_VMReg());
  162 reg_def R26_H(SOC, SOC, Op_RegI, 26, r26->as_VMReg()->next());
  163 
  164 reg_def R27  (SOC, SOC, Op_RegI, 27, r27->as_VMReg());
  165 reg_def R27_H(SOC, SOC, Op_RegI, 27, r27->as_VMReg()->next());
  166 
  167 reg_def R28  (SOC, SOC, Op_RegI, 28, r28->as_VMReg());
  168 reg_def R28_H(SOC, SOC, Op_RegI, 28, r28->as_VMReg()->next());
  169 
  170 reg_def R29  (SOC, SOC, Op_RegI, 29, r29->as_VMReg());
  171 reg_def R29_H(SOC, SOC, Op_RegI, 29, r29->as_VMReg()->next());
  172 
  173 reg_def R30  (SOC, SOC, Op_RegI, 30, r30->as_VMReg());
  174 reg_def R30_H(SOC, SOC, Op_RegI, 30, r30->as_VMReg()->next());
  175 
  176 reg_def R31  (SOC, SOC, Op_RegI, 31, r31->as_VMReg());
  177 reg_def R31_H(SOC, SOC, Op_RegI, 31, r31->as_VMReg()->next());
  178 
  179 // Floating Point Registers
  180 
  181 // Specify priority of register selection within phases of register
  182 // allocation.  Highest priority is first.  A useful heuristic is to
  183 // give registers a low priority when they are required by machine
  184 // instructions, like EAX and EDX on I486, and choose no-save registers
  185 // before save-on-call, & save-on-call before save-on-entry.  Registers
  186 // which participate in fixed calling sequences should come last.
  187 // Registers which are used as pairs must fall on an even boundary.
  188 
  189 alloc_class chunk0(R10,         R10_H,
  190                    R11,         R11_H,
  191                    R8,          R8_H,
  192                    R9,          R9_H,
  193                    R12,         R12_H,
  194                    RCX,         RCX_H,
  195                    RBX,         RBX_H,
  196                    RDI,         RDI_H,
  197                    RDX,         RDX_H,
  198                    RSI,         RSI_H,
  199                    RAX,         RAX_H,
  200                    RBP,         RBP_H,
  201                    R13,         R13_H,
  202                    R14,         R14_H,
  203                    R15,         R15_H,
  204                    R16,         R16_H,
  205                    R17,         R17_H,
  206                    R18,         R18_H,
  207                    R19,         R19_H,
  208                    R20,         R20_H,
  209                    R21,         R21_H,
  210                    R22,         R22_H,
  211                    R23,         R23_H,
  212                    R24,         R24_H,
  213                    R25,         R25_H,
  214                    R26,         R26_H,
  215                    R27,         R27_H,
  216                    R28,         R28_H,
  217                    R29,         R29_H,
  218                    R30,         R30_H,
  219                    R31,         R31_H,
  220                    RSP,         RSP_H);
  221 
  222 
  223 //----------Architecture Description Register Classes--------------------------
  224 // Several register classes are automatically defined based upon information in
  225 // this architecture description.
  226 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
  227 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
  228 //
  229 
  230 // Empty register class.
  231 reg_class no_reg();
  232 
  233 // Class for all pointer/long registers including APX extended GPRs.
  234 reg_class all_reg(RAX, RAX_H,
  235                   RDX, RDX_H,
  236                   RBP, RBP_H,
  237                   RDI, RDI_H,
  238                   RSI, RSI_H,
  239                   RCX, RCX_H,
  240                   RBX, RBX_H,
  241                   RSP, RSP_H,
  242                   R8,  R8_H,
  243                   R9,  R9_H,
  244                   R10, R10_H,
  245                   R11, R11_H,
  246                   R12, R12_H,
  247                   R13, R13_H,
  248                   R14, R14_H,
  249                   R15, R15_H,
  250                   R16, R16_H,
  251                   R17, R17_H,
  252                   R18, R18_H,
  253                   R19, R19_H,
  254                   R20, R20_H,
  255                   R21, R21_H,
  256                   R22, R22_H,
  257                   R23, R23_H,
  258                   R24, R24_H,
  259                   R25, R25_H,
  260                   R26, R26_H,
  261                   R27, R27_H,
  262                   R28, R28_H,
  263                   R29, R29_H,
  264                   R30, R30_H,
  265                   R31, R31_H);
  266 
  267 // Class for all int registers including APX extended GPRs.
  268 reg_class all_int_reg(RAX
  269                       RDX,
  270                       RBP,
  271                       RDI,
  272                       RSI,
  273                       RCX,
  274                       RBX,
  275                       R8,
  276                       R9,
  277                       R10,
  278                       R11,
  279                       R12,
  280                       R13,
  281                       R14,
  282                       R16,
  283                       R17,
  284                       R18,
  285                       R19,
  286                       R20,
  287                       R21,
  288                       R22,
  289                       R23,
  290                       R24,
  291                       R25,
  292                       R26,
  293                       R27,
  294                       R28,
  295                       R29,
  296                       R30,
  297                       R31);
  298 
  299 // Class for all pointer registers
  300 reg_class any_reg %{
  301   return _ANY_REG_mask;
  302 %}
  303 
  304 // Class for all pointer registers (excluding RSP)
  305 reg_class ptr_reg %{
  306   return _PTR_REG_mask;
  307 %}
  308 
  309 // Class for all pointer registers (excluding RSP and RBP)
  310 reg_class ptr_reg_no_rbp %{
  311   return _PTR_REG_NO_RBP_mask;
  312 %}
  313 
  314 // Class for all pointer registers (excluding RAX and RSP)
  315 reg_class ptr_no_rax_reg %{
  316   return _PTR_NO_RAX_REG_mask;
  317 %}
  318 
  319 // Class for all pointer registers (excluding RAX, RBX, and RSP)
  320 reg_class ptr_no_rax_rbx_reg %{
  321   return _PTR_NO_RAX_RBX_REG_mask;
  322 %}
  323 
  324 // Class for all long registers (excluding RSP)
  325 reg_class long_reg %{
  326   return _LONG_REG_mask;
  327 %}
  328 
  329 // Class for all long registers (excluding RAX, RDX and RSP)
  330 reg_class long_no_rax_rdx_reg %{
  331   return _LONG_NO_RAX_RDX_REG_mask;
  332 %}
  333 
  334 // Class for all long registers (excluding RCX and RSP)
  335 reg_class long_no_rcx_reg %{
  336   return _LONG_NO_RCX_REG_mask;
  337 %}
  338 
  339 // Class for all long registers (excluding RBP and R13)
  340 reg_class long_no_rbp_r13_reg %{
  341   return _LONG_NO_RBP_R13_REG_mask;
  342 %}
  343 
  344 // Class for all int registers (excluding RSP)
  345 reg_class int_reg %{
  346   return _INT_REG_mask;
  347 %}
  348 
  349 // Class for all int registers (excluding RAX, RDX, and RSP)
  350 reg_class int_no_rax_rdx_reg %{
  351   return _INT_NO_RAX_RDX_REG_mask;
  352 %}
  353 
  354 // Class for all int registers (excluding RCX and RSP)
  355 reg_class int_no_rcx_reg %{
  356   return _INT_NO_RCX_REG_mask;
  357 %}
  358 
  359 // Class for all int registers (excluding RBP and R13)
  360 reg_class int_no_rbp_r13_reg %{
  361   return _INT_NO_RBP_R13_REG_mask;
  362 %}
  363 
  364 // Singleton class for RAX pointer register
  365 reg_class ptr_rax_reg(RAX, RAX_H);
  366 
  367 // Singleton class for RBX pointer register
  368 reg_class ptr_rbx_reg(RBX, RBX_H);
  369 
  370 // Singleton class for RSI pointer register
  371 reg_class ptr_rsi_reg(RSI, RSI_H);
  372 
  373 // Singleton class for RBP pointer register
  374 reg_class ptr_rbp_reg(RBP, RBP_H);
  375 
  376 // Singleton class for RDI pointer register
  377 reg_class ptr_rdi_reg(RDI, RDI_H);
  378 
  379 // Singleton class for stack pointer
  380 reg_class ptr_rsp_reg(RSP, RSP_H);
  381 
  382 // Singleton class for TLS pointer
  383 reg_class ptr_r15_reg(R15, R15_H);
  384 
  385 // Singleton class for RAX long register
  386 reg_class long_rax_reg(RAX, RAX_H);
  387 
  388 // Singleton class for RCX long register
  389 reg_class long_rcx_reg(RCX, RCX_H);
  390 
  391 // Singleton class for RDX long register
  392 reg_class long_rdx_reg(RDX, RDX_H);
  393 
  394 // Singleton class for R11 long register
  395 reg_class long_r11_reg(R11, R11_H);
  396 
  397 // Singleton class for RAX int register
  398 reg_class int_rax_reg(RAX);
  399 
  400 // Singleton class for RBX int register
  401 reg_class int_rbx_reg(RBX);
  402 
  403 // Singleton class for RCX int register
  404 reg_class int_rcx_reg(RCX);
  405 
  406 // Singleton class for RDX int register
  407 reg_class int_rdx_reg(RDX);
  408 
  409 // Singleton class for RDI int register
  410 reg_class int_rdi_reg(RDI);
  411 
  412 // Singleton class for instruction pointer
  413 // reg_class ip_reg(RIP);
  414 
  415 %}
  416 
  417 //----------SOURCE BLOCK-------------------------------------------------------
  418 // This is a block of C++ code which provides values, functions, and
  419 // definitions necessary in the rest of the architecture description
  420 
  421 source_hpp %{
  422 
  423 #include "peephole_x86_64.hpp"
  424 
  425 bool castLL_is_imm32(const Node* n);
  426 
  427 %}
  428 
  429 source %{
  430 
  431 bool castLL_is_imm32(const Node* n) {
  432   assert(n->is_CastLL(), "must be a CastLL");
  433   const TypeLong* t = n->bottom_type()->is_long();
  434   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
  435 }
  436 
  437 %}
  438 
  439 // Register masks
  440 source_hpp %{
  441 extern RegMask _ANY_REG_mask;
  442 extern RegMask _PTR_REG_mask;
  443 extern RegMask _PTR_REG_NO_RBP_mask;
  444 extern RegMask _PTR_NO_RAX_REG_mask;
  445 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
  446 extern RegMask _LONG_REG_mask;
  447 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
  448 extern RegMask _LONG_NO_RCX_REG_mask;
  449 extern RegMask _LONG_NO_RBP_R13_REG_mask;
  450 extern RegMask _INT_REG_mask;
  451 extern RegMask _INT_NO_RAX_RDX_REG_mask;
  452 extern RegMask _INT_NO_RCX_REG_mask;
  453 extern RegMask _INT_NO_RBP_R13_REG_mask;
  454 extern RegMask _FLOAT_REG_mask;
  455 
  456 extern RegMask _STACK_OR_PTR_REG_mask;
  457 extern RegMask _STACK_OR_LONG_REG_mask;
  458 extern RegMask _STACK_OR_INT_REG_mask;
  459 
  460 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
  461 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
  462 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
  463 
  464 %}
  465 
  466 source %{
  467 #define   RELOC_IMM64    Assembler::imm_operand
  468 #define   RELOC_DISP32   Assembler::disp32_operand
  469 
  470 #define __ masm->
  471 
  472 RegMask _ANY_REG_mask;
  473 RegMask _PTR_REG_mask;
  474 RegMask _PTR_REG_NO_RBP_mask;
  475 RegMask _PTR_NO_RAX_REG_mask;
  476 RegMask _PTR_NO_RAX_RBX_REG_mask;
  477 RegMask _LONG_REG_mask;
  478 RegMask _LONG_NO_RAX_RDX_REG_mask;
  479 RegMask _LONG_NO_RCX_REG_mask;
  480 RegMask _LONG_NO_RBP_R13_REG_mask;
  481 RegMask _INT_REG_mask;
  482 RegMask _INT_NO_RAX_RDX_REG_mask;
  483 RegMask _INT_NO_RCX_REG_mask;
  484 RegMask _INT_NO_RBP_R13_REG_mask;
  485 RegMask _FLOAT_REG_mask;
  486 RegMask _STACK_OR_PTR_REG_mask;
  487 RegMask _STACK_OR_LONG_REG_mask;
  488 RegMask _STACK_OR_INT_REG_mask;
  489 
  490 static bool need_r12_heapbase() {
  491   return UseCompressedOops;
  492 }
  493 
  494 void reg_mask_init() {
  495   constexpr Register egprs[] = {r16, r17, r18, r19, r20, r21, r22, r23, r24, r25, r26, r27, r28, r29, r30, r31};
  496 
  497   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
  498   // We derive a number of subsets from it.
  499   _ANY_REG_mask = _ALL_REG_mask;
  500 
  501   if (PreserveFramePointer) {
  502     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  503     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  504   }
  505   if (need_r12_heapbase()) {
  506     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  507     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
  508   }
  509 
  510   _PTR_REG_mask = _ANY_REG_mask;
  511   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
  512   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
  513   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
  514   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
  515   if (!UseAPX) {
  516     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  517       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  518       _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
  519     }
  520   }
  521 
  522   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
  523   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  524 
  525   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
  526   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  527   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  528 
  529   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
  530   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  531   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  532 
  533   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
  534   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
  535   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
  536 
  537 
  538   _LONG_REG_mask = _PTR_REG_mask;
  539   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
  540   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  541 
  542   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
  543   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  544   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
  545   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  546   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
  547 
  548   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
  549   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  550   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
  551 
  552   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
  553   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  554   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
  555   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  556   _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
  557 
  558   _INT_REG_mask = _ALL_INT_REG_mask;
  559   if (!UseAPX) {
  560     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
  561       _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
  562     }
  563   }
  564 
  565   if (PreserveFramePointer) {
  566     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  567   }
  568   if (need_r12_heapbase()) {
  569     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
  570   }
  571 
  572   _STACK_OR_INT_REG_mask = _INT_REG_mask;
  573   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
  574 
  575   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
  576   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
  577   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
  578 
  579   _INT_NO_RCX_REG_mask = _INT_REG_mask;
  580   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
  581 
  582   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
  583   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
  584   _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
  585 
  586   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
  587   // from the float_reg_legacy/float_reg_evex register class.
  588   _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
  589 }
  590 
  591 static bool generate_vzeroupper(Compile* C) {
  592   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
  593 }
  594 
  595 static int clear_avx_size() {
  596   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
  597 }
  598 
  599 // !!!!! Special hack to get all types of calls to specify the byte offset
  600 //       from the start of the call to the point where the return address
  601 //       will point.
  602 int MachCallStaticJavaNode::ret_addr_offset()
  603 {
  604   int offset = 5; // 5 bytes from start of call to where return address points
  605   offset += clear_avx_size();
  606   return offset;
  607 }
  608 
  609 int MachCallDynamicJavaNode::ret_addr_offset()
  610 {
  611   int offset = 15; // 15 bytes from start of call to where return address points
  612   offset += clear_avx_size();
  613   return offset;
  614 }
  615 
  616 int MachCallRuntimeNode::ret_addr_offset() {
  617   int offset = 13; // movq r10,#addr; callq (r10)
  618   if (this->ideal_Opcode() != Op_CallLeafVector) {
  619     offset += clear_avx_size();
  620   }
  621   return offset;
  622 }
  623 //
  624 // Compute padding required for nodes which need alignment
  625 //
  626 
  627 // The address of the call instruction needs to be 4-byte aligned to
  628 // ensure that it does not span a cache line so that it can be patched.
  629 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
  630 {
  631   current_offset += clear_avx_size(); // skip vzeroupper
  632   current_offset += 1; // skip call opcode byte
  633   return align_up(current_offset, alignment_required()) - current_offset;
  634 }
  635 
  636 // The address of the call instruction needs to be 4-byte aligned to
  637 // ensure that it does not span a cache line so that it can be patched.
  638 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
  639 {
  640   current_offset += clear_avx_size(); // skip vzeroupper
  641   current_offset += 11; // skip movq instruction + call opcode byte
  642   return align_up(current_offset, alignment_required()) - current_offset;
  643 }
  644 
  645 // This could be in MacroAssembler but it's fairly C2 specific
  646 static void emit_cmpfp_fixup(MacroAssembler* masm) {
  647   Label exit;
  648   __ jccb(Assembler::noParity, exit);
  649   __ pushf();
  650   //
  651   // comiss/ucomiss instructions set ZF,PF,CF flags and
  652   // zero OF,AF,SF for NaN values.
  653   // Fixup flags by zeroing ZF,PF so that compare of NaN
  654   // values returns 'less than' result (CF is set).
  655   // Leave the rest of flags unchanged.
  656   //
  657   //    7 6 5 4 3 2 1 0
  658   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
  659   //    0 0 1 0 1 0 1 1   (0x2B)
  660   //
  661   __ andq(Address(rsp, 0), 0xffffff2b);
  662   __ popf();
  663   __ bind(exit);
  664 }
  665 
  666 static void emit_cmpfp3(MacroAssembler* masm, Register dst) {
  667   Label done;
  668   __ movl(dst, -1);
  669   __ jcc(Assembler::parity, done);
  670   __ jcc(Assembler::below, done);
  671   __ setcc(Assembler::notEqual, dst);
  672   __ bind(done);
  673 }
  674 
  675 // Math.min()    # Math.max()
  676 // --------------------------
  677 // ucomis[s/d]   #
  678 // ja   -> b     # a
  679 // jp   -> NaN   # NaN
  680 // jb   -> a     # b
  681 // je            #
  682 // |-jz -> a | b # a & b
  683 // |    -> a     #
  684 static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
  685                             XMMRegister a, XMMRegister b,
  686                             XMMRegister xmmt, Register rt,
  687                             bool min, bool single) {
  688 
  689   Label nan, zero, below, above, done;
  690 
  691   if (single)
  692     __ ucomiss(a, b);
  693   else
  694     __ ucomisd(a, b);
  695 
  696   if (dst->encoding() != (min ? b : a)->encoding())
  697     __ jccb(Assembler::above, above); // CF=0 & ZF=0
  698   else
  699     __ jccb(Assembler::above, done);
  700 
  701   __ jccb(Assembler::parity, nan);  // PF=1
  702   __ jccb(Assembler::below, below); // CF=1
  703 
  704   // equal
  705   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
  706   if (single) {
  707     __ ucomiss(a, xmmt);
  708     __ jccb(Assembler::equal, zero);
  709 
  710     __ movflt(dst, a);
  711     __ jmp(done);
  712   }
  713   else {
  714     __ ucomisd(a, xmmt);
  715     __ jccb(Assembler::equal, zero);
  716 
  717     __ movdbl(dst, a);
  718     __ jmp(done);
  719   }
  720 
  721   __ bind(zero);
  722   if (min)
  723     __ vpor(dst, a, b, Assembler::AVX_128bit);
  724   else
  725     __ vpand(dst, a, b, Assembler::AVX_128bit);
  726 
  727   __ jmp(done);
  728 
  729   __ bind(above);
  730   if (single)
  731     __ movflt(dst, min ? b : a);
  732   else
  733     __ movdbl(dst, min ? b : a);
  734 
  735   __ jmp(done);
  736 
  737   __ bind(nan);
  738   if (single) {
  739     __ movl(rt, 0x7fc00000); // Float.NaN
  740     __ movdl(dst, rt);
  741   }
  742   else {
  743     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
  744     __ movdq(dst, rt);
  745   }
  746   __ jmp(done);
  747 
  748   __ bind(below);
  749   if (single)
  750     __ movflt(dst, min ? a : b);
  751   else
  752     __ movdbl(dst, min ? a : b);
  753 
  754   __ bind(done);
  755 }
  756 
  757 //=============================================================================
  758 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
  759 
  760 int ConstantTable::calculate_table_base_offset() const {
  761   return 0;  // absolute addressing, no offset
  762 }
  763 
  764 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
  765 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
  766   ShouldNotReachHere();
  767 }
  768 
  769 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
  770   // Empty encoding
  771 }
  772 
  773 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
  774   return 0;
  775 }
  776 
  777 #ifndef PRODUCT
  778 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  779   st->print("# MachConstantBaseNode (empty encoding)");
  780 }
  781 #endif
  782 
  783 
  784 //=============================================================================
  785 #ifndef PRODUCT
  786 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
  787   Compile* C = ra_->C;
  788 
  789   int framesize = C->output()->frame_size_in_bytes();
  790   int bangsize = C->output()->bang_size_in_bytes();
  791   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  792   // Remove wordSize for return addr which is already pushed.
  793   framesize -= wordSize;
  794 
  795   if (C->output()->need_stack_bang(bangsize)) {
  796     framesize -= wordSize;
  797     st->print("# stack bang (%d bytes)", bangsize);
  798     st->print("\n\t");
  799     st->print("pushq   rbp\t# Save rbp");
  800     if (PreserveFramePointer) {
  801         st->print("\n\t");
  802         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  803     }
  804     if (framesize) {
  805       st->print("\n\t");
  806       st->print("subq    rsp, #%d\t# Create frame",framesize);
  807     }
  808   } else {
  809     st->print("subq    rsp, #%d\t# Create frame",framesize);
  810     st->print("\n\t");
  811     framesize -= wordSize;
  812     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
  813     if (PreserveFramePointer) {
  814       st->print("\n\t");
  815       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
  816       if (framesize > 0) {
  817         st->print("\n\t");
  818         st->print("addq    rbp, #%d", framesize);
  819       }
  820     }
  821   }
  822 
  823   if (VerifyStackAtCalls) {
  824     st->print("\n\t");
  825     framesize -= wordSize;
  826     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
  827 #ifdef ASSERT
  828     st->print("\n\t");
  829     st->print("# stack alignment check");
  830 #endif
  831   }
  832   if (C->stub_function() != nullptr) {
  833     st->print("\n\t");
  834     st->print("cmpl    [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
  835     st->print("\n\t");
  836     st->print("je      fast_entry\t");
  837     st->print("\n\t");
  838     st->print("call    #nmethod_entry_barrier_stub\t");
  839     st->print("\n\tfast_entry:");
  840   }
  841   st->cr();
  842 }
  843 #endif
  844 
  845 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
  846   Compile* C = ra_->C;
  847 
  848   int framesize = C->output()->frame_size_in_bytes();
  849   int bangsize = C->output()->bang_size_in_bytes();
  850 
  851   if (C->clinit_barrier_on_entry()) {
  852     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
  853     assert(!C->method()->holder()->is_not_initialized() || C->do_clinit_barriers(), "initialization should have been started");
  854 
  855     Label L_skip_barrier;
  856     Register klass = rscratch1;
  857 
  858     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
  859     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
  860 
  861     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
  862 
  863     __ bind(L_skip_barrier);
  864   }
  865 
  866   __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != nullptr);
  867 
  868   C->output()->set_frame_complete(__ offset());
  869 
  870   if (C->has_mach_constant_base_node()) {
  871     // NOTE: We set the table base offset here because users might be
  872     // emitted before MachConstantBaseNode.
  873     ConstantTable& constant_table = C->output()->constant_table();
  874     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
  875   }
  876 }
  877 
  878 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
  879 {
  880   return MachNode::size(ra_); // too many variables; just compute it
  881                               // the hard way
  882 }
  883 
  884 int MachPrologNode::reloc() const
  885 {
  886   return 0; // a large enough number
  887 }
  888 
  889 //=============================================================================
  890 #ifndef PRODUCT
  891 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
  892 {
  893   Compile* C = ra_->C;
  894   if (generate_vzeroupper(C)) {
  895     st->print("vzeroupper");
  896     st->cr(); st->print("\t");
  897   }
  898 
  899   int framesize = C->output()->frame_size_in_bytes();
  900   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  901   // Remove word for return adr already pushed
  902   // and RBP
  903   framesize -= 2*wordSize;
  904 
  905   if (framesize) {
  906     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
  907     st->print("\t");
  908   }
  909 
  910   st->print_cr("popq    rbp");
  911   if (do_polling() && C->is_method_compilation()) {
  912     st->print("\t");
  913     st->print_cr("cmpq    rsp, poll_offset[r15_thread] \n\t"
  914                  "ja      #safepoint_stub\t"
  915                  "# Safepoint: poll for GC");
  916   }
  917 }
  918 #endif
  919 
  920 void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
  921 {
  922   Compile* C = ra_->C;
  923 
  924   if (generate_vzeroupper(C)) {
  925     // Clear upper bits of YMM registers when current compiled code uses
  926     // wide vectors to avoid AVX <-> SSE transition penalty during call.
  927     __ vzeroupper();
  928   }
  929 
  930   int framesize = C->output()->frame_size_in_bytes();
  931   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
  932   // Remove word for return adr already pushed
  933   // and RBP
  934   framesize -= 2*wordSize;
  935 
  936   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
  937 
  938   if (framesize) {
  939     __ addq(rsp, framesize);
  940   }
  941 
  942   __ popq(rbp);
  943 
  944   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
  945     __ reserved_stack_check();
  946   }
  947 
  948   if (do_polling() && C->is_method_compilation()) {
  949     Label dummy_label;
  950     Label* code_stub = &dummy_label;
  951     if (!C->output()->in_scratch_emit_size()) {
  952       C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
  953       C->output()->add_stub(stub);
  954       code_stub = &stub->entry();
  955     }
  956     __ relocate(relocInfo::poll_return_type);
  957     __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
  958   }
  959 }
  960 
  961 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
  962 {
  963   return MachNode::size(ra_); // too many variables; just compute it
  964                               // the hard way
  965 }
  966 
  967 int MachEpilogNode::reloc() const
  968 {
  969   return 2; // a large enough number
  970 }
  971 
  972 const Pipeline* MachEpilogNode::pipeline() const
  973 {
  974   return MachNode::pipeline_class();
  975 }
  976 
  977 //=============================================================================
  978 
  979 enum RC {
  980   rc_bad,
  981   rc_int,
  982   rc_kreg,
  983   rc_float,
  984   rc_stack
  985 };
  986 
  987 static enum RC rc_class(OptoReg::Name reg)
  988 {
  989   if( !OptoReg::is_valid(reg)  ) return rc_bad;
  990 
  991   if (OptoReg::is_stack(reg)) return rc_stack;
  992 
  993   VMReg r = OptoReg::as_VMReg(reg);
  994 
  995   if (r->is_Register()) return rc_int;
  996 
  997   if (r->is_KRegister()) return rc_kreg;
  998 
  999   assert(r->is_XMMRegister(), "must be");
 1000   return rc_float;
 1001 }
 1002 
 1003 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 1004 static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
 1005                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 1006 
 1007 void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
 1008                      int stack_offset, int reg, uint ireg, outputStream* st);
 1009 
 1010 static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
 1011                                       int dst_offset, uint ireg, outputStream* st) {
 1012   if (masm) {
 1013     switch (ireg) {
 1014     case Op_VecS:
 1015       __ movq(Address(rsp, -8), rax);
 1016       __ movl(rax, Address(rsp, src_offset));
 1017       __ movl(Address(rsp, dst_offset), rax);
 1018       __ movq(rax, Address(rsp, -8));
 1019       break;
 1020     case Op_VecD:
 1021       __ pushq(Address(rsp, src_offset));
 1022       __ popq (Address(rsp, dst_offset));
 1023       break;
 1024     case Op_VecX:
 1025       __ pushq(Address(rsp, src_offset));
 1026       __ popq (Address(rsp, dst_offset));
 1027       __ pushq(Address(rsp, src_offset+8));
 1028       __ popq (Address(rsp, dst_offset+8));
 1029       break;
 1030     case Op_VecY:
 1031       __ vmovdqu(Address(rsp, -32), xmm0);
 1032       __ vmovdqu(xmm0, Address(rsp, src_offset));
 1033       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 1034       __ vmovdqu(xmm0, Address(rsp, -32));
 1035       break;
 1036     case Op_VecZ:
 1037       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 1038       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 1039       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
 1040       __ evmovdquq(xmm0, Address(rsp, -64), 2);
 1041       break;
 1042     default:
 1043       ShouldNotReachHere();
 1044     }
 1045 #ifndef PRODUCT
 1046   } else {
 1047     switch (ireg) {
 1048     case Op_VecS:
 1049       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1050                 "movl    rax, [rsp + #%d]\n\t"
 1051                 "movl    [rsp + #%d], rax\n\t"
 1052                 "movq    rax, [rsp - #8]",
 1053                 src_offset, dst_offset);
 1054       break;
 1055     case Op_VecD:
 1056       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1057                 "popq    [rsp + #%d]",
 1058                 src_offset, dst_offset);
 1059       break;
 1060      case Op_VecX:
 1061       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 1062                 "popq    [rsp + #%d]\n\t"
 1063                 "pushq   [rsp + #%d]\n\t"
 1064                 "popq    [rsp + #%d]",
 1065                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 1066       break;
 1067     case Op_VecY:
 1068       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 1069                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1070                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1071                 "vmovdqu xmm0, [rsp - #32]",
 1072                 src_offset, dst_offset);
 1073       break;
 1074     case Op_VecZ:
 1075       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
 1076                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 1077                 "vmovdqu [rsp + #%d], xmm0\n\t"
 1078                 "vmovdqu xmm0, [rsp - #64]",
 1079                 src_offset, dst_offset);
 1080       break;
 1081     default:
 1082       ShouldNotReachHere();
 1083     }
 1084 #endif
 1085   }
 1086 }
 1087 
 1088 uint MachSpillCopyNode::implementation(C2_MacroAssembler* masm,
 1089                                        PhaseRegAlloc* ra_,
 1090                                        bool do_size,
 1091                                        outputStream* st) const {
 1092   assert(masm != nullptr || st  != nullptr, "sanity");
 1093   // Get registers to move
 1094   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 1095   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 1096   OptoReg::Name dst_second = ra_->get_reg_second(this);
 1097   OptoReg::Name dst_first = ra_->get_reg_first(this);
 1098 
 1099   enum RC src_second_rc = rc_class(src_second);
 1100   enum RC src_first_rc = rc_class(src_first);
 1101   enum RC dst_second_rc = rc_class(dst_second);
 1102   enum RC dst_first_rc = rc_class(dst_first);
 1103 
 1104   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 1105          "must move at least 1 register" );
 1106 
 1107   if (src_first == dst_first && src_second == dst_second) {
 1108     // Self copy, no move
 1109     return 0;
 1110   }
 1111   if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
 1112     uint ireg = ideal_reg();
 1113     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
 1114     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
 1115     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 1116       // mem -> mem
 1117       int src_offset = ra_->reg2offset(src_first);
 1118       int dst_offset = ra_->reg2offset(dst_first);
 1119       vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
 1120     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
 1121       vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
 1122     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 1123       int stack_offset = ra_->reg2offset(dst_first);
 1124       vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
 1125     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
 1126       int stack_offset = ra_->reg2offset(src_first);
 1127       vec_spill_helper(masm, true,  stack_offset, dst_first, ireg, st);
 1128     } else {
 1129       ShouldNotReachHere();
 1130     }
 1131     return 0;
 1132   }
 1133   if (src_first_rc == rc_stack) {
 1134     // mem ->
 1135     if (dst_first_rc == rc_stack) {
 1136       // mem -> mem
 1137       assert(src_second != dst_first, "overlap");
 1138       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1139           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1140         // 64-bit
 1141         int src_offset = ra_->reg2offset(src_first);
 1142         int dst_offset = ra_->reg2offset(dst_first);
 1143         if (masm) {
 1144           __ pushq(Address(rsp, src_offset));
 1145           __ popq (Address(rsp, dst_offset));
 1146 #ifndef PRODUCT
 1147         } else {
 1148           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 1149                     "popq    [rsp + #%d]",
 1150                      src_offset, dst_offset);
 1151 #endif
 1152         }
 1153       } else {
 1154         // 32-bit
 1155         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1156         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1157         // No pushl/popl, so:
 1158         int src_offset = ra_->reg2offset(src_first);
 1159         int dst_offset = ra_->reg2offset(dst_first);
 1160         if (masm) {
 1161           __ movq(Address(rsp, -8), rax);
 1162           __ movl(rax, Address(rsp, src_offset));
 1163           __ movl(Address(rsp, dst_offset), rax);
 1164           __ movq(rax, Address(rsp, -8));
 1165 #ifndef PRODUCT
 1166         } else {
 1167           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 1168                     "movl    rax, [rsp + #%d]\n\t"
 1169                     "movl    [rsp + #%d], rax\n\t"
 1170                     "movq    rax, [rsp - #8]",
 1171                      src_offset, dst_offset);
 1172 #endif
 1173         }
 1174       }
 1175       return 0;
 1176     } else if (dst_first_rc == rc_int) {
 1177       // mem -> gpr
 1178       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1179           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1180         // 64-bit
 1181         int offset = ra_->reg2offset(src_first);
 1182         if (masm) {
 1183           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1184 #ifndef PRODUCT
 1185         } else {
 1186           st->print("movq    %s, [rsp + #%d]\t# spill",
 1187                      Matcher::regName[dst_first],
 1188                      offset);
 1189 #endif
 1190         }
 1191       } else {
 1192         // 32-bit
 1193         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1194         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1195         int offset = ra_->reg2offset(src_first);
 1196         if (masm) {
 1197           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1198 #ifndef PRODUCT
 1199         } else {
 1200           st->print("movl    %s, [rsp + #%d]\t# spill",
 1201                      Matcher::regName[dst_first],
 1202                      offset);
 1203 #endif
 1204         }
 1205       }
 1206       return 0;
 1207     } else if (dst_first_rc == rc_float) {
 1208       // mem-> xmm
 1209       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1210           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1211         // 64-bit
 1212         int offset = ra_->reg2offset(src_first);
 1213         if (masm) {
 1214           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1215 #ifndef PRODUCT
 1216         } else {
 1217           st->print("%s  %s, [rsp + #%d]\t# spill",
 1218                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 1219                      Matcher::regName[dst_first],
 1220                      offset);
 1221 #endif
 1222         }
 1223       } else {
 1224         // 32-bit
 1225         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1226         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1227         int offset = ra_->reg2offset(src_first);
 1228         if (masm) {
 1229           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1230 #ifndef PRODUCT
 1231         } else {
 1232           st->print("movss   %s, [rsp + #%d]\t# spill",
 1233                      Matcher::regName[dst_first],
 1234                      offset);
 1235 #endif
 1236         }
 1237       }
 1238       return 0;
 1239     } else if (dst_first_rc == rc_kreg) {
 1240       // mem -> kreg
 1241       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1242           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1243         // 64-bit
 1244         int offset = ra_->reg2offset(src_first);
 1245         if (masm) {
 1246           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
 1247 #ifndef PRODUCT
 1248         } else {
 1249           st->print("kmovq   %s, [rsp + #%d]\t# spill",
 1250                      Matcher::regName[dst_first],
 1251                      offset);
 1252 #endif
 1253         }
 1254       }
 1255       return 0;
 1256     }
 1257   } else if (src_first_rc == rc_int) {
 1258     // gpr ->
 1259     if (dst_first_rc == rc_stack) {
 1260       // gpr -> mem
 1261       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1262           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1263         // 64-bit
 1264         int offset = ra_->reg2offset(dst_first);
 1265         if (masm) {
 1266           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1267 #ifndef PRODUCT
 1268         } else {
 1269           st->print("movq    [rsp + #%d], %s\t# spill",
 1270                      offset,
 1271                      Matcher::regName[src_first]);
 1272 #endif
 1273         }
 1274       } else {
 1275         // 32-bit
 1276         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1277         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1278         int offset = ra_->reg2offset(dst_first);
 1279         if (masm) {
 1280           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
 1281 #ifndef PRODUCT
 1282         } else {
 1283           st->print("movl    [rsp + #%d], %s\t# spill",
 1284                      offset,
 1285                      Matcher::regName[src_first]);
 1286 #endif
 1287         }
 1288       }
 1289       return 0;
 1290     } else if (dst_first_rc == rc_int) {
 1291       // gpr -> gpr
 1292       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1293           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1294         // 64-bit
 1295         if (masm) {
 1296           __ movq(as_Register(Matcher::_regEncode[dst_first]),
 1297                   as_Register(Matcher::_regEncode[src_first]));
 1298 #ifndef PRODUCT
 1299         } else {
 1300           st->print("movq    %s, %s\t# spill",
 1301                      Matcher::regName[dst_first],
 1302                      Matcher::regName[src_first]);
 1303 #endif
 1304         }
 1305         return 0;
 1306       } else {
 1307         // 32-bit
 1308         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1309         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1310         if (masm) {
 1311           __ movl(as_Register(Matcher::_regEncode[dst_first]),
 1312                   as_Register(Matcher::_regEncode[src_first]));
 1313 #ifndef PRODUCT
 1314         } else {
 1315           st->print("movl    %s, %s\t# spill",
 1316                      Matcher::regName[dst_first],
 1317                      Matcher::regName[src_first]);
 1318 #endif
 1319         }
 1320         return 0;
 1321       }
 1322     } else if (dst_first_rc == rc_float) {
 1323       // gpr -> xmm
 1324       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1325           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1326         // 64-bit
 1327         if (masm) {
 1328           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1329 #ifndef PRODUCT
 1330         } else {
 1331           st->print("movdq   %s, %s\t# spill",
 1332                      Matcher::regName[dst_first],
 1333                      Matcher::regName[src_first]);
 1334 #endif
 1335         }
 1336       } else {
 1337         // 32-bit
 1338         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1339         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1340         if (masm) {
 1341           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1342 #ifndef PRODUCT
 1343         } else {
 1344           st->print("movdl   %s, %s\t# spill",
 1345                      Matcher::regName[dst_first],
 1346                      Matcher::regName[src_first]);
 1347 #endif
 1348         }
 1349       }
 1350       return 0;
 1351     } else if (dst_first_rc == rc_kreg) {
 1352       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1353           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1354         // 64-bit
 1355         if (masm) {
 1356           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
 1357   #ifndef PRODUCT
 1358         } else {
 1359            st->print("kmovq   %s, %s\t# spill",
 1360                        Matcher::regName[dst_first],
 1361                        Matcher::regName[src_first]);
 1362   #endif
 1363         }
 1364       }
 1365       Unimplemented();
 1366       return 0;
 1367     }
 1368   } else if (src_first_rc == rc_float) {
 1369     // xmm ->
 1370     if (dst_first_rc == rc_stack) {
 1371       // xmm -> mem
 1372       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1373           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1374         // 64-bit
 1375         int offset = ra_->reg2offset(dst_first);
 1376         if (masm) {
 1377           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1378 #ifndef PRODUCT
 1379         } else {
 1380           st->print("movsd   [rsp + #%d], %s\t# spill",
 1381                      offset,
 1382                      Matcher::regName[src_first]);
 1383 #endif
 1384         }
 1385       } else {
 1386         // 32-bit
 1387         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1388         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1389         int offset = ra_->reg2offset(dst_first);
 1390         if (masm) {
 1391           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
 1392 #ifndef PRODUCT
 1393         } else {
 1394           st->print("movss   [rsp + #%d], %s\t# spill",
 1395                      offset,
 1396                      Matcher::regName[src_first]);
 1397 #endif
 1398         }
 1399       }
 1400       return 0;
 1401     } else if (dst_first_rc == rc_int) {
 1402       // xmm -> gpr
 1403       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1404           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1405         // 64-bit
 1406         if (masm) {
 1407           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1408 #ifndef PRODUCT
 1409         } else {
 1410           st->print("movdq   %s, %s\t# spill",
 1411                      Matcher::regName[dst_first],
 1412                      Matcher::regName[src_first]);
 1413 #endif
 1414         }
 1415       } else {
 1416         // 32-bit
 1417         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1418         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1419         if (masm) {
 1420           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1421 #ifndef PRODUCT
 1422         } else {
 1423           st->print("movdl   %s, %s\t# spill",
 1424                      Matcher::regName[dst_first],
 1425                      Matcher::regName[src_first]);
 1426 #endif
 1427         }
 1428       }
 1429       return 0;
 1430     } else if (dst_first_rc == rc_float) {
 1431       // xmm -> xmm
 1432       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1433           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1434         // 64-bit
 1435         if (masm) {
 1436           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1437 #ifndef PRODUCT
 1438         } else {
 1439           st->print("%s  %s, %s\t# spill",
 1440                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 1441                      Matcher::regName[dst_first],
 1442                      Matcher::regName[src_first]);
 1443 #endif
 1444         }
 1445       } else {
 1446         // 32-bit
 1447         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 1448         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 1449         if (masm) {
 1450           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
 1451 #ifndef PRODUCT
 1452         } else {
 1453           st->print("%s  %s, %s\t# spill",
 1454                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 1455                      Matcher::regName[dst_first],
 1456                      Matcher::regName[src_first]);
 1457 #endif
 1458         }
 1459       }
 1460       return 0;
 1461     } else if (dst_first_rc == rc_kreg) {
 1462       assert(false, "Illegal spilling");
 1463       return 0;
 1464     }
 1465   } else if (src_first_rc == rc_kreg) {
 1466     if (dst_first_rc == rc_stack) {
 1467       // mem -> kreg
 1468       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1469           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1470         // 64-bit
 1471         int offset = ra_->reg2offset(dst_first);
 1472         if (masm) {
 1473           __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
 1474 #ifndef PRODUCT
 1475         } else {
 1476           st->print("kmovq   [rsp + #%d] , %s\t# spill",
 1477                      offset,
 1478                      Matcher::regName[src_first]);
 1479 #endif
 1480         }
 1481       }
 1482       return 0;
 1483     } else if (dst_first_rc == rc_int) {
 1484       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1485           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1486         // 64-bit
 1487         if (masm) {
 1488           __ kmov(as_Register(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1489 #ifndef PRODUCT
 1490         } else {
 1491          st->print("kmovq   %s, %s\t# spill",
 1492                      Matcher::regName[dst_first],
 1493                      Matcher::regName[src_first]);
 1494 #endif
 1495         }
 1496       }
 1497       Unimplemented();
 1498       return 0;
 1499     } else if (dst_first_rc == rc_kreg) {
 1500       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 1501           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 1502         // 64-bit
 1503         if (masm) {
 1504           __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
 1505 #ifndef PRODUCT
 1506         } else {
 1507          st->print("kmovq   %s, %s\t# spill",
 1508                      Matcher::regName[dst_first],
 1509                      Matcher::regName[src_first]);
 1510 #endif
 1511         }
 1512       }
 1513       return 0;
 1514     } else if (dst_first_rc == rc_float) {
 1515       assert(false, "Illegal spill");
 1516       return 0;
 1517     }
 1518   }
 1519 
 1520   assert(0," foo ");
 1521   Unimplemented();
 1522   return 0;
 1523 }
 1524 
 1525 #ifndef PRODUCT
 1526 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
 1527   implementation(nullptr, ra_, false, st);
 1528 }
 1529 #endif
 1530 
 1531 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
 1532   implementation(masm, ra_, false, nullptr);
 1533 }
 1534 
 1535 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
 1536   return MachNode::size(ra_);
 1537 }
 1538 
 1539 //=============================================================================
 1540 #ifndef PRODUCT
 1541 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1542 {
 1543   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1544   int reg = ra_->get_reg_first(this);
 1545   st->print("leaq    %s, [rsp + #%d]\t# box lock",
 1546             Matcher::regName[reg], offset);
 1547 }
 1548 #endif
 1549 
 1550 void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1551 {
 1552   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1553   int reg = ra_->get_encode(this);
 1554 
 1555   __ lea(as_Register(reg), Address(rsp, offset));
 1556 }
 1557 
 1558 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
 1559 {
 1560   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
 1561   if (ra_->get_encode(this) > 15) {
 1562     return (offset < 0x80) ? 6 : 9; // REX2
 1563   } else {
 1564     return (offset < 0x80) ? 5 : 8; // REX
 1565   }
 1566 }
 1567 
 1568 //=============================================================================
 1569 #ifndef PRODUCT
 1570 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 1571 {
 1572   if (UseCompressedClassPointers) {
 1573     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1574     st->print_cr("\tcmpl    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1575   } else {
 1576     st->print_cr("movq    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
 1577     st->print_cr("\tcmpq    rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
 1578   }
 1579   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
 1580 }
 1581 #endif
 1582 
 1583 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
 1584 {
 1585   __ ic_check(InteriorEntryAlignment);
 1586 }
 1587 
 1588 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 1589 {
 1590   return MachNode::size(ra_); // too many variables; just compute it
 1591                               // the hard way
 1592 }
 1593 
 1594 
 1595 //=============================================================================
 1596 
 1597 bool Matcher::supports_vector_calling_convention(void) {
 1598   return EnableVectorSupport;
 1599 }
 1600 
 1601 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
 1602   assert(EnableVectorSupport, "sanity");
 1603   int lo = XMM0_num;
 1604   int hi = XMM0b_num;
 1605   if (ideal_reg == Op_VecX) hi = XMM0d_num;
 1606   else if (ideal_reg == Op_VecY) hi = XMM0h_num;
 1607   else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
 1608   return OptoRegPair(hi, lo);
 1609 }
 1610 
 1611 // Is this branch offset short enough that a short branch can be used?
 1612 //
 1613 // NOTE: If the platform does not provide any short branch variants, then
 1614 //       this method should return false for offset 0.
 1615 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
 1616   // The passed offset is relative to address of the branch.
 1617   // On 86 a branch displacement is calculated relative to address
 1618   // of a next instruction.
 1619   offset -= br_size;
 1620 
 1621   // the short version of jmpConUCF2 contains multiple branches,
 1622   // making the reach slightly less
 1623   if (rule == jmpConUCF2_rule)
 1624     return (-126 <= offset && offset <= 125);
 1625   return (-128 <= offset && offset <= 127);
 1626 }
 1627 
 1628 // Return whether or not this register is ever used as an argument.
 1629 // This function is used on startup to build the trampoline stubs in
 1630 // generateOptoStub.  Registers not mentioned will be killed by the VM
 1631 // call in the trampoline, and arguments in those registers not be
 1632 // available to the callee.
 1633 bool Matcher::can_be_java_arg(int reg)
 1634 {
 1635   return
 1636     reg ==  RDI_num || reg == RDI_H_num ||
 1637     reg ==  RSI_num || reg == RSI_H_num ||
 1638     reg ==  RDX_num || reg == RDX_H_num ||
 1639     reg ==  RCX_num || reg == RCX_H_num ||
 1640     reg ==   R8_num || reg ==  R8_H_num ||
 1641     reg ==   R9_num || reg ==  R9_H_num ||
 1642     reg ==  R12_num || reg == R12_H_num ||
 1643     reg == XMM0_num || reg == XMM0b_num ||
 1644     reg == XMM1_num || reg == XMM1b_num ||
 1645     reg == XMM2_num || reg == XMM2b_num ||
 1646     reg == XMM3_num || reg == XMM3b_num ||
 1647     reg == XMM4_num || reg == XMM4b_num ||
 1648     reg == XMM5_num || reg == XMM5b_num ||
 1649     reg == XMM6_num || reg == XMM6b_num ||
 1650     reg == XMM7_num || reg == XMM7b_num;
 1651 }
 1652 
 1653 bool Matcher::is_spillable_arg(int reg)
 1654 {
 1655   return can_be_java_arg(reg);
 1656 }
 1657 
 1658 uint Matcher::int_pressure_limit()
 1659 {
 1660   return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
 1661 }
 1662 
 1663 uint Matcher::float_pressure_limit()
 1664 {
 1665   // After experiment around with different values, the following default threshold
 1666   // works best for LCM's register pressure scheduling on x64.
 1667   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
 1668   uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
 1669   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 1670 }
 1671 
 1672 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
 1673   // In 64 bit mode a code which use multiply when
 1674   // devisor is constant is faster than hardware
 1675   // DIV instruction (it uses MulHiL).
 1676   return false;
 1677 }
 1678 
 1679 // Register for DIVI projection of divmodI
 1680 RegMask Matcher::divI_proj_mask() {
 1681   return INT_RAX_REG_mask();
 1682 }
 1683 
 1684 // Register for MODI projection of divmodI
 1685 RegMask Matcher::modI_proj_mask() {
 1686   return INT_RDX_REG_mask();
 1687 }
 1688 
 1689 // Register for DIVL projection of divmodL
 1690 RegMask Matcher::divL_proj_mask() {
 1691   return LONG_RAX_REG_mask();
 1692 }
 1693 
 1694 // Register for MODL projection of divmodL
 1695 RegMask Matcher::modL_proj_mask() {
 1696   return LONG_RDX_REG_mask();
 1697 }
 1698 
 1699 %}
 1700 
 1701 //----------ENCODING BLOCK-----------------------------------------------------
 1702 // This block specifies the encoding classes used by the compiler to
 1703 // output byte streams.  Encoding classes are parameterized macros
 1704 // used by Machine Instruction Nodes in order to generate the bit
 1705 // encoding of the instruction.  Operands specify their base encoding
 1706 // interface with the interface keyword.  There are currently
 1707 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
 1708 // COND_INTER.  REG_INTER causes an operand to generate a function
 1709 // which returns its register number when queried.  CONST_INTER causes
 1710 // an operand to generate a function which returns the value of the
 1711 // constant when queried.  MEMORY_INTER causes an operand to generate
 1712 // four functions which return the Base Register, the Index Register,
 1713 // the Scale Value, and the Offset Value of the operand when queried.
 1714 // COND_INTER causes an operand to generate six functions which return
 1715 // the encoding code (ie - encoding bits for the instruction)
 1716 // associated with each basic boolean condition for a conditional
 1717 // instruction.
 1718 //
 1719 // Instructions specify two basic values for encoding.  Again, a
 1720 // function is available to check if the constant displacement is an
 1721 // oop. They use the ins_encode keyword to specify their encoding
 1722 // classes (which must be a sequence of enc_class names, and their
 1723 // parameters, specified in the encoding block), and they use the
 1724 // opcode keyword to specify, in order, their primary, secondary, and
 1725 // tertiary opcode.  Only the opcode sections which a particular
 1726 // instruction needs for encoding need to be specified.
 1727 encode %{
 1728   enc_class cdql_enc(no_rax_rdx_RegI div)
 1729   %{
 1730     // Full implementation of Java idiv and irem; checks for
 1731     // special case as described in JVM spec., p.243 & p.271.
 1732     //
 1733     //         normal case                           special case
 1734     //
 1735     // input : rax: dividend                         min_int
 1736     //         reg: divisor                          -1
 1737     //
 1738     // output: rax: quotient  (= rax idiv reg)       min_int
 1739     //         rdx: remainder (= rax irem reg)       0
 1740     //
 1741     //  Code sequnce:
 1742     //
 1743     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
 1744     //    5:   75 07/08                jne    e <normal>
 1745     //    7:   33 d2                   xor    %edx,%edx
 1746     //  [div >= 8 -> offset + 1]
 1747     //  [REX_B]
 1748     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
 1749     //    c:   74 03/04                je     11 <done>
 1750     // 000000000000000e <normal>:
 1751     //    e:   99                      cltd
 1752     //  [div >= 8 -> offset + 1]
 1753     //  [REX_B]
 1754     //    f:   f7 f9                   idiv   $div
 1755     // 0000000000000011 <done>:
 1756     Label normal;
 1757     Label done;
 1758 
 1759     // cmp    $0x80000000,%eax
 1760     __ cmpl(as_Register(RAX_enc), 0x80000000);
 1761 
 1762     // jne    e <normal>
 1763     __ jccb(Assembler::notEqual, normal);
 1764 
 1765     // xor    %edx,%edx
 1766     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1767 
 1768     // cmp    $0xffffffffffffffff,%ecx
 1769     __ cmpl($div$$Register, -1);
 1770 
 1771     // je     11 <done>
 1772     __ jccb(Assembler::equal, done);
 1773 
 1774     // <normal>
 1775     // cltd
 1776     __ bind(normal);
 1777     __ cdql();
 1778 
 1779     // idivl
 1780     // <done>
 1781     __ idivl($div$$Register);
 1782     __ bind(done);
 1783   %}
 1784 
 1785   enc_class cdqq_enc(no_rax_rdx_RegL div)
 1786   %{
 1787     // Full implementation of Java ldiv and lrem; checks for
 1788     // special case as described in JVM spec., p.243 & p.271.
 1789     //
 1790     //         normal case                           special case
 1791     //
 1792     // input : rax: dividend                         min_long
 1793     //         reg: divisor                          -1
 1794     //
 1795     // output: rax: quotient  (= rax idiv reg)       min_long
 1796     //         rdx: remainder (= rax irem reg)       0
 1797     //
 1798     //  Code sequnce:
 1799     //
 1800     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
 1801     //    7:   00 00 80
 1802     //    a:   48 39 d0                cmp    %rdx,%rax
 1803     //    d:   75 08                   jne    17 <normal>
 1804     //    f:   33 d2                   xor    %edx,%edx
 1805     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
 1806     //   15:   74 05                   je     1c <done>
 1807     // 0000000000000017 <normal>:
 1808     //   17:   48 99                   cqto
 1809     //   19:   48 f7 f9                idiv   $div
 1810     // 000000000000001c <done>:
 1811     Label normal;
 1812     Label done;
 1813 
 1814     // mov    $0x8000000000000000,%rdx
 1815     __ mov64(as_Register(RDX_enc), 0x8000000000000000);
 1816 
 1817     // cmp    %rdx,%rax
 1818     __ cmpq(as_Register(RAX_enc), as_Register(RDX_enc));
 1819 
 1820     // jne    17 <normal>
 1821     __ jccb(Assembler::notEqual, normal);
 1822 
 1823     // xor    %edx,%edx
 1824     __ xorl(as_Register(RDX_enc), as_Register(RDX_enc));
 1825 
 1826     // cmp    $0xffffffffffffffff,$div
 1827     __ cmpq($div$$Register, -1);
 1828 
 1829     // je     1e <done>
 1830     __ jccb(Assembler::equal, done);
 1831 
 1832     // <normal>
 1833     // cqto
 1834     __ bind(normal);
 1835     __ cdqq();
 1836 
 1837     // idivq (note: must be emitted by the user of this rule)
 1838     // <done>
 1839     __ idivq($div$$Register);
 1840     __ bind(done);
 1841   %}
 1842 
 1843   enc_class clear_avx %{
 1844     DEBUG_ONLY(int off0 = __ offset());
 1845     if (generate_vzeroupper(Compile::current())) {
 1846       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
 1847       // Clear upper bits of YMM registers when current compiled code uses
 1848       // wide vectors to avoid AVX <-> SSE transition penalty during call.
 1849       __ vzeroupper();
 1850     }
 1851     DEBUG_ONLY(int off1 = __ offset());
 1852     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
 1853   %}
 1854 
 1855   enc_class Java_To_Runtime(method meth) %{
 1856     __ lea(r10, RuntimeAddress((address)$meth$$method));
 1857     __ call(r10);
 1858     __ post_call_nop();
 1859   %}
 1860 
 1861   enc_class Java_Static_Call(method meth)
 1862   %{
 1863     // JAVA STATIC CALL
 1864     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
 1865     // determine who we intended to call.
 1866     if (!_method) {
 1867       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, $meth$$method)));
 1868     } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
 1869       // The NOP here is purely to ensure that eliding a call to
 1870       // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
 1871       __ addr_nop_5();
 1872       __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
 1873     } else {
 1874       int method_index = resolved_method_index(masm);
 1875       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
 1876                                                   : static_call_Relocation::spec(method_index);
 1877       address mark = __ pc();
 1878       int call_offset = __ offset();
 1879       __ call(AddressLiteral(CAST_FROM_FN_PTR(address, $meth$$method), rspec));
 1880       if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
 1881         // Calls of the same statically bound method can share
 1882         // a stub to the interpreter.
 1883         __ code()->shared_stub_to_interp_for(_method, call_offset);
 1884       } else {
 1885         // Emit stubs for static call.
 1886         address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
 1887         __ clear_inst_mark();
 1888         if (stub == nullptr) {
 1889           ciEnv::current()->record_failure("CodeCache is full");
 1890           return;
 1891         }
 1892       }
 1893     }
 1894     __ post_call_nop();
 1895   %}
 1896 
 1897   enc_class Java_Dynamic_Call(method meth) %{
 1898     __ ic_call((address)$meth$$method, resolved_method_index(masm));
 1899     __ post_call_nop();
 1900   %}
 1901 
 1902 %}
 1903 
 1904 
 1905 
 1906 //----------FRAME--------------------------------------------------------------
 1907 // Definition of frame structure and management information.
 1908 //
 1909 //  S T A C K   L A Y O U T    Allocators stack-slot number
 1910 //                             |   (to get allocators register number
 1911 //  G  Owned by    |        |  v    add OptoReg::stack0())
 1912 //  r   CALLER     |        |
 1913 //  o     |        +--------+      pad to even-align allocators stack-slot
 1914 //  w     V        |  pad0  |        numbers; owned by CALLER
 1915 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
 1916 //  h     ^        |   in   |  5
 1917 //        |        |  args  |  4   Holes in incoming args owned by SELF
 1918 //  |     |        |        |  3
 1919 //  |     |        +--------+
 1920 //  V     |        | old out|      Empty on Intel, window on Sparc
 1921 //        |    old |preserve|      Must be even aligned.
 1922 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
 1923 //        |        |   in   |  3   area for Intel ret address
 1924 //     Owned by    |preserve|      Empty on Sparc.
 1925 //       SELF      +--------+
 1926 //        |        |  pad2  |  2   pad to align old SP
 1927 //        |        +--------+  1
 1928 //        |        | locks  |  0
 1929 //        |        +--------+----> OptoReg::stack0(), even aligned
 1930 //        |        |  pad1  | 11   pad to align new SP
 1931 //        |        +--------+
 1932 //        |        |        | 10
 1933 //        |        | spills |  9   spills
 1934 //        V        |        |  8   (pad0 slot for callee)
 1935 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
 1936 //        ^        |  out   |  7
 1937 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
 1938 //     Owned by    +--------+
 1939 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
 1940 //        |    new |preserve|      Must be even-aligned.
 1941 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
 1942 //        |        |        |
 1943 //
 1944 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
 1945 //         known from SELF's arguments and the Java calling convention.
 1946 //         Region 6-7 is determined per call site.
 1947 // Note 2: If the calling convention leaves holes in the incoming argument
 1948 //         area, those holes are owned by SELF.  Holes in the outgoing area
 1949 //         are owned by the CALLEE.  Holes should not be necessary in the
 1950 //         incoming area, as the Java calling convention is completely under
 1951 //         the control of the AD file.  Doubles can be sorted and packed to
 1952 //         avoid holes.  Holes in the outgoing arguments may be necessary for
 1953 //         varargs C calling conventions.
 1954 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
 1955 //         even aligned with pad0 as needed.
 1956 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
 1957 //         region 6-11 is even aligned; it may be padded out more so that
 1958 //         the region from SP to FP meets the minimum stack alignment.
 1959 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
 1960 //         alignment.  Region 11, pad1, may be dynamically extended so that
 1961 //         SP meets the minimum alignment.
 1962 
 1963 frame
 1964 %{
 1965   // These three registers define part of the calling convention
 1966   // between compiled code and the interpreter.
 1967   inline_cache_reg(RAX);                // Inline Cache Register
 1968 
 1969   // Optional: name the operand used by cisc-spilling to access
 1970   // [stack_pointer + offset]
 1971   cisc_spilling_operand_name(indOffset32);
 1972 
 1973   // Number of stack slots consumed by locking an object
 1974   sync_stack_slots(2);
 1975 
 1976   // Compiled code's Frame Pointer
 1977   frame_pointer(RSP);
 1978 
 1979   // Interpreter stores its frame pointer in a register which is
 1980   // stored to the stack by I2CAdaptors.
 1981   // I2CAdaptors convert from interpreted java to compiled java.
 1982   interpreter_frame_pointer(RBP);
 1983 
 1984   // Stack alignment requirement
 1985   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
 1986 
 1987   // Number of outgoing stack slots killed above the out_preserve_stack_slots
 1988   // for calls to C.  Supports the var-args backing area for register parms.
 1989   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
 1990 
 1991   // The after-PROLOG location of the return address.  Location of
 1992   // return address specifies a type (REG or STACK) and a number
 1993   // representing the register number (i.e. - use a register name) or
 1994   // stack slot.
 1995   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 1996   // Otherwise, it is above the locks and verification slot and alignment word
 1997   return_addr(STACK - 2 +
 1998               align_up((Compile::current()->in_preserve_stack_slots() +
 1999                         Compile::current()->fixed_slots()),
 2000                        stack_alignment_in_slots()));
 2001 
 2002   // Location of compiled Java return values.  Same as C for now.
 2003   return_value
 2004   %{
 2005     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
 2006            "only return normal values");
 2007 
 2008     static const int lo[Op_RegL + 1] = {
 2009       0,
 2010       0,
 2011       RAX_num,  // Op_RegN
 2012       RAX_num,  // Op_RegI
 2013       RAX_num,  // Op_RegP
 2014       XMM0_num, // Op_RegF
 2015       XMM0_num, // Op_RegD
 2016       RAX_num   // Op_RegL
 2017     };
 2018     static const int hi[Op_RegL + 1] = {
 2019       0,
 2020       0,
 2021       OptoReg::Bad, // Op_RegN
 2022       OptoReg::Bad, // Op_RegI
 2023       RAX_H_num,    // Op_RegP
 2024       OptoReg::Bad, // Op_RegF
 2025       XMM0b_num,    // Op_RegD
 2026       RAX_H_num     // Op_RegL
 2027     };
 2028     // Excluded flags and vector registers.
 2029     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 8, "missing type");
 2030     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
 2031   %}
 2032 %}
 2033 
 2034 //----------ATTRIBUTES---------------------------------------------------------
 2035 //----------Operand Attributes-------------------------------------------------
 2036 op_attrib op_cost(0);        // Required cost attribute
 2037 
 2038 //----------Instruction Attributes---------------------------------------------
 2039 ins_attrib ins_cost(100);       // Required cost attribute
 2040 ins_attrib ins_size(8);         // Required size attribute (in bits)
 2041 ins_attrib ins_short_branch(0); // Required flag: is this instruction
 2042                                 // a non-matching short branch variant
 2043                                 // of some long branch?
 2044 ins_attrib ins_alignment(1);    // Required alignment attribute (must
 2045                                 // be a power of 2) specifies the
 2046                                 // alignment that some part of the
 2047                                 // instruction (not necessarily the
 2048                                 // start) requires.  If > 1, a
 2049                                 // compute_padding() function must be
 2050                                 // provided for the instruction
 2051 
 2052 // Whether this node is expanded during code emission into a sequence of
 2053 // instructions and the first instruction can perform an implicit null check.
 2054 ins_attrib ins_is_late_expanded_null_check_candidate(false);
 2055 
 2056 //----------OPERANDS-----------------------------------------------------------
 2057 // Operand definitions must precede instruction definitions for correct parsing
 2058 // in the ADLC because operands constitute user defined types which are used in
 2059 // instruction definitions.
 2060 
 2061 //----------Simple Operands----------------------------------------------------
 2062 // Immediate Operands
 2063 // Integer Immediate
 2064 operand immI()
 2065 %{
 2066   match(ConI);
 2067 
 2068   op_cost(10);
 2069   format %{ %}
 2070   interface(CONST_INTER);
 2071 %}
 2072 
 2073 // Constant for test vs zero
 2074 operand immI_0()
 2075 %{
 2076   predicate(n->get_int() == 0);
 2077   match(ConI);
 2078 
 2079   op_cost(0);
 2080   format %{ %}
 2081   interface(CONST_INTER);
 2082 %}
 2083 
 2084 // Constant for increment
 2085 operand immI_1()
 2086 %{
 2087   predicate(n->get_int() == 1);
 2088   match(ConI);
 2089 
 2090   op_cost(0);
 2091   format %{ %}
 2092   interface(CONST_INTER);
 2093 %}
 2094 
 2095 // Constant for decrement
 2096 operand immI_M1()
 2097 %{
 2098   predicate(n->get_int() == -1);
 2099   match(ConI);
 2100 
 2101   op_cost(0);
 2102   format %{ %}
 2103   interface(CONST_INTER);
 2104 %}
 2105 
 2106 operand immI_2()
 2107 %{
 2108   predicate(n->get_int() == 2);
 2109   match(ConI);
 2110 
 2111   op_cost(0);
 2112   format %{ %}
 2113   interface(CONST_INTER);
 2114 %}
 2115 
 2116 operand immI_4()
 2117 %{
 2118   predicate(n->get_int() == 4);
 2119   match(ConI);
 2120 
 2121   op_cost(0);
 2122   format %{ %}
 2123   interface(CONST_INTER);
 2124 %}
 2125 
 2126 operand immI_8()
 2127 %{
 2128   predicate(n->get_int() == 8);
 2129   match(ConI);
 2130 
 2131   op_cost(0);
 2132   format %{ %}
 2133   interface(CONST_INTER);
 2134 %}
 2135 
 2136 // Valid scale values for addressing modes
 2137 operand immI2()
 2138 %{
 2139   predicate(0 <= n->get_int() && (n->get_int() <= 3));
 2140   match(ConI);
 2141 
 2142   format %{ %}
 2143   interface(CONST_INTER);
 2144 %}
 2145 
 2146 operand immU7()
 2147 %{
 2148   predicate((0 <= n->get_int()) && (n->get_int() <= 0x7F));
 2149   match(ConI);
 2150 
 2151   op_cost(5);
 2152   format %{ %}
 2153   interface(CONST_INTER);
 2154 %}
 2155 
 2156 operand immI8()
 2157 %{
 2158   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
 2159   match(ConI);
 2160 
 2161   op_cost(5);
 2162   format %{ %}
 2163   interface(CONST_INTER);
 2164 %}
 2165 
 2166 operand immU8()
 2167 %{
 2168   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
 2169   match(ConI);
 2170 
 2171   op_cost(5);
 2172   format %{ %}
 2173   interface(CONST_INTER);
 2174 %}
 2175 
 2176 operand immI16()
 2177 %{
 2178   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
 2179   match(ConI);
 2180 
 2181   op_cost(10);
 2182   format %{ %}
 2183   interface(CONST_INTER);
 2184 %}
 2185 
 2186 // Int Immediate non-negative
 2187 operand immU31()
 2188 %{
 2189   predicate(n->get_int() >= 0);
 2190   match(ConI);
 2191 
 2192   op_cost(0);
 2193   format %{ %}
 2194   interface(CONST_INTER);
 2195 %}
 2196 
 2197 // Pointer Immediate
 2198 operand immP()
 2199 %{
 2200   match(ConP);
 2201 
 2202   op_cost(10);
 2203   format %{ %}
 2204   interface(CONST_INTER);
 2205 %}
 2206 
 2207 // Null Pointer Immediate
 2208 operand immP0()
 2209 %{
 2210   predicate(n->get_ptr() == 0);
 2211   match(ConP);
 2212 
 2213   op_cost(5);
 2214   format %{ %}
 2215   interface(CONST_INTER);
 2216 %}
 2217 
 2218 // Pointer Immediate
 2219 operand immN() %{
 2220   match(ConN);
 2221 
 2222   op_cost(10);
 2223   format %{ %}
 2224   interface(CONST_INTER);
 2225 %}
 2226 
 2227 operand immNKlass() %{
 2228   match(ConNKlass);
 2229 
 2230   op_cost(10);
 2231   format %{ %}
 2232   interface(CONST_INTER);
 2233 %}
 2234 
 2235 // Null Pointer Immediate
 2236 operand immN0() %{
 2237   predicate(n->get_narrowcon() == 0);
 2238   match(ConN);
 2239 
 2240   op_cost(5);
 2241   format %{ %}
 2242   interface(CONST_INTER);
 2243 %}
 2244 
 2245 operand immP31()
 2246 %{
 2247   predicate(n->as_Type()->type()->reloc() == relocInfo::none
 2248             && (n->get_ptr() >> 31) == 0);
 2249   match(ConP);
 2250 
 2251   op_cost(5);
 2252   format %{ %}
 2253   interface(CONST_INTER);
 2254 %}
 2255 
 2256 
 2257 // Long Immediate
 2258 operand immL()
 2259 %{
 2260   match(ConL);
 2261 
 2262   op_cost(20);
 2263   format %{ %}
 2264   interface(CONST_INTER);
 2265 %}
 2266 
 2267 // Long Immediate 8-bit
 2268 operand immL8()
 2269 %{
 2270   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
 2271   match(ConL);
 2272 
 2273   op_cost(5);
 2274   format %{ %}
 2275   interface(CONST_INTER);
 2276 %}
 2277 
 2278 // Long Immediate 32-bit unsigned
 2279 operand immUL32()
 2280 %{
 2281   predicate(n->get_long() == (unsigned int) (n->get_long()));
 2282   match(ConL);
 2283 
 2284   op_cost(10);
 2285   format %{ %}
 2286   interface(CONST_INTER);
 2287 %}
 2288 
 2289 // Long Immediate 32-bit signed
 2290 operand immL32()
 2291 %{
 2292   predicate(n->get_long() == (int) (n->get_long()));
 2293   match(ConL);
 2294 
 2295   op_cost(15);
 2296   format %{ %}
 2297   interface(CONST_INTER);
 2298 %}
 2299 
 2300 operand immL_Pow2()
 2301 %{
 2302   predicate(is_power_of_2((julong)n->get_long()));
 2303   match(ConL);
 2304 
 2305   op_cost(15);
 2306   format %{ %}
 2307   interface(CONST_INTER);
 2308 %}
 2309 
 2310 operand immL_NotPow2()
 2311 %{
 2312   predicate(is_power_of_2((julong)~n->get_long()));
 2313   match(ConL);
 2314 
 2315   op_cost(15);
 2316   format %{ %}
 2317   interface(CONST_INTER);
 2318 %}
 2319 
 2320 // Long Immediate zero
 2321 operand immL0()
 2322 %{
 2323   predicate(n->get_long() == 0L);
 2324   match(ConL);
 2325 
 2326   op_cost(10);
 2327   format %{ %}
 2328   interface(CONST_INTER);
 2329 %}
 2330 
 2331 // Constant for increment
 2332 operand immL1()
 2333 %{
 2334   predicate(n->get_long() == 1);
 2335   match(ConL);
 2336 
 2337   format %{ %}
 2338   interface(CONST_INTER);
 2339 %}
 2340 
 2341 // Constant for decrement
 2342 operand immL_M1()
 2343 %{
 2344   predicate(n->get_long() == -1);
 2345   match(ConL);
 2346 
 2347   format %{ %}
 2348   interface(CONST_INTER);
 2349 %}
 2350 
 2351 // Long Immediate: low 32-bit mask
 2352 operand immL_32bits()
 2353 %{
 2354   predicate(n->get_long() == 0xFFFFFFFFL);
 2355   match(ConL);
 2356   op_cost(20);
 2357 
 2358   format %{ %}
 2359   interface(CONST_INTER);
 2360 %}
 2361 
 2362 // Int Immediate: 2^n-1, positive
 2363 operand immI_Pow2M1()
 2364 %{
 2365   predicate((n->get_int() > 0)
 2366             && is_power_of_2((juint)n->get_int() + 1));
 2367   match(ConI);
 2368 
 2369   op_cost(20);
 2370   format %{ %}
 2371   interface(CONST_INTER);
 2372 %}
 2373 
 2374 // Float Immediate zero
 2375 operand immF0()
 2376 %{
 2377   predicate(jint_cast(n->getf()) == 0);
 2378   match(ConF);
 2379 
 2380   op_cost(5);
 2381   format %{ %}
 2382   interface(CONST_INTER);
 2383 %}
 2384 
 2385 // Float Immediate
 2386 operand immF()
 2387 %{
 2388   match(ConF);
 2389 
 2390   op_cost(15);
 2391   format %{ %}
 2392   interface(CONST_INTER);
 2393 %}
 2394 
 2395 // Half Float Immediate
 2396 operand immH()
 2397 %{
 2398   match(ConH);
 2399 
 2400   op_cost(15);
 2401   format %{ %}
 2402   interface(CONST_INTER);
 2403 %}
 2404 
 2405 // Double Immediate zero
 2406 operand immD0()
 2407 %{
 2408   predicate(jlong_cast(n->getd()) == 0);
 2409   match(ConD);
 2410 
 2411   op_cost(5);
 2412   format %{ %}
 2413   interface(CONST_INTER);
 2414 %}
 2415 
 2416 // Double Immediate
 2417 operand immD()
 2418 %{
 2419   match(ConD);
 2420 
 2421   op_cost(15);
 2422   format %{ %}
 2423   interface(CONST_INTER);
 2424 %}
 2425 
 2426 // Immediates for special shifts (sign extend)
 2427 
 2428 // Constants for increment
 2429 operand immI_16()
 2430 %{
 2431   predicate(n->get_int() == 16);
 2432   match(ConI);
 2433 
 2434   format %{ %}
 2435   interface(CONST_INTER);
 2436 %}
 2437 
 2438 operand immI_24()
 2439 %{
 2440   predicate(n->get_int() == 24);
 2441   match(ConI);
 2442 
 2443   format %{ %}
 2444   interface(CONST_INTER);
 2445 %}
 2446 
 2447 // Constant for byte-wide masking
 2448 operand immI_255()
 2449 %{
 2450   predicate(n->get_int() == 255);
 2451   match(ConI);
 2452 
 2453   format %{ %}
 2454   interface(CONST_INTER);
 2455 %}
 2456 
 2457 // Constant for short-wide masking
 2458 operand immI_65535()
 2459 %{
 2460   predicate(n->get_int() == 65535);
 2461   match(ConI);
 2462 
 2463   format %{ %}
 2464   interface(CONST_INTER);
 2465 %}
 2466 
 2467 // Constant for byte-wide masking
 2468 operand immL_255()
 2469 %{
 2470   predicate(n->get_long() == 255);
 2471   match(ConL);
 2472 
 2473   format %{ %}
 2474   interface(CONST_INTER);
 2475 %}
 2476 
 2477 // Constant for short-wide masking
 2478 operand immL_65535()
 2479 %{
 2480   predicate(n->get_long() == 65535);
 2481   match(ConL);
 2482 
 2483   format %{ %}
 2484   interface(CONST_INTER);
 2485 %}
 2486 
 2487 // AOT Runtime Constants Address
 2488 operand immAOTRuntimeConstantsAddress()
 2489 %{
 2490   // Check if the address is in the range of AOT Runtime Constants
 2491   predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
 2492   match(ConP);
 2493 
 2494   op_cost(0);
 2495   format %{ %}
 2496   interface(CONST_INTER);
 2497 %}
 2498 
 2499 operand kReg()
 2500 %{
 2501   constraint(ALLOC_IN_RC(vectmask_reg));
 2502   match(RegVectMask);
 2503   format %{%}
 2504   interface(REG_INTER);
 2505 %}
 2506 
 2507 // Register Operands
 2508 // Integer Register
 2509 operand rRegI()
 2510 %{
 2511   constraint(ALLOC_IN_RC(int_reg));
 2512   match(RegI);
 2513 
 2514   match(rax_RegI);
 2515   match(rbx_RegI);
 2516   match(rcx_RegI);
 2517   match(rdx_RegI);
 2518   match(rdi_RegI);
 2519 
 2520   format %{ %}
 2521   interface(REG_INTER);
 2522 %}
 2523 
 2524 // Special Registers
 2525 operand rax_RegI()
 2526 %{
 2527   constraint(ALLOC_IN_RC(int_rax_reg));
 2528   match(RegI);
 2529   match(rRegI);
 2530 
 2531   format %{ "RAX" %}
 2532   interface(REG_INTER);
 2533 %}
 2534 
 2535 // Special Registers
 2536 operand rbx_RegI()
 2537 %{
 2538   constraint(ALLOC_IN_RC(int_rbx_reg));
 2539   match(RegI);
 2540   match(rRegI);
 2541 
 2542   format %{ "RBX" %}
 2543   interface(REG_INTER);
 2544 %}
 2545 
 2546 operand rcx_RegI()
 2547 %{
 2548   constraint(ALLOC_IN_RC(int_rcx_reg));
 2549   match(RegI);
 2550   match(rRegI);
 2551 
 2552   format %{ "RCX" %}
 2553   interface(REG_INTER);
 2554 %}
 2555 
 2556 operand rdx_RegI()
 2557 %{
 2558   constraint(ALLOC_IN_RC(int_rdx_reg));
 2559   match(RegI);
 2560   match(rRegI);
 2561 
 2562   format %{ "RDX" %}
 2563   interface(REG_INTER);
 2564 %}
 2565 
 2566 operand rdi_RegI()
 2567 %{
 2568   constraint(ALLOC_IN_RC(int_rdi_reg));
 2569   match(RegI);
 2570   match(rRegI);
 2571 
 2572   format %{ "RDI" %}
 2573   interface(REG_INTER);
 2574 %}
 2575 
 2576 operand no_rax_rdx_RegI()
 2577 %{
 2578   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
 2579   match(RegI);
 2580   match(rbx_RegI);
 2581   match(rcx_RegI);
 2582   match(rdi_RegI);
 2583 
 2584   format %{ %}
 2585   interface(REG_INTER);
 2586 %}
 2587 
 2588 operand no_rbp_r13_RegI()
 2589 %{
 2590   constraint(ALLOC_IN_RC(int_no_rbp_r13_reg));
 2591   match(RegI);
 2592   match(rRegI);
 2593   match(rax_RegI);
 2594   match(rbx_RegI);
 2595   match(rcx_RegI);
 2596   match(rdx_RegI);
 2597   match(rdi_RegI);
 2598 
 2599   format %{ %}
 2600   interface(REG_INTER);
 2601 %}
 2602 
 2603 // Pointer Register
 2604 operand any_RegP()
 2605 %{
 2606   constraint(ALLOC_IN_RC(any_reg));
 2607   match(RegP);
 2608   match(rax_RegP);
 2609   match(rbx_RegP);
 2610   match(rdi_RegP);
 2611   match(rsi_RegP);
 2612   match(rbp_RegP);
 2613   match(r15_RegP);
 2614   match(rRegP);
 2615 
 2616   format %{ %}
 2617   interface(REG_INTER);
 2618 %}
 2619 
 2620 operand rRegP()
 2621 %{
 2622   constraint(ALLOC_IN_RC(ptr_reg));
 2623   match(RegP);
 2624   match(rax_RegP);
 2625   match(rbx_RegP);
 2626   match(rdi_RegP);
 2627   match(rsi_RegP);
 2628   match(rbp_RegP);  // See Q&A below about
 2629   match(r15_RegP);  // r15_RegP and rbp_RegP.
 2630 
 2631   format %{ %}
 2632   interface(REG_INTER);
 2633 %}
 2634 
 2635 operand rRegN() %{
 2636   constraint(ALLOC_IN_RC(int_reg));
 2637   match(RegN);
 2638 
 2639   format %{ %}
 2640   interface(REG_INTER);
 2641 %}
 2642 
 2643 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
 2644 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
 2645 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
 2646 // The output of an instruction is controlled by the allocator, which respects
 2647 // register class masks, not match rules.  Unless an instruction mentions
 2648 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
 2649 // by the allocator as an input.
 2650 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
 2651 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
 2652 // result, RBP is not included in the output of the instruction either.
 2653 
 2654 // This operand is not allowed to use RBP even if
 2655 // RBP is not used to hold the frame pointer.
 2656 operand no_rbp_RegP()
 2657 %{
 2658   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
 2659   match(RegP);
 2660   match(rbx_RegP);
 2661   match(rsi_RegP);
 2662   match(rdi_RegP);
 2663 
 2664   format %{ %}
 2665   interface(REG_INTER);
 2666 %}
 2667 
 2668 // Special Registers
 2669 // Return a pointer value
 2670 operand rax_RegP()
 2671 %{
 2672   constraint(ALLOC_IN_RC(ptr_rax_reg));
 2673   match(RegP);
 2674   match(rRegP);
 2675 
 2676   format %{ %}
 2677   interface(REG_INTER);
 2678 %}
 2679 
 2680 // Special Registers
 2681 // Return a compressed pointer value
 2682 operand rax_RegN()
 2683 %{
 2684   constraint(ALLOC_IN_RC(int_rax_reg));
 2685   match(RegN);
 2686   match(rRegN);
 2687 
 2688   format %{ %}
 2689   interface(REG_INTER);
 2690 %}
 2691 
 2692 // Used in AtomicAdd
 2693 operand rbx_RegP()
 2694 %{
 2695   constraint(ALLOC_IN_RC(ptr_rbx_reg));
 2696   match(RegP);
 2697   match(rRegP);
 2698 
 2699   format %{ %}
 2700   interface(REG_INTER);
 2701 %}
 2702 
 2703 operand rsi_RegP()
 2704 %{
 2705   constraint(ALLOC_IN_RC(ptr_rsi_reg));
 2706   match(RegP);
 2707   match(rRegP);
 2708 
 2709   format %{ %}
 2710   interface(REG_INTER);
 2711 %}
 2712 
 2713 operand rbp_RegP()
 2714 %{
 2715   constraint(ALLOC_IN_RC(ptr_rbp_reg));
 2716   match(RegP);
 2717   match(rRegP);
 2718 
 2719   format %{ %}
 2720   interface(REG_INTER);
 2721 %}
 2722 
 2723 // Used in rep stosq
 2724 operand rdi_RegP()
 2725 %{
 2726   constraint(ALLOC_IN_RC(ptr_rdi_reg));
 2727   match(RegP);
 2728   match(rRegP);
 2729 
 2730   format %{ %}
 2731   interface(REG_INTER);
 2732 %}
 2733 
 2734 operand r15_RegP()
 2735 %{
 2736   constraint(ALLOC_IN_RC(ptr_r15_reg));
 2737   match(RegP);
 2738   match(rRegP);
 2739 
 2740   format %{ %}
 2741   interface(REG_INTER);
 2742 %}
 2743 
 2744 operand rRegL()
 2745 %{
 2746   constraint(ALLOC_IN_RC(long_reg));
 2747   match(RegL);
 2748   match(rax_RegL);
 2749   match(rdx_RegL);
 2750 
 2751   format %{ %}
 2752   interface(REG_INTER);
 2753 %}
 2754 
 2755 // Special Registers
 2756 operand no_rax_rdx_RegL()
 2757 %{
 2758   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
 2759   match(RegL);
 2760   match(rRegL);
 2761 
 2762   format %{ %}
 2763   interface(REG_INTER);
 2764 %}
 2765 
 2766 operand rax_RegL()
 2767 %{
 2768   constraint(ALLOC_IN_RC(long_rax_reg));
 2769   match(RegL);
 2770   match(rRegL);
 2771 
 2772   format %{ "RAX" %}
 2773   interface(REG_INTER);
 2774 %}
 2775 
 2776 operand rcx_RegL()
 2777 %{
 2778   constraint(ALLOC_IN_RC(long_rcx_reg));
 2779   match(RegL);
 2780   match(rRegL);
 2781 
 2782   format %{ %}
 2783   interface(REG_INTER);
 2784 %}
 2785 
 2786 operand rdx_RegL()
 2787 %{
 2788   constraint(ALLOC_IN_RC(long_rdx_reg));
 2789   match(RegL);
 2790   match(rRegL);
 2791 
 2792   format %{ %}
 2793   interface(REG_INTER);
 2794 %}
 2795 
 2796 operand r11_RegL()
 2797 %{
 2798   constraint(ALLOC_IN_RC(long_r11_reg));
 2799   match(RegL);
 2800   match(rRegL);
 2801 
 2802   format %{ %}
 2803   interface(REG_INTER);
 2804 %}
 2805 
 2806 operand no_rbp_r13_RegL()
 2807 %{
 2808   constraint(ALLOC_IN_RC(long_no_rbp_r13_reg));
 2809   match(RegL);
 2810   match(rRegL);
 2811   match(rax_RegL);
 2812   match(rcx_RegL);
 2813   match(rdx_RegL);
 2814 
 2815   format %{ %}
 2816   interface(REG_INTER);
 2817 %}
 2818 
 2819 // Flags register, used as output of compare instructions
 2820 operand rFlagsReg()
 2821 %{
 2822   constraint(ALLOC_IN_RC(int_flags));
 2823   match(RegFlags);
 2824 
 2825   format %{ "RFLAGS" %}
 2826   interface(REG_INTER);
 2827 %}
 2828 
 2829 // Flags register, used as output of FLOATING POINT compare instructions
 2830 operand rFlagsRegU()
 2831 %{
 2832   constraint(ALLOC_IN_RC(int_flags));
 2833   match(RegFlags);
 2834 
 2835   format %{ "RFLAGS_U" %}
 2836   interface(REG_INTER);
 2837 %}
 2838 
 2839 operand rFlagsRegUCF() %{
 2840   constraint(ALLOC_IN_RC(int_flags));
 2841   match(RegFlags);
 2842   predicate(false);
 2843 
 2844   format %{ "RFLAGS_U_CF" %}
 2845   interface(REG_INTER);
 2846 %}
 2847 
 2848 // Float register operands
 2849 operand regF() %{
 2850    constraint(ALLOC_IN_RC(float_reg));
 2851    match(RegF);
 2852 
 2853    format %{ %}
 2854    interface(REG_INTER);
 2855 %}
 2856 
 2857 // Float register operands
 2858 operand legRegF() %{
 2859    constraint(ALLOC_IN_RC(float_reg_legacy));
 2860    match(RegF);
 2861 
 2862    format %{ %}
 2863    interface(REG_INTER);
 2864 %}
 2865 
 2866 // Float register operands
 2867 operand vlRegF() %{
 2868    constraint(ALLOC_IN_RC(float_reg_vl));
 2869    match(RegF);
 2870 
 2871    format %{ %}
 2872    interface(REG_INTER);
 2873 %}
 2874 
 2875 // Double register operands
 2876 operand regD() %{
 2877    constraint(ALLOC_IN_RC(double_reg));
 2878    match(RegD);
 2879 
 2880    format %{ %}
 2881    interface(REG_INTER);
 2882 %}
 2883 
 2884 // Double register operands
 2885 operand legRegD() %{
 2886    constraint(ALLOC_IN_RC(double_reg_legacy));
 2887    match(RegD);
 2888 
 2889    format %{ %}
 2890    interface(REG_INTER);
 2891 %}
 2892 
 2893 // Double register operands
 2894 operand vlRegD() %{
 2895    constraint(ALLOC_IN_RC(double_reg_vl));
 2896    match(RegD);
 2897 
 2898    format %{ %}
 2899    interface(REG_INTER);
 2900 %}
 2901 
 2902 //----------Memory Operands----------------------------------------------------
 2903 // Direct Memory Operand
 2904 // operand direct(immP addr)
 2905 // %{
 2906 //   match(addr);
 2907 
 2908 //   format %{ "[$addr]" %}
 2909 //   interface(MEMORY_INTER) %{
 2910 //     base(0xFFFFFFFF);
 2911 //     index(0x4);
 2912 //     scale(0x0);
 2913 //     disp($addr);
 2914 //   %}
 2915 // %}
 2916 
 2917 // Indirect Memory Operand
 2918 operand indirect(any_RegP reg)
 2919 %{
 2920   constraint(ALLOC_IN_RC(ptr_reg));
 2921   match(reg);
 2922 
 2923   format %{ "[$reg]" %}
 2924   interface(MEMORY_INTER) %{
 2925     base($reg);
 2926     index(0x4);
 2927     scale(0x0);
 2928     disp(0x0);
 2929   %}
 2930 %}
 2931 
 2932 // Indirect Memory Plus Short Offset Operand
 2933 operand indOffset8(any_RegP reg, immL8 off)
 2934 %{
 2935   constraint(ALLOC_IN_RC(ptr_reg));
 2936   match(AddP reg off);
 2937 
 2938   format %{ "[$reg + $off (8-bit)]" %}
 2939   interface(MEMORY_INTER) %{
 2940     base($reg);
 2941     index(0x4);
 2942     scale(0x0);
 2943     disp($off);
 2944   %}
 2945 %}
 2946 
 2947 // Indirect Memory Plus Long Offset Operand
 2948 operand indOffset32(any_RegP reg, immL32 off)
 2949 %{
 2950   constraint(ALLOC_IN_RC(ptr_reg));
 2951   match(AddP reg off);
 2952 
 2953   format %{ "[$reg + $off (32-bit)]" %}
 2954   interface(MEMORY_INTER) %{
 2955     base($reg);
 2956     index(0x4);
 2957     scale(0x0);
 2958     disp($off);
 2959   %}
 2960 %}
 2961 
 2962 // Indirect Memory Plus Index Register Plus Offset Operand
 2963 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
 2964 %{
 2965   constraint(ALLOC_IN_RC(ptr_reg));
 2966   match(AddP (AddP reg lreg) off);
 2967 
 2968   op_cost(10);
 2969   format %{"[$reg + $off + $lreg]" %}
 2970   interface(MEMORY_INTER) %{
 2971     base($reg);
 2972     index($lreg);
 2973     scale(0x0);
 2974     disp($off);
 2975   %}
 2976 %}
 2977 
 2978 // Indirect Memory Plus Index Register Plus Offset Operand
 2979 operand indIndex(any_RegP reg, rRegL lreg)
 2980 %{
 2981   constraint(ALLOC_IN_RC(ptr_reg));
 2982   match(AddP reg lreg);
 2983 
 2984   op_cost(10);
 2985   format %{"[$reg + $lreg]" %}
 2986   interface(MEMORY_INTER) %{
 2987     base($reg);
 2988     index($lreg);
 2989     scale(0x0);
 2990     disp(0x0);
 2991   %}
 2992 %}
 2993 
 2994 // Indirect Memory Times Scale Plus Index Register
 2995 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
 2996 %{
 2997   constraint(ALLOC_IN_RC(ptr_reg));
 2998   match(AddP reg (LShiftL lreg scale));
 2999 
 3000   op_cost(10);
 3001   format %{"[$reg + $lreg << $scale]" %}
 3002   interface(MEMORY_INTER) %{
 3003     base($reg);
 3004     index($lreg);
 3005     scale($scale);
 3006     disp(0x0);
 3007   %}
 3008 %}
 3009 
 3010 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
 3011 %{
 3012   constraint(ALLOC_IN_RC(ptr_reg));
 3013   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3014   match(AddP reg (LShiftL (ConvI2L idx) scale));
 3015 
 3016   op_cost(10);
 3017   format %{"[$reg + pos $idx << $scale]" %}
 3018   interface(MEMORY_INTER) %{
 3019     base($reg);
 3020     index($idx);
 3021     scale($scale);
 3022     disp(0x0);
 3023   %}
 3024 %}
 3025 
 3026 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3027 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
 3028 %{
 3029   constraint(ALLOC_IN_RC(ptr_reg));
 3030   match(AddP (AddP reg (LShiftL lreg scale)) off);
 3031 
 3032   op_cost(10);
 3033   format %{"[$reg + $off + $lreg << $scale]" %}
 3034   interface(MEMORY_INTER) %{
 3035     base($reg);
 3036     index($lreg);
 3037     scale($scale);
 3038     disp($off);
 3039   %}
 3040 %}
 3041 
 3042 // Indirect Memory Plus Positive Index Register Plus Offset Operand
 3043 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
 3044 %{
 3045   constraint(ALLOC_IN_RC(ptr_reg));
 3046   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3047   match(AddP (AddP reg (ConvI2L idx)) off);
 3048 
 3049   op_cost(10);
 3050   format %{"[$reg + $off + $idx]" %}
 3051   interface(MEMORY_INTER) %{
 3052     base($reg);
 3053     index($idx);
 3054     scale(0x0);
 3055     disp($off);
 3056   %}
 3057 %}
 3058 
 3059 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3060 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
 3061 %{
 3062   constraint(ALLOC_IN_RC(ptr_reg));
 3063   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3064   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
 3065 
 3066   op_cost(10);
 3067   format %{"[$reg + $off + $idx << $scale]" %}
 3068   interface(MEMORY_INTER) %{
 3069     base($reg);
 3070     index($idx);
 3071     scale($scale);
 3072     disp($off);
 3073   %}
 3074 %}
 3075 
 3076 // Indirect Narrow Oop Plus Offset Operand
 3077 // Note: x86 architecture doesn't support "scale * index + offset" without a base
 3078 // we can't free r12 even with CompressedOops::base() == nullptr.
 3079 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
 3080   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
 3081   constraint(ALLOC_IN_RC(ptr_reg));
 3082   match(AddP (DecodeN reg) off);
 3083 
 3084   op_cost(10);
 3085   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
 3086   interface(MEMORY_INTER) %{
 3087     base(0xc); // R12
 3088     index($reg);
 3089     scale(0x3);
 3090     disp($off);
 3091   %}
 3092 %}
 3093 
 3094 // Indirect Memory Operand
 3095 operand indirectNarrow(rRegN reg)
 3096 %{
 3097   predicate(CompressedOops::shift() == 0);
 3098   constraint(ALLOC_IN_RC(ptr_reg));
 3099   match(DecodeN reg);
 3100 
 3101   format %{ "[$reg]" %}
 3102   interface(MEMORY_INTER) %{
 3103     base($reg);
 3104     index(0x4);
 3105     scale(0x0);
 3106     disp(0x0);
 3107   %}
 3108 %}
 3109 
 3110 // Indirect Memory Plus Short Offset Operand
 3111 operand indOffset8Narrow(rRegN reg, immL8 off)
 3112 %{
 3113   predicate(CompressedOops::shift() == 0);
 3114   constraint(ALLOC_IN_RC(ptr_reg));
 3115   match(AddP (DecodeN reg) off);
 3116 
 3117   format %{ "[$reg + $off (8-bit)]" %}
 3118   interface(MEMORY_INTER) %{
 3119     base($reg);
 3120     index(0x4);
 3121     scale(0x0);
 3122     disp($off);
 3123   %}
 3124 %}
 3125 
 3126 // Indirect Memory Plus Long Offset Operand
 3127 operand indOffset32Narrow(rRegN reg, immL32 off)
 3128 %{
 3129   predicate(CompressedOops::shift() == 0);
 3130   constraint(ALLOC_IN_RC(ptr_reg));
 3131   match(AddP (DecodeN reg) off);
 3132 
 3133   format %{ "[$reg + $off (32-bit)]" %}
 3134   interface(MEMORY_INTER) %{
 3135     base($reg);
 3136     index(0x4);
 3137     scale(0x0);
 3138     disp($off);
 3139   %}
 3140 %}
 3141 
 3142 // Indirect Memory Plus Index Register Plus Offset Operand
 3143 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
 3144 %{
 3145   predicate(CompressedOops::shift() == 0);
 3146   constraint(ALLOC_IN_RC(ptr_reg));
 3147   match(AddP (AddP (DecodeN reg) lreg) off);
 3148 
 3149   op_cost(10);
 3150   format %{"[$reg + $off + $lreg]" %}
 3151   interface(MEMORY_INTER) %{
 3152     base($reg);
 3153     index($lreg);
 3154     scale(0x0);
 3155     disp($off);
 3156   %}
 3157 %}
 3158 
 3159 // Indirect Memory Plus Index Register Plus Offset Operand
 3160 operand indIndexNarrow(rRegN reg, rRegL lreg)
 3161 %{
 3162   predicate(CompressedOops::shift() == 0);
 3163   constraint(ALLOC_IN_RC(ptr_reg));
 3164   match(AddP (DecodeN reg) lreg);
 3165 
 3166   op_cost(10);
 3167   format %{"[$reg + $lreg]" %}
 3168   interface(MEMORY_INTER) %{
 3169     base($reg);
 3170     index($lreg);
 3171     scale(0x0);
 3172     disp(0x0);
 3173   %}
 3174 %}
 3175 
 3176 // Indirect Memory Times Scale Plus Index Register
 3177 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
 3178 %{
 3179   predicate(CompressedOops::shift() == 0);
 3180   constraint(ALLOC_IN_RC(ptr_reg));
 3181   match(AddP (DecodeN reg) (LShiftL lreg scale));
 3182 
 3183   op_cost(10);
 3184   format %{"[$reg + $lreg << $scale]" %}
 3185   interface(MEMORY_INTER) %{
 3186     base($reg);
 3187     index($lreg);
 3188     scale($scale);
 3189     disp(0x0);
 3190   %}
 3191 %}
 3192 
 3193 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
 3194 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
 3195 %{
 3196   predicate(CompressedOops::shift() == 0);
 3197   constraint(ALLOC_IN_RC(ptr_reg));
 3198   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
 3199 
 3200   op_cost(10);
 3201   format %{"[$reg + $off + $lreg << $scale]" %}
 3202   interface(MEMORY_INTER) %{
 3203     base($reg);
 3204     index($lreg);
 3205     scale($scale);
 3206     disp($off);
 3207   %}
 3208 %}
 3209 
 3210 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
 3211 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
 3212 %{
 3213   constraint(ALLOC_IN_RC(ptr_reg));
 3214   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
 3215   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
 3216 
 3217   op_cost(10);
 3218   format %{"[$reg + $off + $idx]" %}
 3219   interface(MEMORY_INTER) %{
 3220     base($reg);
 3221     index($idx);
 3222     scale(0x0);
 3223     disp($off);
 3224   %}
 3225 %}
 3226 
 3227 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
 3228 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
 3229 %{
 3230   constraint(ALLOC_IN_RC(ptr_reg));
 3231   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
 3232   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
 3233 
 3234   op_cost(10);
 3235   format %{"[$reg + $off + $idx << $scale]" %}
 3236   interface(MEMORY_INTER) %{
 3237     base($reg);
 3238     index($idx);
 3239     scale($scale);
 3240     disp($off);
 3241   %}
 3242 %}
 3243 
 3244 //----------Special Memory Operands--------------------------------------------
 3245 // Stack Slot Operand - This operand is used for loading and storing temporary
 3246 //                      values on the stack where a match requires a value to
 3247 //                      flow through memory.
 3248 operand stackSlotP(sRegP reg)
 3249 %{
 3250   constraint(ALLOC_IN_RC(stack_slots));
 3251   // No match rule because this operand is only generated in matching
 3252 
 3253   format %{ "[$reg]" %}
 3254   interface(MEMORY_INTER) %{
 3255     base(0x4);   // RSP
 3256     index(0x4);  // No Index
 3257     scale(0x0);  // No Scale
 3258     disp($reg);  // Stack Offset
 3259   %}
 3260 %}
 3261 
 3262 operand stackSlotI(sRegI reg)
 3263 %{
 3264   constraint(ALLOC_IN_RC(stack_slots));
 3265   // No match rule because this operand is only generated in matching
 3266 
 3267   format %{ "[$reg]" %}
 3268   interface(MEMORY_INTER) %{
 3269     base(0x4);   // RSP
 3270     index(0x4);  // No Index
 3271     scale(0x0);  // No Scale
 3272     disp($reg);  // Stack Offset
 3273   %}
 3274 %}
 3275 
 3276 operand stackSlotF(sRegF reg)
 3277 %{
 3278   constraint(ALLOC_IN_RC(stack_slots));
 3279   // No match rule because this operand is only generated in matching
 3280 
 3281   format %{ "[$reg]" %}
 3282   interface(MEMORY_INTER) %{
 3283     base(0x4);   // RSP
 3284     index(0x4);  // No Index
 3285     scale(0x0);  // No Scale
 3286     disp($reg);  // Stack Offset
 3287   %}
 3288 %}
 3289 
 3290 operand stackSlotD(sRegD reg)
 3291 %{
 3292   constraint(ALLOC_IN_RC(stack_slots));
 3293   // No match rule because this operand is only generated in matching
 3294 
 3295   format %{ "[$reg]" %}
 3296   interface(MEMORY_INTER) %{
 3297     base(0x4);   // RSP
 3298     index(0x4);  // No Index
 3299     scale(0x0);  // No Scale
 3300     disp($reg);  // Stack Offset
 3301   %}
 3302 %}
 3303 operand stackSlotL(sRegL reg)
 3304 %{
 3305   constraint(ALLOC_IN_RC(stack_slots));
 3306   // No match rule because this operand is only generated in matching
 3307 
 3308   format %{ "[$reg]" %}
 3309   interface(MEMORY_INTER) %{
 3310     base(0x4);   // RSP
 3311     index(0x4);  // No Index
 3312     scale(0x0);  // No Scale
 3313     disp($reg);  // Stack Offset
 3314   %}
 3315 %}
 3316 
 3317 //----------Conditional Branch Operands----------------------------------------
 3318 // Comparison Op  - This is the operation of the comparison, and is limited to
 3319 //                  the following set of codes:
 3320 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
 3321 //
 3322 // Other attributes of the comparison, such as unsignedness, are specified
 3323 // by the comparison instruction that sets a condition code flags register.
 3324 // That result is represented by a flags operand whose subtype is appropriate
 3325 // to the unsignedness (etc.) of the comparison.
 3326 //
 3327 // Later, the instruction which matches both the Comparison Op (a Bool) and
 3328 // the flags (produced by the Cmp) specifies the coding of the comparison op
 3329 // by matching a specific subtype of Bool operand below, such as cmpOpU.
 3330 
 3331 // Comparison Code
 3332 operand cmpOp()
 3333 %{
 3334   match(Bool);
 3335 
 3336   format %{ "" %}
 3337   interface(COND_INTER) %{
 3338     equal(0x4, "e");
 3339     not_equal(0x5, "ne");
 3340     less(0xC, "l");
 3341     greater_equal(0xD, "ge");
 3342     less_equal(0xE, "le");
 3343     greater(0xF, "g");
 3344     overflow(0x0, "o");
 3345     no_overflow(0x1, "no");
 3346   %}
 3347 %}
 3348 
 3349 // Comparison Code, unsigned compare.  Used by FP also, with
 3350 // C2 (unordered) turned into GT or LT already.  The other bits
 3351 // C0 and C3 are turned into Carry & Zero flags.
 3352 operand cmpOpU()
 3353 %{
 3354   match(Bool);
 3355 
 3356   format %{ "" %}
 3357   interface(COND_INTER) %{
 3358     equal(0x4, "e");
 3359     not_equal(0x5, "ne");
 3360     less(0x2, "b");
 3361     greater_equal(0x3, "ae");
 3362     less_equal(0x6, "be");
 3363     greater(0x7, "a");
 3364     overflow(0x0, "o");
 3365     no_overflow(0x1, "no");
 3366   %}
 3367 %}
 3368 
 3369 
 3370 // Floating comparisons that don't require any fixup for the unordered case,
 3371 // If both inputs of the comparison are the same, ZF is always set so we
 3372 // don't need to use cmpOpUCF2 for eq/ne
 3373 operand cmpOpUCF() %{
 3374   match(Bool);
 3375   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
 3376             n->as_Bool()->_test._test == BoolTest::ge ||
 3377             n->as_Bool()->_test._test == BoolTest::le ||
 3378             n->as_Bool()->_test._test == BoolTest::gt ||
 3379             n->in(1)->in(1) == n->in(1)->in(2));
 3380   format %{ "" %}
 3381   interface(COND_INTER) %{
 3382     equal(0xb, "np");
 3383     not_equal(0xa, "p");
 3384     less(0x2, "b");
 3385     greater_equal(0x3, "ae");
 3386     less_equal(0x6, "be");
 3387     greater(0x7, "a");
 3388     overflow(0x0, "o");
 3389     no_overflow(0x1, "no");
 3390   %}
 3391 %}
 3392 
 3393 
 3394 // Floating comparisons that can be fixed up with extra conditional jumps
 3395 operand cmpOpUCF2() %{
 3396   match(Bool);
 3397   predicate((n->as_Bool()->_test._test == BoolTest::ne ||
 3398              n->as_Bool()->_test._test == BoolTest::eq) &&
 3399             n->in(1)->in(1) != n->in(1)->in(2));
 3400   format %{ "" %}
 3401   interface(COND_INTER) %{
 3402     equal(0x4, "e");
 3403     not_equal(0x5, "ne");
 3404     less(0x2, "b");
 3405     greater_equal(0x3, "ae");
 3406     less_equal(0x6, "be");
 3407     greater(0x7, "a");
 3408     overflow(0x0, "o");
 3409     no_overflow(0x1, "no");
 3410   %}
 3411 %}
 3412 
 3413 //----------OPERAND CLASSES----------------------------------------------------
 3414 // Operand Classes are groups of operands that are used as to simplify
 3415 // instruction definitions by not requiring the AD writer to specify separate
 3416 // instructions for every form of operand when the instruction accepts
 3417 // multiple operand types with the same basic encoding and format.  The classic
 3418 // case of this is memory operands.
 3419 
 3420 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
 3421                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
 3422                indCompressedOopOffset,
 3423                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
 3424                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
 3425                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
 3426 
 3427 //----------PIPELINE-----------------------------------------------------------
 3428 // Rules which define the behavior of the target architectures pipeline.
 3429 pipeline %{
 3430 
 3431 //----------ATTRIBUTES---------------------------------------------------------
 3432 attributes %{
 3433   variable_size_instructions;        // Fixed size instructions
 3434   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
 3435   instruction_unit_size = 1;         // An instruction is 1 bytes long
 3436   instruction_fetch_unit_size = 16;  // The processor fetches one line
 3437   instruction_fetch_units = 1;       // of 16 bytes
 3438 %}
 3439 
 3440 //----------RESOURCES----------------------------------------------------------
 3441 // Resources are the functional units available to the machine
 3442 
 3443 // Generic P2/P3 pipeline
 3444 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
 3445 // 3 instructions decoded per cycle.
 3446 // 2 load/store ops per cycle, 1 branch, 1 FPU,
 3447 // 3 ALU op, only ALU0 handles mul instructions.
 3448 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
 3449            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
 3450            BR, FPU,
 3451            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
 3452 
 3453 //----------PIPELINE DESCRIPTION-----------------------------------------------
 3454 // Pipeline Description specifies the stages in the machine's pipeline
 3455 
 3456 // Generic P2/P3 pipeline
 3457 pipe_desc(S0, S1, S2, S3, S4, S5);
 3458 
 3459 //----------PIPELINE CLASSES---------------------------------------------------
 3460 // Pipeline Classes describe the stages in which input and output are
 3461 // referenced by the hardware pipeline.
 3462 
 3463 // Naming convention: ialu or fpu
 3464 // Then: _reg
 3465 // Then: _reg if there is a 2nd register
 3466 // Then: _long if it's a pair of instructions implementing a long
 3467 // Then: _fat if it requires the big decoder
 3468 //   Or: _mem if it requires the big decoder and a memory unit.
 3469 
 3470 // Integer ALU reg operation
 3471 pipe_class ialu_reg(rRegI dst)
 3472 %{
 3473     single_instruction;
 3474     dst    : S4(write);
 3475     dst    : S3(read);
 3476     DECODE : S0;        // any decoder
 3477     ALU    : S3;        // any alu
 3478 %}
 3479 
 3480 // Long ALU reg operation
 3481 pipe_class ialu_reg_long(rRegL dst)
 3482 %{
 3483     instruction_count(2);
 3484     dst    : S4(write);
 3485     dst    : S3(read);
 3486     DECODE : S0(2);     // any 2 decoders
 3487     ALU    : S3(2);     // both alus
 3488 %}
 3489 
 3490 // Integer ALU reg operation using big decoder
 3491 pipe_class ialu_reg_fat(rRegI dst)
 3492 %{
 3493     single_instruction;
 3494     dst    : S4(write);
 3495     dst    : S3(read);
 3496     D0     : S0;        // big decoder only
 3497     ALU    : S3;        // any alu
 3498 %}
 3499 
 3500 // Integer ALU reg-reg operation
 3501 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
 3502 %{
 3503     single_instruction;
 3504     dst    : S4(write);
 3505     src    : S3(read);
 3506     DECODE : S0;        // any decoder
 3507     ALU    : S3;        // any alu
 3508 %}
 3509 
 3510 // Integer ALU reg-reg operation
 3511 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
 3512 %{
 3513     single_instruction;
 3514     dst    : S4(write);
 3515     src    : S3(read);
 3516     D0     : S0;        // big decoder only
 3517     ALU    : S3;        // any alu
 3518 %}
 3519 
 3520 // Integer ALU reg-mem operation
 3521 pipe_class ialu_reg_mem(rRegI dst, memory mem)
 3522 %{
 3523     single_instruction;
 3524     dst    : S5(write);
 3525     mem    : S3(read);
 3526     D0     : S0;        // big decoder only
 3527     ALU    : S4;        // any alu
 3528     MEM    : S3;        // any mem
 3529 %}
 3530 
 3531 // Integer mem operation (prefetch)
 3532 pipe_class ialu_mem(memory mem)
 3533 %{
 3534     single_instruction;
 3535     mem    : S3(read);
 3536     D0     : S0;        // big decoder only
 3537     MEM    : S3;        // any mem
 3538 %}
 3539 
 3540 // Integer Store to Memory
 3541 pipe_class ialu_mem_reg(memory mem, rRegI src)
 3542 %{
 3543     single_instruction;
 3544     mem    : S3(read);
 3545     src    : S5(read);
 3546     D0     : S0;        // big decoder only
 3547     ALU    : S4;        // any alu
 3548     MEM    : S3;
 3549 %}
 3550 
 3551 // // Long Store to Memory
 3552 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
 3553 // %{
 3554 //     instruction_count(2);
 3555 //     mem    : S3(read);
 3556 //     src    : S5(read);
 3557 //     D0     : S0(2);          // big decoder only; twice
 3558 //     ALU    : S4(2);     // any 2 alus
 3559 //     MEM    : S3(2);  // Both mems
 3560 // %}
 3561 
 3562 // Integer Store to Memory
 3563 pipe_class ialu_mem_imm(memory mem)
 3564 %{
 3565     single_instruction;
 3566     mem    : S3(read);
 3567     D0     : S0;        // big decoder only
 3568     ALU    : S4;        // any alu
 3569     MEM    : S3;
 3570 %}
 3571 
 3572 // Integer ALU0 reg-reg operation
 3573 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
 3574 %{
 3575     single_instruction;
 3576     dst    : S4(write);
 3577     src    : S3(read);
 3578     D0     : S0;        // Big decoder only
 3579     ALU0   : S3;        // only alu0
 3580 %}
 3581 
 3582 // Integer ALU0 reg-mem operation
 3583 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
 3584 %{
 3585     single_instruction;
 3586     dst    : S5(write);
 3587     mem    : S3(read);
 3588     D0     : S0;        // big decoder only
 3589     ALU0   : S4;        // ALU0 only
 3590     MEM    : S3;        // any mem
 3591 %}
 3592 
 3593 // Integer ALU reg-reg operation
 3594 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
 3595 %{
 3596     single_instruction;
 3597     cr     : S4(write);
 3598     src1   : S3(read);
 3599     src2   : S3(read);
 3600     DECODE : S0;        // any decoder
 3601     ALU    : S3;        // any alu
 3602 %}
 3603 
 3604 // Integer ALU reg-imm operation
 3605 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
 3606 %{
 3607     single_instruction;
 3608     cr     : S4(write);
 3609     src1   : S3(read);
 3610     DECODE : S0;        // any decoder
 3611     ALU    : S3;        // any alu
 3612 %}
 3613 
 3614 // Integer ALU reg-mem operation
 3615 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
 3616 %{
 3617     single_instruction;
 3618     cr     : S4(write);
 3619     src1   : S3(read);
 3620     src2   : S3(read);
 3621     D0     : S0;        // big decoder only
 3622     ALU    : S4;        // any alu
 3623     MEM    : S3;
 3624 %}
 3625 
 3626 // Conditional move reg-reg
 3627 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
 3628 %{
 3629     instruction_count(4);
 3630     y      : S4(read);
 3631     q      : S3(read);
 3632     p      : S3(read);
 3633     DECODE : S0(4);     // any decoder
 3634 %}
 3635 
 3636 // Conditional move reg-reg
 3637 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
 3638 %{
 3639     single_instruction;
 3640     dst    : S4(write);
 3641     src    : S3(read);
 3642     cr     : S3(read);
 3643     DECODE : S0;        // any decoder
 3644 %}
 3645 
 3646 // Conditional move reg-mem
 3647 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
 3648 %{
 3649     single_instruction;
 3650     dst    : S4(write);
 3651     src    : S3(read);
 3652     cr     : S3(read);
 3653     DECODE : S0;        // any decoder
 3654     MEM    : S3;
 3655 %}
 3656 
 3657 // Conditional move reg-reg long
 3658 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
 3659 %{
 3660     single_instruction;
 3661     dst    : S4(write);
 3662     src    : S3(read);
 3663     cr     : S3(read);
 3664     DECODE : S0(2);     // any 2 decoders
 3665 %}
 3666 
 3667 // Float reg-reg operation
 3668 pipe_class fpu_reg(regD dst)
 3669 %{
 3670     instruction_count(2);
 3671     dst    : S3(read);
 3672     DECODE : S0(2);     // any 2 decoders
 3673     FPU    : S3;
 3674 %}
 3675 
 3676 // Float reg-reg operation
 3677 pipe_class fpu_reg_reg(regD dst, regD src)
 3678 %{
 3679     instruction_count(2);
 3680     dst    : S4(write);
 3681     src    : S3(read);
 3682     DECODE : S0(2);     // any 2 decoders
 3683     FPU    : S3;
 3684 %}
 3685 
 3686 // Float reg-reg operation
 3687 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
 3688 %{
 3689     instruction_count(3);
 3690     dst    : S4(write);
 3691     src1   : S3(read);
 3692     src2   : S3(read);
 3693     DECODE : S0(3);     // any 3 decoders
 3694     FPU    : S3(2);
 3695 %}
 3696 
 3697 // Float reg-reg operation
 3698 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
 3699 %{
 3700     instruction_count(4);
 3701     dst    : S4(write);
 3702     src1   : S3(read);
 3703     src2   : S3(read);
 3704     src3   : S3(read);
 3705     DECODE : S0(4);     // any 3 decoders
 3706     FPU    : S3(2);
 3707 %}
 3708 
 3709 // Float reg-reg operation
 3710 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
 3711 %{
 3712     instruction_count(4);
 3713     dst    : S4(write);
 3714     src1   : S3(read);
 3715     src2   : S3(read);
 3716     src3   : S3(read);
 3717     DECODE : S1(3);     // any 3 decoders
 3718     D0     : S0;        // Big decoder only
 3719     FPU    : S3(2);
 3720     MEM    : S3;
 3721 %}
 3722 
 3723 // Float reg-mem operation
 3724 pipe_class fpu_reg_mem(regD dst, memory mem)
 3725 %{
 3726     instruction_count(2);
 3727     dst    : S5(write);
 3728     mem    : S3(read);
 3729     D0     : S0;        // big decoder only
 3730     DECODE : S1;        // any decoder for FPU POP
 3731     FPU    : S4;
 3732     MEM    : S3;        // any mem
 3733 %}
 3734 
 3735 // Float reg-mem operation
 3736 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
 3737 %{
 3738     instruction_count(3);
 3739     dst    : S5(write);
 3740     src1   : S3(read);
 3741     mem    : S3(read);
 3742     D0     : S0;        // big decoder only
 3743     DECODE : S1(2);     // any decoder for FPU POP
 3744     FPU    : S4;
 3745     MEM    : S3;        // any mem
 3746 %}
 3747 
 3748 // Float mem-reg operation
 3749 pipe_class fpu_mem_reg(memory mem, regD src)
 3750 %{
 3751     instruction_count(2);
 3752     src    : S5(read);
 3753     mem    : S3(read);
 3754     DECODE : S0;        // any decoder for FPU PUSH
 3755     D0     : S1;        // big decoder only
 3756     FPU    : S4;
 3757     MEM    : S3;        // any mem
 3758 %}
 3759 
 3760 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
 3761 %{
 3762     instruction_count(3);
 3763     src1   : S3(read);
 3764     src2   : S3(read);
 3765     mem    : S3(read);
 3766     DECODE : S0(2);     // any decoder for FPU PUSH
 3767     D0     : S1;        // big decoder only
 3768     FPU    : S4;
 3769     MEM    : S3;        // any mem
 3770 %}
 3771 
 3772 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
 3773 %{
 3774     instruction_count(3);
 3775     src1   : S3(read);
 3776     src2   : S3(read);
 3777     mem    : S4(read);
 3778     DECODE : S0;        // any decoder for FPU PUSH
 3779     D0     : S0(2);     // big decoder only
 3780     FPU    : S4;
 3781     MEM    : S3(2);     // any mem
 3782 %}
 3783 
 3784 pipe_class fpu_mem_mem(memory dst, memory src1)
 3785 %{
 3786     instruction_count(2);
 3787     src1   : S3(read);
 3788     dst    : S4(read);
 3789     D0     : S0(2);     // big decoder only
 3790     MEM    : S3(2);     // any mem
 3791 %}
 3792 
 3793 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
 3794 %{
 3795     instruction_count(3);
 3796     src1   : S3(read);
 3797     src2   : S3(read);
 3798     dst    : S4(read);
 3799     D0     : S0(3);     // big decoder only
 3800     FPU    : S4;
 3801     MEM    : S3(3);     // any mem
 3802 %}
 3803 
 3804 pipe_class fpu_mem_reg_con(memory mem, regD src1)
 3805 %{
 3806     instruction_count(3);
 3807     src1   : S4(read);
 3808     mem    : S4(read);
 3809     DECODE : S0;        // any decoder for FPU PUSH
 3810     D0     : S0(2);     // big decoder only
 3811     FPU    : S4;
 3812     MEM    : S3(2);     // any mem
 3813 %}
 3814 
 3815 // Float load constant
 3816 pipe_class fpu_reg_con(regD dst)
 3817 %{
 3818     instruction_count(2);
 3819     dst    : S5(write);
 3820     D0     : S0;        // big decoder only for the load
 3821     DECODE : S1;        // any decoder for FPU POP
 3822     FPU    : S4;
 3823     MEM    : S3;        // any mem
 3824 %}
 3825 
 3826 // Float load constant
 3827 pipe_class fpu_reg_reg_con(regD dst, regD src)
 3828 %{
 3829     instruction_count(3);
 3830     dst    : S5(write);
 3831     src    : S3(read);
 3832     D0     : S0;        // big decoder only for the load
 3833     DECODE : S1(2);     // any decoder for FPU POP
 3834     FPU    : S4;
 3835     MEM    : S3;        // any mem
 3836 %}
 3837 
 3838 // UnConditional branch
 3839 pipe_class pipe_jmp(label labl)
 3840 %{
 3841     single_instruction;
 3842     BR   : S3;
 3843 %}
 3844 
 3845 // Conditional branch
 3846 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
 3847 %{
 3848     single_instruction;
 3849     cr    : S1(read);
 3850     BR    : S3;
 3851 %}
 3852 
 3853 // Allocation idiom
 3854 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
 3855 %{
 3856     instruction_count(1); force_serialization;
 3857     fixed_latency(6);
 3858     heap_ptr : S3(read);
 3859     DECODE   : S0(3);
 3860     D0       : S2;
 3861     MEM      : S3;
 3862     ALU      : S3(2);
 3863     dst      : S5(write);
 3864     BR       : S5;
 3865 %}
 3866 
 3867 // Generic big/slow expanded idiom
 3868 pipe_class pipe_slow()
 3869 %{
 3870     instruction_count(10); multiple_bundles; force_serialization;
 3871     fixed_latency(100);
 3872     D0  : S0(2);
 3873     MEM : S3(2);
 3874 %}
 3875 
 3876 // The real do-nothing guy
 3877 pipe_class empty()
 3878 %{
 3879     instruction_count(0);
 3880 %}
 3881 
 3882 // Define the class for the Nop node
 3883 define
 3884 %{
 3885    MachNop = empty;
 3886 %}
 3887 
 3888 %}
 3889 
 3890 //----------INSTRUCTIONS-------------------------------------------------------
 3891 //
 3892 // match      -- States which machine-independent subtree may be replaced
 3893 //               by this instruction.
 3894 // ins_cost   -- The estimated cost of this instruction is used by instruction
 3895 //               selection to identify a minimum cost tree of machine
 3896 //               instructions that matches a tree of machine-independent
 3897 //               instructions.
 3898 // format     -- A string providing the disassembly for this instruction.
 3899 //               The value of an instruction's operand may be inserted
 3900 //               by referring to it with a '$' prefix.
 3901 // opcode     -- Three instruction opcodes may be provided.  These are referred
 3902 //               to within an encode class as $primary, $secondary, and $tertiary
 3903 //               rrspectively.  The primary opcode is commonly used to
 3904 //               indicate the type of machine instruction, while secondary
 3905 //               and tertiary are often used for prefix options or addressing
 3906 //               modes.
 3907 // ins_encode -- A list of encode classes with parameters. The encode class
 3908 //               name must have been defined in an 'enc_class' specification
 3909 //               in the encode section of the architecture description.
 3910 
 3911 // Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
 3912 // Load Float
 3913 instruct MoveF2VL(vlRegF dst, regF src) %{
 3914   match(Set dst src);
 3915   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3916   ins_encode %{
 3917     ShouldNotReachHere();
 3918   %}
 3919   ins_pipe( fpu_reg_reg );
 3920 %}
 3921 
 3922 // Load Float
 3923 instruct MoveF2LEG(legRegF dst, regF src) %{
 3924   match(Set dst src);
 3925   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3926   ins_encode %{
 3927     ShouldNotReachHere();
 3928   %}
 3929   ins_pipe( fpu_reg_reg );
 3930 %}
 3931 
 3932 // Load Float
 3933 instruct MoveVL2F(regF dst, vlRegF src) %{
 3934   match(Set dst src);
 3935   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
 3936   ins_encode %{
 3937     ShouldNotReachHere();
 3938   %}
 3939   ins_pipe( fpu_reg_reg );
 3940 %}
 3941 
 3942 // Load Float
 3943 instruct MoveLEG2F(regF dst, legRegF src) %{
 3944   match(Set dst src);
 3945   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
 3946   ins_encode %{
 3947     ShouldNotReachHere();
 3948   %}
 3949   ins_pipe( fpu_reg_reg );
 3950 %}
 3951 
 3952 // Load Double
 3953 instruct MoveD2VL(vlRegD dst, regD src) %{
 3954   match(Set dst src);
 3955   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3956   ins_encode %{
 3957     ShouldNotReachHere();
 3958   %}
 3959   ins_pipe( fpu_reg_reg );
 3960 %}
 3961 
 3962 // Load Double
 3963 instruct MoveD2LEG(legRegD dst, regD src) %{
 3964   match(Set dst src);
 3965   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3966   ins_encode %{
 3967     ShouldNotReachHere();
 3968   %}
 3969   ins_pipe( fpu_reg_reg );
 3970 %}
 3971 
 3972 // Load Double
 3973 instruct MoveVL2D(regD dst, vlRegD src) %{
 3974   match(Set dst src);
 3975   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
 3976   ins_encode %{
 3977     ShouldNotReachHere();
 3978   %}
 3979   ins_pipe( fpu_reg_reg );
 3980 %}
 3981 
 3982 // Load Double
 3983 instruct MoveLEG2D(regD dst, legRegD src) %{
 3984   match(Set dst src);
 3985   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
 3986   ins_encode %{
 3987     ShouldNotReachHere();
 3988   %}
 3989   ins_pipe( fpu_reg_reg );
 3990 %}
 3991 
 3992 //----------Load/Store/Move Instructions---------------------------------------
 3993 //----------Load Instructions--------------------------------------------------
 3994 
 3995 // Load Byte (8 bit signed)
 3996 instruct loadB(rRegI dst, memory mem)
 3997 %{
 3998   match(Set dst (LoadB mem));
 3999 
 4000   ins_cost(125);
 4001   format %{ "movsbl  $dst, $mem\t# byte" %}
 4002 
 4003   ins_encode %{
 4004     __ movsbl($dst$$Register, $mem$$Address);
 4005   %}
 4006 
 4007   ins_pipe(ialu_reg_mem);
 4008 %}
 4009 
 4010 // Load Byte (8 bit signed) into Long Register
 4011 instruct loadB2L(rRegL dst, memory mem)
 4012 %{
 4013   match(Set dst (ConvI2L (LoadB mem)));
 4014 
 4015   ins_cost(125);
 4016   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
 4017 
 4018   ins_encode %{
 4019     __ movsbq($dst$$Register, $mem$$Address);
 4020   %}
 4021 
 4022   ins_pipe(ialu_reg_mem);
 4023 %}
 4024 
 4025 // Load Unsigned Byte (8 bit UNsigned)
 4026 instruct loadUB(rRegI dst, memory mem)
 4027 %{
 4028   match(Set dst (LoadUB mem));
 4029 
 4030   ins_cost(125);
 4031   format %{ "movzbl  $dst, $mem\t# ubyte" %}
 4032 
 4033   ins_encode %{
 4034     __ movzbl($dst$$Register, $mem$$Address);
 4035   %}
 4036 
 4037   ins_pipe(ialu_reg_mem);
 4038 %}
 4039 
 4040 // Load Unsigned Byte (8 bit UNsigned) into Long Register
 4041 instruct loadUB2L(rRegL dst, memory mem)
 4042 %{
 4043   match(Set dst (ConvI2L (LoadUB mem)));
 4044 
 4045   ins_cost(125);
 4046   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
 4047 
 4048   ins_encode %{
 4049     __ movzbq($dst$$Register, $mem$$Address);
 4050   %}
 4051 
 4052   ins_pipe(ialu_reg_mem);
 4053 %}
 4054 
 4055 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
 4056 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4057   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
 4058   effect(KILL cr);
 4059 
 4060   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
 4061             "andl    $dst, right_n_bits($mask, 8)" %}
 4062   ins_encode %{
 4063     Register Rdst = $dst$$Register;
 4064     __ movzbq(Rdst, $mem$$Address);
 4065     __ andl(Rdst, $mask$$constant & right_n_bits(8));
 4066   %}
 4067   ins_pipe(ialu_reg_mem);
 4068 %}
 4069 
 4070 // Load Short (16 bit signed)
 4071 instruct loadS(rRegI dst, memory mem)
 4072 %{
 4073   match(Set dst (LoadS mem));
 4074 
 4075   ins_cost(125);
 4076   format %{ "movswl $dst, $mem\t# short" %}
 4077 
 4078   ins_encode %{
 4079     __ movswl($dst$$Register, $mem$$Address);
 4080   %}
 4081 
 4082   ins_pipe(ialu_reg_mem);
 4083 %}
 4084 
 4085 // Load Short (16 bit signed) to Byte (8 bit signed)
 4086 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4087   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
 4088 
 4089   ins_cost(125);
 4090   format %{ "movsbl $dst, $mem\t# short -> byte" %}
 4091   ins_encode %{
 4092     __ movsbl($dst$$Register, $mem$$Address);
 4093   %}
 4094   ins_pipe(ialu_reg_mem);
 4095 %}
 4096 
 4097 // Load Short (16 bit signed) into Long Register
 4098 instruct loadS2L(rRegL dst, memory mem)
 4099 %{
 4100   match(Set dst (ConvI2L (LoadS mem)));
 4101 
 4102   ins_cost(125);
 4103   format %{ "movswq $dst, $mem\t# short -> long" %}
 4104 
 4105   ins_encode %{
 4106     __ movswq($dst$$Register, $mem$$Address);
 4107   %}
 4108 
 4109   ins_pipe(ialu_reg_mem);
 4110 %}
 4111 
 4112 // Load Unsigned Short/Char (16 bit UNsigned)
 4113 instruct loadUS(rRegI dst, memory mem)
 4114 %{
 4115   match(Set dst (LoadUS mem));
 4116 
 4117   ins_cost(125);
 4118   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
 4119 
 4120   ins_encode %{
 4121     __ movzwl($dst$$Register, $mem$$Address);
 4122   %}
 4123 
 4124   ins_pipe(ialu_reg_mem);
 4125 %}
 4126 
 4127 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
 4128 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4129   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
 4130 
 4131   ins_cost(125);
 4132   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
 4133   ins_encode %{
 4134     __ movsbl($dst$$Register, $mem$$Address);
 4135   %}
 4136   ins_pipe(ialu_reg_mem);
 4137 %}
 4138 
 4139 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
 4140 instruct loadUS2L(rRegL dst, memory mem)
 4141 %{
 4142   match(Set dst (ConvI2L (LoadUS mem)));
 4143 
 4144   ins_cost(125);
 4145   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
 4146 
 4147   ins_encode %{
 4148     __ movzwq($dst$$Register, $mem$$Address);
 4149   %}
 4150 
 4151   ins_pipe(ialu_reg_mem);
 4152 %}
 4153 
 4154 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
 4155 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4156   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4157 
 4158   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
 4159   ins_encode %{
 4160     __ movzbq($dst$$Register, $mem$$Address);
 4161   %}
 4162   ins_pipe(ialu_reg_mem);
 4163 %}
 4164 
 4165 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
 4166 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 4167   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
 4168   effect(KILL cr);
 4169 
 4170   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
 4171             "andl    $dst, right_n_bits($mask, 16)" %}
 4172   ins_encode %{
 4173     Register Rdst = $dst$$Register;
 4174     __ movzwq(Rdst, $mem$$Address);
 4175     __ andl(Rdst, $mask$$constant & right_n_bits(16));
 4176   %}
 4177   ins_pipe(ialu_reg_mem);
 4178 %}
 4179 
 4180 // Load Integer
 4181 instruct loadI(rRegI dst, memory mem)
 4182 %{
 4183   match(Set dst (LoadI mem));
 4184 
 4185   ins_cost(125);
 4186   format %{ "movl    $dst, $mem\t# int" %}
 4187 
 4188   ins_encode %{
 4189     __ movl($dst$$Register, $mem$$Address);
 4190   %}
 4191 
 4192   ins_pipe(ialu_reg_mem);
 4193 %}
 4194 
 4195 // Load Integer (32 bit signed) to Byte (8 bit signed)
 4196 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
 4197   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
 4198 
 4199   ins_cost(125);
 4200   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
 4201   ins_encode %{
 4202     __ movsbl($dst$$Register, $mem$$Address);
 4203   %}
 4204   ins_pipe(ialu_reg_mem);
 4205 %}
 4206 
 4207 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
 4208 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
 4209   match(Set dst (AndI (LoadI mem) mask));
 4210 
 4211   ins_cost(125);
 4212   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
 4213   ins_encode %{
 4214     __ movzbl($dst$$Register, $mem$$Address);
 4215   %}
 4216   ins_pipe(ialu_reg_mem);
 4217 %}
 4218 
 4219 // Load Integer (32 bit signed) to Short (16 bit signed)
 4220 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
 4221   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
 4222 
 4223   ins_cost(125);
 4224   format %{ "movswl  $dst, $mem\t# int -> short" %}
 4225   ins_encode %{
 4226     __ movswl($dst$$Register, $mem$$Address);
 4227   %}
 4228   ins_pipe(ialu_reg_mem);
 4229 %}
 4230 
 4231 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
 4232 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
 4233   match(Set dst (AndI (LoadI mem) mask));
 4234 
 4235   ins_cost(125);
 4236   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
 4237   ins_encode %{
 4238     __ movzwl($dst$$Register, $mem$$Address);
 4239   %}
 4240   ins_pipe(ialu_reg_mem);
 4241 %}
 4242 
 4243 // Load Integer into Long Register
 4244 instruct loadI2L(rRegL dst, memory mem)
 4245 %{
 4246   match(Set dst (ConvI2L (LoadI mem)));
 4247 
 4248   ins_cost(125);
 4249   format %{ "movslq  $dst, $mem\t# int -> long" %}
 4250 
 4251   ins_encode %{
 4252     __ movslq($dst$$Register, $mem$$Address);
 4253   %}
 4254 
 4255   ins_pipe(ialu_reg_mem);
 4256 %}
 4257 
 4258 // Load Integer with mask 0xFF into Long Register
 4259 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
 4260   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4261 
 4262   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
 4263   ins_encode %{
 4264     __ movzbq($dst$$Register, $mem$$Address);
 4265   %}
 4266   ins_pipe(ialu_reg_mem);
 4267 %}
 4268 
 4269 // Load Integer with mask 0xFFFF into Long Register
 4270 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
 4271   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4272 
 4273   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
 4274   ins_encode %{
 4275     __ movzwq($dst$$Register, $mem$$Address);
 4276   %}
 4277   ins_pipe(ialu_reg_mem);
 4278 %}
 4279 
 4280 // Load Integer with a 31-bit mask into Long Register
 4281 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
 4282   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
 4283   effect(KILL cr);
 4284 
 4285   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
 4286             "andl    $dst, $mask" %}
 4287   ins_encode %{
 4288     Register Rdst = $dst$$Register;
 4289     __ movl(Rdst, $mem$$Address);
 4290     __ andl(Rdst, $mask$$constant);
 4291   %}
 4292   ins_pipe(ialu_reg_mem);
 4293 %}
 4294 
 4295 // Load Unsigned Integer into Long Register
 4296 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
 4297 %{
 4298   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 4299 
 4300   ins_cost(125);
 4301   format %{ "movl    $dst, $mem\t# uint -> long" %}
 4302 
 4303   ins_encode %{
 4304     __ movl($dst$$Register, $mem$$Address);
 4305   %}
 4306 
 4307   ins_pipe(ialu_reg_mem);
 4308 %}
 4309 
 4310 // Load Long
 4311 instruct loadL(rRegL dst, memory mem)
 4312 %{
 4313   match(Set dst (LoadL mem));
 4314 
 4315   ins_cost(125);
 4316   format %{ "movq    $dst, $mem\t# long" %}
 4317 
 4318   ins_encode %{
 4319     __ movq($dst$$Register, $mem$$Address);
 4320   %}
 4321 
 4322   ins_pipe(ialu_reg_mem); // XXX
 4323 %}
 4324 
 4325 // Load Range
 4326 instruct loadRange(rRegI dst, memory mem)
 4327 %{
 4328   match(Set dst (LoadRange mem));
 4329 
 4330   ins_cost(125); // XXX
 4331   format %{ "movl    $dst, $mem\t# range" %}
 4332   ins_encode %{
 4333     __ movl($dst$$Register, $mem$$Address);
 4334   %}
 4335   ins_pipe(ialu_reg_mem);
 4336 %}
 4337 
 4338 // Load Pointer
 4339 instruct loadP(rRegP dst, memory mem)
 4340 %{
 4341   match(Set dst (LoadP mem));
 4342   predicate(n->as_Load()->barrier_data() == 0);
 4343 
 4344   ins_cost(125); // XXX
 4345   format %{ "movq    $dst, $mem\t# ptr" %}
 4346   ins_encode %{
 4347     __ movq($dst$$Register, $mem$$Address);
 4348   %}
 4349   ins_pipe(ialu_reg_mem); // XXX
 4350 %}
 4351 
 4352 // Load Compressed Pointer
 4353 instruct loadN(rRegN dst, memory mem)
 4354 %{
 4355    predicate(n->as_Load()->barrier_data() == 0);
 4356    match(Set dst (LoadN mem));
 4357 
 4358    ins_cost(125); // XXX
 4359    format %{ "movl    $dst, $mem\t# compressed ptr" %}
 4360    ins_encode %{
 4361      __ movl($dst$$Register, $mem$$Address);
 4362    %}
 4363    ins_pipe(ialu_reg_mem); // XXX
 4364 %}
 4365 
 4366 
 4367 // Load Klass Pointer
 4368 instruct loadKlass(rRegP dst, memory mem)
 4369 %{
 4370   match(Set dst (LoadKlass mem));
 4371 
 4372   ins_cost(125); // XXX
 4373   format %{ "movq    $dst, $mem\t# class" %}
 4374   ins_encode %{
 4375     __ movq($dst$$Register, $mem$$Address);
 4376   %}
 4377   ins_pipe(ialu_reg_mem); // XXX
 4378 %}
 4379 
 4380 // Load narrow Klass Pointer
 4381 instruct loadNKlass(rRegN dst, memory mem)
 4382 %{
 4383   predicate(!UseCompactObjectHeaders);
 4384   match(Set dst (LoadNKlass mem));
 4385 
 4386   ins_cost(125); // XXX
 4387   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
 4388   ins_encode %{
 4389     __ movl($dst$$Register, $mem$$Address);
 4390   %}
 4391   ins_pipe(ialu_reg_mem); // XXX
 4392 %}
 4393 
 4394 instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
 4395 %{
 4396   predicate(UseCompactObjectHeaders);
 4397   match(Set dst (LoadNKlass mem));
 4398   effect(KILL cr);
 4399   ins_cost(125);
 4400   format %{
 4401     "movl    $dst, $mem\t# compressed klass ptr, shifted\n\t"
 4402     "shrl    $dst, markWord::klass_shift_at_offset"
 4403   %}
 4404   ins_encode %{
 4405     if (UseAPX) {
 4406       __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
 4407     }
 4408     else {
 4409       __ movl($dst$$Register, $mem$$Address);
 4410       __ shrl($dst$$Register, markWord::klass_shift_at_offset);
 4411     }
 4412   %}
 4413   ins_pipe(ialu_reg_mem);
 4414 %}
 4415 
 4416 // Load Float
 4417 instruct loadF(regF dst, memory mem)
 4418 %{
 4419   match(Set dst (LoadF mem));
 4420 
 4421   ins_cost(145); // XXX
 4422   format %{ "movss   $dst, $mem\t# float" %}
 4423   ins_encode %{
 4424     __ movflt($dst$$XMMRegister, $mem$$Address);
 4425   %}
 4426   ins_pipe(pipe_slow); // XXX
 4427 %}
 4428 
 4429 // Load Double
 4430 instruct loadD_partial(regD dst, memory mem)
 4431 %{
 4432   predicate(!UseXmmLoadAndClearUpper);
 4433   match(Set dst (LoadD mem));
 4434 
 4435   ins_cost(145); // XXX
 4436   format %{ "movlpd  $dst, $mem\t# double" %}
 4437   ins_encode %{
 4438     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4439   %}
 4440   ins_pipe(pipe_slow); // XXX
 4441 %}
 4442 
 4443 instruct loadD(regD dst, memory mem)
 4444 %{
 4445   predicate(UseXmmLoadAndClearUpper);
 4446   match(Set dst (LoadD mem));
 4447 
 4448   ins_cost(145); // XXX
 4449   format %{ "movsd   $dst, $mem\t# double" %}
 4450   ins_encode %{
 4451     __ movdbl($dst$$XMMRegister, $mem$$Address);
 4452   %}
 4453   ins_pipe(pipe_slow); // XXX
 4454 %}
 4455 
 4456 instruct loadAOTRCAddress(rRegP dst, immAOTRuntimeConstantsAddress con)
 4457 %{
 4458   match(Set dst con);
 4459 
 4460   format %{ "leaq  $dst, $con\t# AOT Runtime Constants Address" %}
 4461 
 4462   ins_encode %{
 4463     __ load_aotrc_address($dst$$Register, (address)$con$$constant);
 4464   %}
 4465 
 4466   ins_pipe(ialu_reg_fat);
 4467 %}
 4468 
 4469 // max = java.lang.Math.max(float a, float b)
 4470 instruct maxF_avx10_reg(regF dst, regF a, regF b) %{
 4471   predicate(VM_Version::supports_avx10_2());
 4472   match(Set dst (MaxF a b));
 4473   format %{ "maxF $dst, $a, $b" %}
 4474   ins_encode %{
 4475     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4476   %}
 4477   ins_pipe( pipe_slow );
 4478 %}
 4479 
 4480 // max = java.lang.Math.max(float a, float b)
 4481 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4482   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4483   match(Set dst (MaxF a b));
 4484   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4485   format %{ "maxF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4486   ins_encode %{
 4487     __ vminmax_fp(Op_MaxV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4488   %}
 4489   ins_pipe( pipe_slow );
 4490 %}
 4491 
 4492 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4493   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4494   match(Set dst (MaxF a b));
 4495   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4496 
 4497   format %{ "maxF_reduction $dst, $a, $b \t!using $xtmp and $rtmp as TEMP" %}
 4498   ins_encode %{
 4499     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4500                     false /*min*/, true /*single*/);
 4501   %}
 4502   ins_pipe( pipe_slow );
 4503 %}
 4504 
 4505 // max = java.lang.Math.max(double a, double b)
 4506 instruct maxD_avx10_reg(regD dst, regD a, regD b) %{
 4507   predicate(VM_Version::supports_avx10_2());
 4508   match(Set dst (MaxD a b));
 4509   format %{ "maxD $dst, $a, $b" %}
 4510   ins_encode %{
 4511     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MAX_COMPARE_SIGN);
 4512   %}
 4513   ins_pipe( pipe_slow );
 4514 %}
 4515 
 4516 // max = java.lang.Math.max(double a, double b)
 4517 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4518   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4519   match(Set dst (MaxD a b));
 4520   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
 4521   format %{ "maxD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4522   ins_encode %{
 4523     __ vminmax_fp(Op_MaxV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4524   %}
 4525   ins_pipe( pipe_slow );
 4526 %}
 4527 
 4528 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4529   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4530   match(Set dst (MaxD a b));
 4531   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4532 
 4533   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4534   ins_encode %{
 4535     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4536                     false /*min*/, false /*single*/);
 4537   %}
 4538   ins_pipe( pipe_slow );
 4539 %}
 4540 
 4541 // max = java.lang.Math.min(float a, float b)
 4542 instruct minF_avx10_reg(regF dst, regF a, regF b) %{
 4543   predicate(VM_Version::supports_avx10_2());
 4544   match(Set dst (MinF a b));
 4545   format %{ "minF $dst, $a, $b" %}
 4546   ins_encode %{
 4547     __ eminmaxss($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4548   %}
 4549   ins_pipe( pipe_slow );
 4550 %}
 4551 
 4552 // min = java.lang.Math.min(float a, float b)
 4553 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
 4554   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4555   match(Set dst (MinF a b));
 4556   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4557   format %{ "minF $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4558   ins_encode %{
 4559     __ vminmax_fp(Op_MinV, T_FLOAT, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4560   %}
 4561   ins_pipe( pipe_slow );
 4562 %}
 4563 
 4564 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xtmp, rRegI rtmp, rFlagsReg cr) %{
 4565   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4566   match(Set dst (MinF a b));
 4567   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4568 
 4569   format %{ "minF_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4570   ins_encode %{
 4571     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4572                     true /*min*/, true /*single*/);
 4573   %}
 4574   ins_pipe( pipe_slow );
 4575 %}
 4576 
 4577 // max = java.lang.Math.min(double a, double b)
 4578 instruct minD_avx10_reg(regD dst, regD a, regD b) %{
 4579   predicate(VM_Version::supports_avx10_2());
 4580   match(Set dst (MinD a b));
 4581   format %{ "minD $dst, $a, $b" %}
 4582   ins_encode %{
 4583     __ eminmaxsd($dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, AVX10_MINMAX_MIN_COMPARE_SIGN);
 4584   %}
 4585   ins_pipe( pipe_slow );
 4586 %}
 4587 
 4588 // min = java.lang.Math.min(double a, double b)
 4589 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
 4590   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && !VLoopReductions::is_reduction(n));
 4591   match(Set dst (MinD a b));
 4592   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
 4593     format %{ "minD $dst, $a, $b \t! using $tmp, $atmp and $btmp as TEMP" %}
 4594   ins_encode %{
 4595     __ vminmax_fp(Op_MinV, T_DOUBLE, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, Assembler::AVX_128bit);
 4596   %}
 4597   ins_pipe( pipe_slow );
 4598 %}
 4599 
 4600 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xtmp, rRegL rtmp, rFlagsReg cr) %{
 4601   predicate(!VM_Version::supports_avx10_2() && UseAVX > 0 && VLoopReductions::is_reduction(n));
 4602   match(Set dst (MinD a b));
 4603   effect(USE a, USE b, TEMP xtmp, TEMP rtmp, KILL cr);
 4604 
 4605   format %{ "maxD_reduction $dst, $a, $b \t! using $xtmp and $rtmp as TEMP" %}
 4606   ins_encode %{
 4607     emit_fp_min_max(masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xtmp$$XMMRegister, $rtmp$$Register,
 4608                     true /*min*/, false /*single*/);
 4609   %}
 4610   ins_pipe( pipe_slow );
 4611 %}
 4612 
 4613 // Load Effective Address
 4614 instruct leaP8(rRegP dst, indOffset8 mem)
 4615 %{
 4616   match(Set dst mem);
 4617 
 4618   ins_cost(110); // XXX
 4619   format %{ "leaq    $dst, $mem\t# ptr 8" %}
 4620   ins_encode %{
 4621     __ leaq($dst$$Register, $mem$$Address);
 4622   %}
 4623   ins_pipe(ialu_reg_reg_fat);
 4624 %}
 4625 
 4626 instruct leaP32(rRegP dst, indOffset32 mem)
 4627 %{
 4628   match(Set dst mem);
 4629 
 4630   ins_cost(110);
 4631   format %{ "leaq    $dst, $mem\t# ptr 32" %}
 4632   ins_encode %{
 4633     __ leaq($dst$$Register, $mem$$Address);
 4634   %}
 4635   ins_pipe(ialu_reg_reg_fat);
 4636 %}
 4637 
 4638 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
 4639 %{
 4640   match(Set dst mem);
 4641 
 4642   ins_cost(110);
 4643   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
 4644   ins_encode %{
 4645     __ leaq($dst$$Register, $mem$$Address);
 4646   %}
 4647   ins_pipe(ialu_reg_reg_fat);
 4648 %}
 4649 
 4650 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
 4651 %{
 4652   match(Set dst mem);
 4653 
 4654   ins_cost(110);
 4655   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4656   ins_encode %{
 4657     __ leaq($dst$$Register, $mem$$Address);
 4658   %}
 4659   ins_pipe(ialu_reg_reg_fat);
 4660 %}
 4661 
 4662 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
 4663 %{
 4664   match(Set dst mem);
 4665 
 4666   ins_cost(110);
 4667   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
 4668   ins_encode %{
 4669     __ leaq($dst$$Register, $mem$$Address);
 4670   %}
 4671   ins_pipe(ialu_reg_reg_fat);
 4672 %}
 4673 
 4674 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
 4675 %{
 4676   match(Set dst mem);
 4677 
 4678   ins_cost(110);
 4679   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
 4680   ins_encode %{
 4681     __ leaq($dst$$Register, $mem$$Address);
 4682   %}
 4683   ins_pipe(ialu_reg_reg_fat);
 4684 %}
 4685 
 4686 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
 4687 %{
 4688   match(Set dst mem);
 4689 
 4690   ins_cost(110);
 4691   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
 4692   ins_encode %{
 4693     __ leaq($dst$$Register, $mem$$Address);
 4694   %}
 4695   ins_pipe(ialu_reg_reg_fat);
 4696 %}
 4697 
 4698 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
 4699 %{
 4700   match(Set dst mem);
 4701 
 4702   ins_cost(110);
 4703   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
 4704   ins_encode %{
 4705     __ leaq($dst$$Register, $mem$$Address);
 4706   %}
 4707   ins_pipe(ialu_reg_reg_fat);
 4708 %}
 4709 
 4710 // Load Effective Address which uses Narrow (32-bits) oop
 4711 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
 4712 %{
 4713   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
 4714   match(Set dst mem);
 4715 
 4716   ins_cost(110);
 4717   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
 4718   ins_encode %{
 4719     __ leaq($dst$$Register, $mem$$Address);
 4720   %}
 4721   ins_pipe(ialu_reg_reg_fat);
 4722 %}
 4723 
 4724 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
 4725 %{
 4726   predicate(CompressedOops::shift() == 0);
 4727   match(Set dst mem);
 4728 
 4729   ins_cost(110); // XXX
 4730   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
 4731   ins_encode %{
 4732     __ leaq($dst$$Register, $mem$$Address);
 4733   %}
 4734   ins_pipe(ialu_reg_reg_fat);
 4735 %}
 4736 
 4737 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
 4738 %{
 4739   predicate(CompressedOops::shift() == 0);
 4740   match(Set dst mem);
 4741 
 4742   ins_cost(110);
 4743   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
 4744   ins_encode %{
 4745     __ leaq($dst$$Register, $mem$$Address);
 4746   %}
 4747   ins_pipe(ialu_reg_reg_fat);
 4748 %}
 4749 
 4750 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
 4751 %{
 4752   predicate(CompressedOops::shift() == 0);
 4753   match(Set dst mem);
 4754 
 4755   ins_cost(110);
 4756   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
 4757   ins_encode %{
 4758     __ leaq($dst$$Register, $mem$$Address);
 4759   %}
 4760   ins_pipe(ialu_reg_reg_fat);
 4761 %}
 4762 
 4763 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
 4764 %{
 4765   predicate(CompressedOops::shift() == 0);
 4766   match(Set dst mem);
 4767 
 4768   ins_cost(110);
 4769   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
 4770   ins_encode %{
 4771     __ leaq($dst$$Register, $mem$$Address);
 4772   %}
 4773   ins_pipe(ialu_reg_reg_fat);
 4774 %}
 4775 
 4776 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
 4777 %{
 4778   predicate(CompressedOops::shift() == 0);
 4779   match(Set dst mem);
 4780 
 4781   ins_cost(110);
 4782   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
 4783   ins_encode %{
 4784     __ leaq($dst$$Register, $mem$$Address);
 4785   %}
 4786   ins_pipe(ialu_reg_reg_fat);
 4787 %}
 4788 
 4789 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
 4790 %{
 4791   predicate(CompressedOops::shift() == 0);
 4792   match(Set dst mem);
 4793 
 4794   ins_cost(110);
 4795   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
 4796   ins_encode %{
 4797     __ leaq($dst$$Register, $mem$$Address);
 4798   %}
 4799   ins_pipe(ialu_reg_reg_fat);
 4800 %}
 4801 
 4802 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
 4803 %{
 4804   predicate(CompressedOops::shift() == 0);
 4805   match(Set dst mem);
 4806 
 4807   ins_cost(110);
 4808   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
 4809   ins_encode %{
 4810     __ leaq($dst$$Register, $mem$$Address);
 4811   %}
 4812   ins_pipe(ialu_reg_reg_fat);
 4813 %}
 4814 
 4815 instruct loadConI(rRegI dst, immI src)
 4816 %{
 4817   match(Set dst src);
 4818 
 4819   format %{ "movl    $dst, $src\t# int" %}
 4820   ins_encode %{
 4821     __ movl($dst$$Register, $src$$constant);
 4822   %}
 4823   ins_pipe(ialu_reg_fat); // XXX
 4824 %}
 4825 
 4826 instruct loadConI0(rRegI dst, immI_0 src, rFlagsReg cr)
 4827 %{
 4828   match(Set dst src);
 4829   effect(KILL cr);
 4830 
 4831   ins_cost(50);
 4832   format %{ "xorl    $dst, $dst\t# int" %}
 4833   ins_encode %{
 4834     __ xorl($dst$$Register, $dst$$Register);
 4835   %}
 4836   ins_pipe(ialu_reg);
 4837 %}
 4838 
 4839 instruct loadConL(rRegL dst, immL src)
 4840 %{
 4841   match(Set dst src);
 4842 
 4843   ins_cost(150);
 4844   format %{ "movq    $dst, $src\t# long" %}
 4845   ins_encode %{
 4846     __ mov64($dst$$Register, $src$$constant);
 4847   %}
 4848   ins_pipe(ialu_reg);
 4849 %}
 4850 
 4851 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
 4852 %{
 4853   match(Set dst src);
 4854   effect(KILL cr);
 4855 
 4856   ins_cost(50);
 4857   format %{ "xorl    $dst, $dst\t# long" %}
 4858   ins_encode %{
 4859     __ xorl($dst$$Register, $dst$$Register);
 4860   %}
 4861   ins_pipe(ialu_reg); // XXX
 4862 %}
 4863 
 4864 instruct loadConUL32(rRegL dst, immUL32 src)
 4865 %{
 4866   match(Set dst src);
 4867 
 4868   ins_cost(60);
 4869   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
 4870   ins_encode %{
 4871     __ movl($dst$$Register, $src$$constant);
 4872   %}
 4873   ins_pipe(ialu_reg);
 4874 %}
 4875 
 4876 instruct loadConL32(rRegL dst, immL32 src)
 4877 %{
 4878   match(Set dst src);
 4879 
 4880   ins_cost(70);
 4881   format %{ "movq    $dst, $src\t# long (32-bit)" %}
 4882   ins_encode %{
 4883     __ movq($dst$$Register, $src$$constant);
 4884   %}
 4885   ins_pipe(ialu_reg);
 4886 %}
 4887 
 4888 instruct loadConP(rRegP dst, immP con) %{
 4889   match(Set dst con);
 4890 
 4891   format %{ "movq    $dst, $con\t# ptr" %}
 4892   ins_encode %{
 4893     __ mov64($dst$$Register, $con$$constant, $con->constant_reloc(), RELOC_IMM64);
 4894   %}
 4895   ins_pipe(ialu_reg_fat); // XXX
 4896 %}
 4897 
 4898 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
 4899 %{
 4900   match(Set dst src);
 4901   effect(KILL cr);
 4902 
 4903   ins_cost(50);
 4904   format %{ "xorl    $dst, $dst\t# ptr" %}
 4905   ins_encode %{
 4906     __ xorl($dst$$Register, $dst$$Register);
 4907   %}
 4908   ins_pipe(ialu_reg);
 4909 %}
 4910 
 4911 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
 4912 %{
 4913   match(Set dst src);
 4914   effect(KILL cr);
 4915 
 4916   ins_cost(60);
 4917   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
 4918   ins_encode %{
 4919     __ movl($dst$$Register, $src$$constant);
 4920   %}
 4921   ins_pipe(ialu_reg);
 4922 %}
 4923 
 4924 instruct loadConF(regF dst, immF con) %{
 4925   match(Set dst con);
 4926   ins_cost(125);
 4927   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
 4928   ins_encode %{
 4929     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4930   %}
 4931   ins_pipe(pipe_slow);
 4932 %}
 4933 
 4934 instruct loadConH(regF dst, immH con) %{
 4935   match(Set dst con);
 4936   ins_cost(125);
 4937   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: halffloat=$con" %}
 4938   ins_encode %{
 4939     __ movflt($dst$$XMMRegister, $constantaddress($con));
 4940   %}
 4941   ins_pipe(pipe_slow);
 4942 %}
 4943 
 4944 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
 4945   match(Set dst src);
 4946   effect(KILL cr);
 4947   format %{ "xorq    $dst, $src\t# compressed null pointer" %}
 4948   ins_encode %{
 4949     __ xorq($dst$$Register, $dst$$Register);
 4950   %}
 4951   ins_pipe(ialu_reg);
 4952 %}
 4953 
 4954 instruct loadConN(rRegN dst, immN src) %{
 4955   match(Set dst src);
 4956 
 4957   ins_cost(125);
 4958   format %{ "movl    $dst, $src\t# compressed ptr" %}
 4959   ins_encode %{
 4960     address con = (address)$src$$constant;
 4961     if (con == nullptr) {
 4962       ShouldNotReachHere();
 4963     } else {
 4964       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
 4965     }
 4966   %}
 4967   ins_pipe(ialu_reg_fat); // XXX
 4968 %}
 4969 
 4970 instruct loadConNKlass(rRegN dst, immNKlass src) %{
 4971   match(Set dst src);
 4972 
 4973   ins_cost(125);
 4974   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
 4975   ins_encode %{
 4976     address con = (address)$src$$constant;
 4977     if (con == nullptr) {
 4978       ShouldNotReachHere();
 4979     } else {
 4980       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
 4981     }
 4982   %}
 4983   ins_pipe(ialu_reg_fat); // XXX
 4984 %}
 4985 
 4986 instruct loadConF0(regF dst, immF0 src)
 4987 %{
 4988   match(Set dst src);
 4989   ins_cost(100);
 4990 
 4991   format %{ "xorps   $dst, $dst\t# float 0.0" %}
 4992   ins_encode %{
 4993     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
 4994   %}
 4995   ins_pipe(pipe_slow);
 4996 %}
 4997 
 4998 // Use the same format since predicate() can not be used here.
 4999 instruct loadConD(regD dst, immD con) %{
 5000   match(Set dst con);
 5001   ins_cost(125);
 5002   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
 5003   ins_encode %{
 5004     __ movdbl($dst$$XMMRegister, $constantaddress($con));
 5005   %}
 5006   ins_pipe(pipe_slow);
 5007 %}
 5008 
 5009 instruct loadConD0(regD dst, immD0 src)
 5010 %{
 5011   match(Set dst src);
 5012   ins_cost(100);
 5013 
 5014   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
 5015   ins_encode %{
 5016     __ xorpd($dst$$XMMRegister, $dst$$XMMRegister);
 5017   %}
 5018   ins_pipe(pipe_slow);
 5019 %}
 5020 
 5021 instruct loadSSI(rRegI dst, stackSlotI src)
 5022 %{
 5023   match(Set dst src);
 5024 
 5025   ins_cost(125);
 5026   format %{ "movl    $dst, $src\t# int stk" %}
 5027   ins_encode %{
 5028     __ movl($dst$$Register, $src$$Address);
 5029   %}
 5030   ins_pipe(ialu_reg_mem);
 5031 %}
 5032 
 5033 instruct loadSSL(rRegL dst, stackSlotL src)
 5034 %{
 5035   match(Set dst src);
 5036 
 5037   ins_cost(125);
 5038   format %{ "movq    $dst, $src\t# long stk" %}
 5039   ins_encode %{
 5040     __ movq($dst$$Register, $src$$Address);
 5041   %}
 5042   ins_pipe(ialu_reg_mem);
 5043 %}
 5044 
 5045 instruct loadSSP(rRegP dst, stackSlotP src)
 5046 %{
 5047   match(Set dst src);
 5048 
 5049   ins_cost(125);
 5050   format %{ "movq    $dst, $src\t# ptr stk" %}
 5051   ins_encode %{
 5052     __ movq($dst$$Register, $src$$Address);
 5053   %}
 5054   ins_pipe(ialu_reg_mem);
 5055 %}
 5056 
 5057 instruct loadSSF(regF dst, stackSlotF src)
 5058 %{
 5059   match(Set dst src);
 5060 
 5061   ins_cost(125);
 5062   format %{ "movss   $dst, $src\t# float stk" %}
 5063   ins_encode %{
 5064     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
 5065   %}
 5066   ins_pipe(pipe_slow); // XXX
 5067 %}
 5068 
 5069 // Use the same format since predicate() can not be used here.
 5070 instruct loadSSD(regD dst, stackSlotD src)
 5071 %{
 5072   match(Set dst src);
 5073 
 5074   ins_cost(125);
 5075   format %{ "movsd   $dst, $src\t# double stk" %}
 5076   ins_encode  %{
 5077     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
 5078   %}
 5079   ins_pipe(pipe_slow); // XXX
 5080 %}
 5081 
 5082 // Prefetch instructions for allocation.
 5083 // Must be safe to execute with invalid address (cannot fault).
 5084 
 5085 instruct prefetchAlloc( memory mem ) %{
 5086   predicate(AllocatePrefetchInstr==3);
 5087   match(PrefetchAllocation mem);
 5088   ins_cost(125);
 5089 
 5090   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
 5091   ins_encode %{
 5092     __ prefetchw($mem$$Address);
 5093   %}
 5094   ins_pipe(ialu_mem);
 5095 %}
 5096 
 5097 instruct prefetchAllocNTA( memory mem ) %{
 5098   predicate(AllocatePrefetchInstr==0);
 5099   match(PrefetchAllocation mem);
 5100   ins_cost(125);
 5101 
 5102   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
 5103   ins_encode %{
 5104     __ prefetchnta($mem$$Address);
 5105   %}
 5106   ins_pipe(ialu_mem);
 5107 %}
 5108 
 5109 instruct prefetchAllocT0( memory mem ) %{
 5110   predicate(AllocatePrefetchInstr==1);
 5111   match(PrefetchAllocation mem);
 5112   ins_cost(125);
 5113 
 5114   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
 5115   ins_encode %{
 5116     __ prefetcht0($mem$$Address);
 5117   %}
 5118   ins_pipe(ialu_mem);
 5119 %}
 5120 
 5121 instruct prefetchAllocT2( memory mem ) %{
 5122   predicate(AllocatePrefetchInstr==2);
 5123   match(PrefetchAllocation mem);
 5124   ins_cost(125);
 5125 
 5126   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
 5127   ins_encode %{
 5128     __ prefetcht2($mem$$Address);
 5129   %}
 5130   ins_pipe(ialu_mem);
 5131 %}
 5132 
 5133 //----------Store Instructions-------------------------------------------------
 5134 
 5135 // Store Byte
 5136 instruct storeB(memory mem, rRegI src)
 5137 %{
 5138   match(Set mem (StoreB mem src));
 5139 
 5140   ins_cost(125); // XXX
 5141   format %{ "movb    $mem, $src\t# byte" %}
 5142   ins_encode %{
 5143     __ movb($mem$$Address, $src$$Register);
 5144   %}
 5145   ins_pipe(ialu_mem_reg);
 5146 %}
 5147 
 5148 // Store Char/Short
 5149 instruct storeC(memory mem, rRegI src)
 5150 %{
 5151   match(Set mem (StoreC mem src));
 5152 
 5153   ins_cost(125); // XXX
 5154   format %{ "movw    $mem, $src\t# char/short" %}
 5155   ins_encode %{
 5156     __ movw($mem$$Address, $src$$Register);
 5157   %}
 5158   ins_pipe(ialu_mem_reg);
 5159 %}
 5160 
 5161 // Store Integer
 5162 instruct storeI(memory mem, rRegI src)
 5163 %{
 5164   match(Set mem (StoreI mem src));
 5165 
 5166   ins_cost(125); // XXX
 5167   format %{ "movl    $mem, $src\t# int" %}
 5168   ins_encode %{
 5169     __ movl($mem$$Address, $src$$Register);
 5170   %}
 5171   ins_pipe(ialu_mem_reg);
 5172 %}
 5173 
 5174 // Store Long
 5175 instruct storeL(memory mem, rRegL src)
 5176 %{
 5177   match(Set mem (StoreL mem src));
 5178 
 5179   ins_cost(125); // XXX
 5180   format %{ "movq    $mem, $src\t# long" %}
 5181   ins_encode %{
 5182     __ movq($mem$$Address, $src$$Register);
 5183   %}
 5184   ins_pipe(ialu_mem_reg); // XXX
 5185 %}
 5186 
 5187 // Store Pointer
 5188 instruct storeP(memory mem, any_RegP src)
 5189 %{
 5190   predicate(n->as_Store()->barrier_data() == 0);
 5191   match(Set mem (StoreP mem src));
 5192 
 5193   ins_cost(125); // XXX
 5194   format %{ "movq    $mem, $src\t# ptr" %}
 5195   ins_encode %{
 5196     __ movq($mem$$Address, $src$$Register);
 5197   %}
 5198   ins_pipe(ialu_mem_reg);
 5199 %}
 5200 
 5201 instruct storeImmP0(memory mem, immP0 zero)
 5202 %{
 5203   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) && n->as_Store()->barrier_data() == 0);
 5204   match(Set mem (StoreP mem zero));
 5205 
 5206   ins_cost(125); // XXX
 5207   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
 5208   ins_encode %{
 5209     __ movq($mem$$Address, r12);
 5210   %}
 5211   ins_pipe(ialu_mem_reg);
 5212 %}
 5213 
 5214 // Store Null Pointer, mark word, or other simple pointer constant.
 5215 instruct storeImmP(memory mem, immP31 src)
 5216 %{
 5217   predicate(n->as_Store()->barrier_data() == 0);
 5218   match(Set mem (StoreP mem src));
 5219 
 5220   ins_cost(150); // XXX
 5221   format %{ "movq    $mem, $src\t# ptr" %}
 5222   ins_encode %{
 5223     __ movq($mem$$Address, $src$$constant);
 5224   %}
 5225   ins_pipe(ialu_mem_imm);
 5226 %}
 5227 
 5228 // Store Compressed Pointer
 5229 instruct storeN(memory mem, rRegN src)
 5230 %{
 5231   predicate(n->as_Store()->barrier_data() == 0);
 5232   match(Set mem (StoreN mem src));
 5233 
 5234   ins_cost(125); // XXX
 5235   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5236   ins_encode %{
 5237     __ movl($mem$$Address, $src$$Register);
 5238   %}
 5239   ins_pipe(ialu_mem_reg);
 5240 %}
 5241 
 5242 instruct storeNKlass(memory mem, rRegN src)
 5243 %{
 5244   match(Set mem (StoreNKlass mem src));
 5245 
 5246   ins_cost(125); // XXX
 5247   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5248   ins_encode %{
 5249     __ movl($mem$$Address, $src$$Register);
 5250   %}
 5251   ins_pipe(ialu_mem_reg);
 5252 %}
 5253 
 5254 instruct storeImmN0(memory mem, immN0 zero)
 5255 %{
 5256   predicate(CompressedOops::base() == nullptr && n->as_Store()->barrier_data() == 0);
 5257   match(Set mem (StoreN mem zero));
 5258 
 5259   ins_cost(125); // XXX
 5260   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
 5261   ins_encode %{
 5262     __ movl($mem$$Address, r12);
 5263   %}
 5264   ins_pipe(ialu_mem_reg);
 5265 %}
 5266 
 5267 instruct storeImmN(memory mem, immN src)
 5268 %{
 5269   predicate(n->as_Store()->barrier_data() == 0);
 5270   match(Set mem (StoreN mem src));
 5271 
 5272   ins_cost(150); // XXX
 5273   format %{ "movl    $mem, $src\t# compressed ptr" %}
 5274   ins_encode %{
 5275     address con = (address)$src$$constant;
 5276     if (con == nullptr) {
 5277       __ movl($mem$$Address, 0);
 5278     } else {
 5279       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
 5280     }
 5281   %}
 5282   ins_pipe(ialu_mem_imm);
 5283 %}
 5284 
 5285 instruct storeImmNKlass(memory mem, immNKlass src)
 5286 %{
 5287   match(Set mem (StoreNKlass mem src));
 5288 
 5289   ins_cost(150); // XXX
 5290   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
 5291   ins_encode %{
 5292     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
 5293   %}
 5294   ins_pipe(ialu_mem_imm);
 5295 %}
 5296 
 5297 // Store Integer Immediate
 5298 instruct storeImmI0(memory mem, immI_0 zero)
 5299 %{
 5300   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5301   match(Set mem (StoreI mem zero));
 5302 
 5303   ins_cost(125); // XXX
 5304   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
 5305   ins_encode %{
 5306     __ movl($mem$$Address, r12);
 5307   %}
 5308   ins_pipe(ialu_mem_reg);
 5309 %}
 5310 
 5311 instruct storeImmI(memory mem, immI src)
 5312 %{
 5313   match(Set mem (StoreI mem src));
 5314 
 5315   ins_cost(150);
 5316   format %{ "movl    $mem, $src\t# int" %}
 5317   ins_encode %{
 5318     __ movl($mem$$Address, $src$$constant);
 5319   %}
 5320   ins_pipe(ialu_mem_imm);
 5321 %}
 5322 
 5323 // Store Long Immediate
 5324 instruct storeImmL0(memory mem, immL0 zero)
 5325 %{
 5326   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5327   match(Set mem (StoreL mem zero));
 5328 
 5329   ins_cost(125); // XXX
 5330   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
 5331   ins_encode %{
 5332     __ movq($mem$$Address, r12);
 5333   %}
 5334   ins_pipe(ialu_mem_reg);
 5335 %}
 5336 
 5337 instruct storeImmL(memory mem, immL32 src)
 5338 %{
 5339   match(Set mem (StoreL mem src));
 5340 
 5341   ins_cost(150);
 5342   format %{ "movq    $mem, $src\t# long" %}
 5343   ins_encode %{
 5344     __ movq($mem$$Address, $src$$constant);
 5345   %}
 5346   ins_pipe(ialu_mem_imm);
 5347 %}
 5348 
 5349 // Store Short/Char Immediate
 5350 instruct storeImmC0(memory mem, immI_0 zero)
 5351 %{
 5352   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5353   match(Set mem (StoreC mem zero));
 5354 
 5355   ins_cost(125); // XXX
 5356   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5357   ins_encode %{
 5358     __ movw($mem$$Address, r12);
 5359   %}
 5360   ins_pipe(ialu_mem_reg);
 5361 %}
 5362 
 5363 instruct storeImmI16(memory mem, immI16 src)
 5364 %{
 5365   predicate(UseStoreImmI16);
 5366   match(Set mem (StoreC mem src));
 5367 
 5368   ins_cost(150);
 5369   format %{ "movw    $mem, $src\t# short/char" %}
 5370   ins_encode %{
 5371     __ movw($mem$$Address, $src$$constant);
 5372   %}
 5373   ins_pipe(ialu_mem_imm);
 5374 %}
 5375 
 5376 // Store Byte Immediate
 5377 instruct storeImmB0(memory mem, immI_0 zero)
 5378 %{
 5379   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5380   match(Set mem (StoreB mem zero));
 5381 
 5382   ins_cost(125); // XXX
 5383   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
 5384   ins_encode %{
 5385     __ movb($mem$$Address, r12);
 5386   %}
 5387   ins_pipe(ialu_mem_reg);
 5388 %}
 5389 
 5390 instruct storeImmB(memory mem, immI8 src)
 5391 %{
 5392   match(Set mem (StoreB mem src));
 5393 
 5394   ins_cost(150); // XXX
 5395   format %{ "movb    $mem, $src\t# byte" %}
 5396   ins_encode %{
 5397     __ movb($mem$$Address, $src$$constant);
 5398   %}
 5399   ins_pipe(ialu_mem_imm);
 5400 %}
 5401 
 5402 // Store Float
 5403 instruct storeF(memory mem, regF src)
 5404 %{
 5405   match(Set mem (StoreF mem src));
 5406 
 5407   ins_cost(95); // XXX
 5408   format %{ "movss   $mem, $src\t# float" %}
 5409   ins_encode %{
 5410     __ movflt($mem$$Address, $src$$XMMRegister);
 5411   %}
 5412   ins_pipe(pipe_slow); // XXX
 5413 %}
 5414 
 5415 // Store immediate Float value (it is faster than store from XMM register)
 5416 instruct storeF0(memory mem, immF0 zero)
 5417 %{
 5418   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5419   match(Set mem (StoreF mem zero));
 5420 
 5421   ins_cost(25); // XXX
 5422   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
 5423   ins_encode %{
 5424     __ movl($mem$$Address, r12);
 5425   %}
 5426   ins_pipe(ialu_mem_reg);
 5427 %}
 5428 
 5429 instruct storeF_imm(memory mem, immF src)
 5430 %{
 5431   match(Set mem (StoreF mem src));
 5432 
 5433   ins_cost(50);
 5434   format %{ "movl    $mem, $src\t# float" %}
 5435   ins_encode %{
 5436     __ movl($mem$$Address, jint_cast($src$$constant));
 5437   %}
 5438   ins_pipe(ialu_mem_imm);
 5439 %}
 5440 
 5441 // Store Double
 5442 instruct storeD(memory mem, regD src)
 5443 %{
 5444   match(Set mem (StoreD mem src));
 5445 
 5446   ins_cost(95); // XXX
 5447   format %{ "movsd   $mem, $src\t# double" %}
 5448   ins_encode %{
 5449     __ movdbl($mem$$Address, $src$$XMMRegister);
 5450   %}
 5451   ins_pipe(pipe_slow); // XXX
 5452 %}
 5453 
 5454 // Store immediate double 0.0 (it is faster than store from XMM register)
 5455 instruct storeD0_imm(memory mem, immD0 src)
 5456 %{
 5457   predicate(!UseCompressedOops || (CompressedOops::base() != nullptr));
 5458   match(Set mem (StoreD mem src));
 5459 
 5460   ins_cost(50);
 5461   format %{ "movq    $mem, $src\t# double 0." %}
 5462   ins_encode %{
 5463     __ movq($mem$$Address, $src$$constant);
 5464   %}
 5465   ins_pipe(ialu_mem_imm);
 5466 %}
 5467 
 5468 instruct storeD0(memory mem, immD0 zero)
 5469 %{
 5470   predicate(UseCompressedOops && (CompressedOops::base() == nullptr));
 5471   match(Set mem (StoreD mem zero));
 5472 
 5473   ins_cost(25); // XXX
 5474   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
 5475   ins_encode %{
 5476     __ movq($mem$$Address, r12);
 5477   %}
 5478   ins_pipe(ialu_mem_reg);
 5479 %}
 5480 
 5481 instruct storeSSI(stackSlotI dst, rRegI src)
 5482 %{
 5483   match(Set dst src);
 5484 
 5485   ins_cost(100);
 5486   format %{ "movl    $dst, $src\t# int stk" %}
 5487   ins_encode %{
 5488     __ movl($dst$$Address, $src$$Register);
 5489   %}
 5490   ins_pipe( ialu_mem_reg );
 5491 %}
 5492 
 5493 instruct storeSSL(stackSlotL dst, rRegL src)
 5494 %{
 5495   match(Set dst src);
 5496 
 5497   ins_cost(100);
 5498   format %{ "movq    $dst, $src\t# long stk" %}
 5499   ins_encode %{
 5500     __ movq($dst$$Address, $src$$Register);
 5501   %}
 5502   ins_pipe(ialu_mem_reg);
 5503 %}
 5504 
 5505 instruct storeSSP(stackSlotP dst, rRegP src)
 5506 %{
 5507   match(Set dst src);
 5508 
 5509   ins_cost(100);
 5510   format %{ "movq    $dst, $src\t# ptr stk" %}
 5511   ins_encode %{
 5512     __ movq($dst$$Address, $src$$Register);
 5513   %}
 5514   ins_pipe(ialu_mem_reg);
 5515 %}
 5516 
 5517 instruct storeSSF(stackSlotF dst, regF src)
 5518 %{
 5519   match(Set dst src);
 5520 
 5521   ins_cost(95); // XXX
 5522   format %{ "movss   $dst, $src\t# float stk" %}
 5523   ins_encode %{
 5524     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5525   %}
 5526   ins_pipe(pipe_slow); // XXX
 5527 %}
 5528 
 5529 instruct storeSSD(stackSlotD dst, regD src)
 5530 %{
 5531   match(Set dst src);
 5532 
 5533   ins_cost(95); // XXX
 5534   format %{ "movsd   $dst, $src\t# double stk" %}
 5535   ins_encode %{
 5536     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
 5537   %}
 5538   ins_pipe(pipe_slow); // XXX
 5539 %}
 5540 
 5541 instruct cacheWB(indirect addr)
 5542 %{
 5543   predicate(VM_Version::supports_data_cache_line_flush());
 5544   match(CacheWB addr);
 5545 
 5546   ins_cost(100);
 5547   format %{"cache wb $addr" %}
 5548   ins_encode %{
 5549     assert($addr->index_position() < 0, "should be");
 5550     assert($addr$$disp == 0, "should be");
 5551     __ cache_wb(Address($addr$$base$$Register, 0));
 5552   %}
 5553   ins_pipe(pipe_slow); // XXX
 5554 %}
 5555 
 5556 instruct cacheWBPreSync()
 5557 %{
 5558   predicate(VM_Version::supports_data_cache_line_flush());
 5559   match(CacheWBPreSync);
 5560 
 5561   ins_cost(100);
 5562   format %{"cache wb presync" %}
 5563   ins_encode %{
 5564     __ cache_wbsync(true);
 5565   %}
 5566   ins_pipe(pipe_slow); // XXX
 5567 %}
 5568 
 5569 instruct cacheWBPostSync()
 5570 %{
 5571   predicate(VM_Version::supports_data_cache_line_flush());
 5572   match(CacheWBPostSync);
 5573 
 5574   ins_cost(100);
 5575   format %{"cache wb postsync" %}
 5576   ins_encode %{
 5577     __ cache_wbsync(false);
 5578   %}
 5579   ins_pipe(pipe_slow); // XXX
 5580 %}
 5581 
 5582 //----------BSWAP Instructions-------------------------------------------------
 5583 instruct bytes_reverse_int(rRegI dst) %{
 5584   match(Set dst (ReverseBytesI dst));
 5585 
 5586   format %{ "bswapl  $dst" %}
 5587   ins_encode %{
 5588     __ bswapl($dst$$Register);
 5589   %}
 5590   ins_pipe( ialu_reg );
 5591 %}
 5592 
 5593 instruct bytes_reverse_long(rRegL dst) %{
 5594   match(Set dst (ReverseBytesL dst));
 5595 
 5596   format %{ "bswapq  $dst" %}
 5597   ins_encode %{
 5598     __ bswapq($dst$$Register);
 5599   %}
 5600   ins_pipe( ialu_reg);
 5601 %}
 5602 
 5603 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
 5604   match(Set dst (ReverseBytesUS dst));
 5605   effect(KILL cr);
 5606 
 5607   format %{ "bswapl  $dst\n\t"
 5608             "shrl    $dst,16\n\t" %}
 5609   ins_encode %{
 5610     __ bswapl($dst$$Register);
 5611     __ shrl($dst$$Register, 16);
 5612   %}
 5613   ins_pipe( ialu_reg );
 5614 %}
 5615 
 5616 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
 5617   match(Set dst (ReverseBytesS dst));
 5618   effect(KILL cr);
 5619 
 5620   format %{ "bswapl  $dst\n\t"
 5621             "sar     $dst,16\n\t" %}
 5622   ins_encode %{
 5623     __ bswapl($dst$$Register);
 5624     __ sarl($dst$$Register, 16);
 5625   %}
 5626   ins_pipe( ialu_reg );
 5627 %}
 5628 
 5629 //---------- Zeros Count Instructions ------------------------------------------
 5630 
 5631 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5632   predicate(UseCountLeadingZerosInstruction);
 5633   match(Set dst (CountLeadingZerosI src));
 5634   effect(KILL cr);
 5635 
 5636   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5637   ins_encode %{
 5638     __ lzcntl($dst$$Register, $src$$Register);
 5639   %}
 5640   ins_pipe(ialu_reg);
 5641 %}
 5642 
 5643 instruct countLeadingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5644   predicate(UseCountLeadingZerosInstruction);
 5645   match(Set dst (CountLeadingZerosI (LoadI src)));
 5646   effect(KILL cr);
 5647   ins_cost(175);
 5648   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
 5649   ins_encode %{
 5650     __ lzcntl($dst$$Register, $src$$Address);
 5651   %}
 5652   ins_pipe(ialu_reg_mem);
 5653 %}
 5654 
 5655 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
 5656   predicate(!UseCountLeadingZerosInstruction);
 5657   match(Set dst (CountLeadingZerosI src));
 5658   effect(KILL cr);
 5659 
 5660   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
 5661             "jnz     skip\n\t"
 5662             "movl    $dst, -1\n"
 5663       "skip:\n\t"
 5664             "negl    $dst\n\t"
 5665             "addl    $dst, 31" %}
 5666   ins_encode %{
 5667     Register Rdst = $dst$$Register;
 5668     Register Rsrc = $src$$Register;
 5669     Label skip;
 5670     __ bsrl(Rdst, Rsrc);
 5671     __ jccb(Assembler::notZero, skip);
 5672     __ movl(Rdst, -1);
 5673     __ bind(skip);
 5674     __ negl(Rdst);
 5675     __ addl(Rdst, BitsPerInt - 1);
 5676   %}
 5677   ins_pipe(ialu_reg);
 5678 %}
 5679 
 5680 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5681   predicate(UseCountLeadingZerosInstruction);
 5682   match(Set dst (CountLeadingZerosL src));
 5683   effect(KILL cr);
 5684 
 5685   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5686   ins_encode %{
 5687     __ lzcntq($dst$$Register, $src$$Register);
 5688   %}
 5689   ins_pipe(ialu_reg);
 5690 %}
 5691 
 5692 instruct countLeadingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5693   predicate(UseCountLeadingZerosInstruction);
 5694   match(Set dst (CountLeadingZerosL (LoadL src)));
 5695   effect(KILL cr);
 5696   ins_cost(175);
 5697   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
 5698   ins_encode %{
 5699     __ lzcntq($dst$$Register, $src$$Address);
 5700   %}
 5701   ins_pipe(ialu_reg_mem);
 5702 %}
 5703 
 5704 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
 5705   predicate(!UseCountLeadingZerosInstruction);
 5706   match(Set dst (CountLeadingZerosL src));
 5707   effect(KILL cr);
 5708 
 5709   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
 5710             "jnz     skip\n\t"
 5711             "movl    $dst, -1\n"
 5712       "skip:\n\t"
 5713             "negl    $dst\n\t"
 5714             "addl    $dst, 63" %}
 5715   ins_encode %{
 5716     Register Rdst = $dst$$Register;
 5717     Register Rsrc = $src$$Register;
 5718     Label skip;
 5719     __ bsrq(Rdst, Rsrc);
 5720     __ jccb(Assembler::notZero, skip);
 5721     __ movl(Rdst, -1);
 5722     __ bind(skip);
 5723     __ negl(Rdst);
 5724     __ addl(Rdst, BitsPerLong - 1);
 5725   %}
 5726   ins_pipe(ialu_reg);
 5727 %}
 5728 
 5729 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5730   predicate(UseCountTrailingZerosInstruction);
 5731   match(Set dst (CountTrailingZerosI src));
 5732   effect(KILL cr);
 5733 
 5734   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5735   ins_encode %{
 5736     __ tzcntl($dst$$Register, $src$$Register);
 5737   %}
 5738   ins_pipe(ialu_reg);
 5739 %}
 5740 
 5741 instruct countTrailingZerosI_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5742   predicate(UseCountTrailingZerosInstruction);
 5743   match(Set dst (CountTrailingZerosI (LoadI src)));
 5744   effect(KILL cr);
 5745   ins_cost(175);
 5746   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
 5747   ins_encode %{
 5748     __ tzcntl($dst$$Register, $src$$Address);
 5749   %}
 5750   ins_pipe(ialu_reg_mem);
 5751 %}
 5752 
 5753 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
 5754   predicate(!UseCountTrailingZerosInstruction);
 5755   match(Set dst (CountTrailingZerosI src));
 5756   effect(KILL cr);
 5757 
 5758   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
 5759             "jnz     done\n\t"
 5760             "movl    $dst, 32\n"
 5761       "done:" %}
 5762   ins_encode %{
 5763     Register Rdst = $dst$$Register;
 5764     Label done;
 5765     __ bsfl(Rdst, $src$$Register);
 5766     __ jccb(Assembler::notZero, done);
 5767     __ movl(Rdst, BitsPerInt);
 5768     __ bind(done);
 5769   %}
 5770   ins_pipe(ialu_reg);
 5771 %}
 5772 
 5773 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5774   predicate(UseCountTrailingZerosInstruction);
 5775   match(Set dst (CountTrailingZerosL src));
 5776   effect(KILL cr);
 5777 
 5778   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5779   ins_encode %{
 5780     __ tzcntq($dst$$Register, $src$$Register);
 5781   %}
 5782   ins_pipe(ialu_reg);
 5783 %}
 5784 
 5785 instruct countTrailingZerosL_mem(rRegI dst, memory src, rFlagsReg cr) %{
 5786   predicate(UseCountTrailingZerosInstruction);
 5787   match(Set dst (CountTrailingZerosL (LoadL src)));
 5788   effect(KILL cr);
 5789   ins_cost(175);
 5790   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
 5791   ins_encode %{
 5792     __ tzcntq($dst$$Register, $src$$Address);
 5793   %}
 5794   ins_pipe(ialu_reg_mem);
 5795 %}
 5796 
 5797 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
 5798   predicate(!UseCountTrailingZerosInstruction);
 5799   match(Set dst (CountTrailingZerosL src));
 5800   effect(KILL cr);
 5801 
 5802   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
 5803             "jnz     done\n\t"
 5804             "movl    $dst, 64\n"
 5805       "done:" %}
 5806   ins_encode %{
 5807     Register Rdst = $dst$$Register;
 5808     Label done;
 5809     __ bsfq(Rdst, $src$$Register);
 5810     __ jccb(Assembler::notZero, done);
 5811     __ movl(Rdst, BitsPerLong);
 5812     __ bind(done);
 5813   %}
 5814   ins_pipe(ialu_reg);
 5815 %}
 5816 
 5817 //--------------- Reverse Operation Instructions ----------------
 5818 instruct bytes_reversebit_int(rRegI dst, rRegI src, rRegI rtmp, rFlagsReg cr) %{
 5819   predicate(!VM_Version::supports_gfni());
 5820   match(Set dst (ReverseI src));
 5821   effect(TEMP dst, TEMP rtmp, KILL cr);
 5822   format %{ "reverse_int $dst $src\t! using $rtmp as TEMP" %}
 5823   ins_encode %{
 5824     __ reverseI($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp$$Register);
 5825   %}
 5826   ins_pipe( ialu_reg );
 5827 %}
 5828 
 5829 instruct bytes_reversebit_int_gfni(rRegI dst, rRegI src, vlRegF xtmp1, vlRegF xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5830   predicate(VM_Version::supports_gfni());
 5831   match(Set dst (ReverseI src));
 5832   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5833   format %{ "reverse_int $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5834   ins_encode %{
 5835     __ reverseI($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register);
 5836   %}
 5837   ins_pipe( ialu_reg );
 5838 %}
 5839 
 5840 instruct bytes_reversebit_long(rRegL dst, rRegL src, rRegL rtmp1, rRegL rtmp2, rFlagsReg cr) %{
 5841   predicate(!VM_Version::supports_gfni());
 5842   match(Set dst (ReverseL src));
 5843   effect(TEMP dst, TEMP rtmp1, TEMP rtmp2, KILL cr);
 5844   format %{ "reverse_long $dst $src\t! using $rtmp1 and $rtmp2 as TEMP" %}
 5845   ins_encode %{
 5846     __ reverseL($dst$$Register, $src$$Register, xnoreg, xnoreg, $rtmp1$$Register, $rtmp2$$Register);
 5847   %}
 5848   ins_pipe( ialu_reg );
 5849 %}
 5850 
 5851 instruct bytes_reversebit_long_gfni(rRegL dst, rRegL src, vlRegD xtmp1, vlRegD xtmp2, rRegL rtmp, rFlagsReg cr) %{
 5852   predicate(VM_Version::supports_gfni());
 5853   match(Set dst (ReverseL src));
 5854   effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP rtmp, KILL cr);
 5855   format %{ "reverse_long $dst $src\t! using $rtmp, $xtmp1 and $xtmp2 as TEMP" %}
 5856   ins_encode %{
 5857     __ reverseL($dst$$Register, $src$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $rtmp$$Register, noreg);
 5858   %}
 5859   ins_pipe( ialu_reg );
 5860 %}
 5861 
 5862 //---------- Population Count Instructions -------------------------------------
 5863 
 5864 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
 5865   predicate(UsePopCountInstruction);
 5866   match(Set dst (PopCountI src));
 5867   effect(KILL cr);
 5868 
 5869   format %{ "popcnt  $dst, $src" %}
 5870   ins_encode %{
 5871     __ popcntl($dst$$Register, $src$$Register);
 5872   %}
 5873   ins_pipe(ialu_reg);
 5874 %}
 5875 
 5876 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5877   predicate(UsePopCountInstruction);
 5878   match(Set dst (PopCountI (LoadI mem)));
 5879   effect(KILL cr);
 5880 
 5881   format %{ "popcnt  $dst, $mem" %}
 5882   ins_encode %{
 5883     __ popcntl($dst$$Register, $mem$$Address);
 5884   %}
 5885   ins_pipe(ialu_reg);
 5886 %}
 5887 
 5888 // Note: Long.bitCount(long) returns an int.
 5889 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
 5890   predicate(UsePopCountInstruction);
 5891   match(Set dst (PopCountL src));
 5892   effect(KILL cr);
 5893 
 5894   format %{ "popcnt  $dst, $src" %}
 5895   ins_encode %{
 5896     __ popcntq($dst$$Register, $src$$Register);
 5897   %}
 5898   ins_pipe(ialu_reg);
 5899 %}
 5900 
 5901 // Note: Long.bitCount(long) returns an int.
 5902 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
 5903   predicate(UsePopCountInstruction);
 5904   match(Set dst (PopCountL (LoadL mem)));
 5905   effect(KILL cr);
 5906 
 5907   format %{ "popcnt  $dst, $mem" %}
 5908   ins_encode %{
 5909     __ popcntq($dst$$Register, $mem$$Address);
 5910   %}
 5911   ins_pipe(ialu_reg);
 5912 %}
 5913 
 5914 
 5915 //----------MemBar Instructions-----------------------------------------------
 5916 // Memory barrier flavors
 5917 
 5918 instruct membar_acquire()
 5919 %{
 5920   match(MemBarAcquire);
 5921   match(LoadFence);
 5922   ins_cost(0);
 5923 
 5924   size(0);
 5925   format %{ "MEMBAR-acquire ! (empty encoding)" %}
 5926   ins_encode();
 5927   ins_pipe(empty);
 5928 %}
 5929 
 5930 instruct membar_acquire_lock()
 5931 %{
 5932   match(MemBarAcquireLock);
 5933   ins_cost(0);
 5934 
 5935   size(0);
 5936   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
 5937   ins_encode();
 5938   ins_pipe(empty);
 5939 %}
 5940 
 5941 instruct membar_release()
 5942 %{
 5943   match(MemBarRelease);
 5944   match(StoreFence);
 5945   ins_cost(0);
 5946 
 5947   size(0);
 5948   format %{ "MEMBAR-release ! (empty encoding)" %}
 5949   ins_encode();
 5950   ins_pipe(empty);
 5951 %}
 5952 
 5953 instruct membar_release_lock()
 5954 %{
 5955   match(MemBarReleaseLock);
 5956   ins_cost(0);
 5957 
 5958   size(0);
 5959   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
 5960   ins_encode();
 5961   ins_pipe(empty);
 5962 %}
 5963 
 5964 instruct membar_volatile(rFlagsReg cr) %{
 5965   match(MemBarVolatile);
 5966   effect(KILL cr);
 5967   ins_cost(400);
 5968 
 5969   format %{
 5970     $$template
 5971     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
 5972   %}
 5973   ins_encode %{
 5974     __ membar(Assembler::StoreLoad);
 5975   %}
 5976   ins_pipe(pipe_slow);
 5977 %}
 5978 
 5979 instruct unnecessary_membar_volatile()
 5980 %{
 5981   match(MemBarVolatile);
 5982   predicate(Matcher::post_store_load_barrier(n));
 5983   ins_cost(0);
 5984 
 5985   size(0);
 5986   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 5987   ins_encode();
 5988   ins_pipe(empty);
 5989 %}
 5990 
 5991 instruct membar_storestore() %{
 5992   match(MemBarStoreStore);
 5993   match(StoreStoreFence);
 5994   ins_cost(0);
 5995 
 5996   size(0);
 5997   format %{ "MEMBAR-storestore (empty encoding)" %}
 5998   ins_encode( );
 5999   ins_pipe(empty);
 6000 %}
 6001 
 6002 //----------Move Instructions--------------------------------------------------
 6003 
 6004 instruct castX2P(rRegP dst, rRegL src)
 6005 %{
 6006   match(Set dst (CastX2P src));
 6007 
 6008   format %{ "movq    $dst, $src\t# long->ptr" %}
 6009   ins_encode %{
 6010     if ($dst$$reg != $src$$reg) {
 6011       __ movptr($dst$$Register, $src$$Register);
 6012     }
 6013   %}
 6014   ins_pipe(ialu_reg_reg); // XXX
 6015 %}
 6016 
 6017 instruct castP2X(rRegL dst, rRegP src)
 6018 %{
 6019   match(Set dst (CastP2X src));
 6020 
 6021   format %{ "movq    $dst, $src\t# ptr -> long" %}
 6022   ins_encode %{
 6023     if ($dst$$reg != $src$$reg) {
 6024       __ movptr($dst$$Register, $src$$Register);
 6025     }
 6026   %}
 6027   ins_pipe(ialu_reg_reg); // XXX
 6028 %}
 6029 
 6030 // Convert oop into int for vectors alignment masking
 6031 instruct convP2I(rRegI dst, rRegP src)
 6032 %{
 6033   match(Set dst (ConvL2I (CastP2X src)));
 6034 
 6035   format %{ "movl    $dst, $src\t# ptr -> int" %}
 6036   ins_encode %{
 6037     __ movl($dst$$Register, $src$$Register);
 6038   %}
 6039   ins_pipe(ialu_reg_reg); // XXX
 6040 %}
 6041 
 6042 // Convert compressed oop into int for vectors alignment masking
 6043 // in case of 32bit oops (heap < 4Gb).
 6044 instruct convN2I(rRegI dst, rRegN src)
 6045 %{
 6046   predicate(CompressedOops::shift() == 0);
 6047   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
 6048 
 6049   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
 6050   ins_encode %{
 6051     __ movl($dst$$Register, $src$$Register);
 6052   %}
 6053   ins_pipe(ialu_reg_reg); // XXX
 6054 %}
 6055 
 6056 // Convert oop pointer into compressed form
 6057 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
 6058   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
 6059   match(Set dst (EncodeP src));
 6060   effect(KILL cr);
 6061   format %{ "encode_heap_oop $dst,$src" %}
 6062   ins_encode %{
 6063     Register s = $src$$Register;
 6064     Register d = $dst$$Register;
 6065     if (s != d) {
 6066       __ movq(d, s);
 6067     }
 6068     __ encode_heap_oop(d);
 6069   %}
 6070   ins_pipe(ialu_reg_long);
 6071 %}
 6072 
 6073 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6074   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
 6075   match(Set dst (EncodeP src));
 6076   effect(KILL cr);
 6077   format %{ "encode_heap_oop_not_null $dst,$src" %}
 6078   ins_encode %{
 6079     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
 6080   %}
 6081   ins_pipe(ialu_reg_long);
 6082 %}
 6083 
 6084 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
 6085   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
 6086             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
 6087   match(Set dst (DecodeN src));
 6088   effect(KILL cr);
 6089   format %{ "decode_heap_oop $dst,$src" %}
 6090   ins_encode %{
 6091     Register s = $src$$Register;
 6092     Register d = $dst$$Register;
 6093     if (s != d) {
 6094       __ movq(d, s);
 6095     }
 6096     __ decode_heap_oop(d);
 6097   %}
 6098   ins_pipe(ialu_reg_long);
 6099 %}
 6100 
 6101 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6102   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
 6103             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
 6104   match(Set dst (DecodeN src));
 6105   effect(KILL cr);
 6106   format %{ "decode_heap_oop_not_null $dst,$src" %}
 6107   ins_encode %{
 6108     Register s = $src$$Register;
 6109     Register d = $dst$$Register;
 6110     if (s != d) {
 6111       __ decode_heap_oop_not_null(d, s);
 6112     } else {
 6113       __ decode_heap_oop_not_null(d);
 6114     }
 6115   %}
 6116   ins_pipe(ialu_reg_long);
 6117 %}
 6118 
 6119 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
 6120   match(Set dst (EncodePKlass src));
 6121   effect(TEMP dst, KILL cr);
 6122   format %{ "encode_and_move_klass_not_null $dst,$src" %}
 6123   ins_encode %{
 6124     __ encode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6125   %}
 6126   ins_pipe(ialu_reg_long);
 6127 %}
 6128 
 6129 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
 6130   match(Set dst (DecodeNKlass src));
 6131   effect(TEMP dst, KILL cr);
 6132   format %{ "decode_and_move_klass_not_null $dst,$src" %}
 6133   ins_encode %{
 6134     __ decode_and_move_klass_not_null($dst$$Register, $src$$Register);
 6135   %}
 6136   ins_pipe(ialu_reg_long);
 6137 %}
 6138 
 6139 //----------Conditional Move---------------------------------------------------
 6140 // Jump
 6141 // dummy instruction for generating temp registers
 6142 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
 6143   match(Jump (LShiftL switch_val shift));
 6144   ins_cost(350);
 6145   predicate(false);
 6146   effect(TEMP dest);
 6147 
 6148   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6149             "jmp     [$dest + $switch_val << $shift]\n\t" %}
 6150   ins_encode %{
 6151     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6152     // to do that and the compiler is using that register as one it can allocate.
 6153     // So we build it all by hand.
 6154     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
 6155     // ArrayAddress dispatch(table, index);
 6156     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
 6157     __ lea($dest$$Register, $constantaddress);
 6158     __ jmp(dispatch);
 6159   %}
 6160   ins_pipe(pipe_jmp);
 6161 %}
 6162 
 6163 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
 6164   match(Jump (AddL (LShiftL switch_val shift) offset));
 6165   ins_cost(350);
 6166   effect(TEMP dest);
 6167 
 6168   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6169             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
 6170   ins_encode %{
 6171     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6172     // to do that and the compiler is using that register as one it can allocate.
 6173     // So we build it all by hand.
 6174     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6175     // ArrayAddress dispatch(table, index);
 6176     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
 6177     __ lea($dest$$Register, $constantaddress);
 6178     __ jmp(dispatch);
 6179   %}
 6180   ins_pipe(pipe_jmp);
 6181 %}
 6182 
 6183 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
 6184   match(Jump switch_val);
 6185   ins_cost(350);
 6186   effect(TEMP dest);
 6187 
 6188   format %{ "leaq    $dest, [$constantaddress]\n\t"
 6189             "jmp     [$dest + $switch_val]\n\t" %}
 6190   ins_encode %{
 6191     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
 6192     // to do that and the compiler is using that register as one it can allocate.
 6193     // So we build it all by hand.
 6194     // Address index(noreg, switch_reg, Address::times_1);
 6195     // ArrayAddress dispatch(table, index);
 6196     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
 6197     __ lea($dest$$Register, $constantaddress);
 6198     __ jmp(dispatch);
 6199   %}
 6200   ins_pipe(pipe_jmp);
 6201 %}
 6202 
 6203 // Conditional move
 6204 instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
 6205 %{
 6206   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6207   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6208 
 6209   ins_cost(100); // XXX
 6210   format %{ "setbn$cop $dst\t# signed, int" %}
 6211   ins_encode %{
 6212     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6213     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6214   %}
 6215   ins_pipe(ialu_reg);
 6216 %}
 6217 
 6218 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
 6219 %{
 6220   predicate(!UseAPX);
 6221   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6222 
 6223   ins_cost(200); // XXX
 6224   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6225   ins_encode %{
 6226     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6227   %}
 6228   ins_pipe(pipe_cmov_reg);
 6229 %}
 6230 
 6231 instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
 6232 %{
 6233   predicate(UseAPX);
 6234   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6235 
 6236   ins_cost(200);
 6237   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6238   ins_encode %{
 6239     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6240   %}
 6241   ins_pipe(pipe_cmov_reg);
 6242 %}
 6243 
 6244 instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
 6245 %{
 6246   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6247   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6248 
 6249   ins_cost(100); // XXX
 6250   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6251   ins_encode %{
 6252     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6253     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6254   %}
 6255   ins_pipe(ialu_reg);
 6256 %}
 6257 
 6258 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
 6259   predicate(!UseAPX);
 6260   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6261 
 6262   ins_cost(200); // XXX
 6263   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6264   ins_encode %{
 6265     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6266   %}
 6267   ins_pipe(pipe_cmov_reg);
 6268 %}
 6269 
 6270 instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
 6271   predicate(UseAPX);
 6272   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6273 
 6274   ins_cost(200);
 6275   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6276   ins_encode %{
 6277     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6278   %}
 6279   ins_pipe(pipe_cmov_reg);
 6280 %}
 6281 
 6282 instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6283 %{
 6284   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
 6285   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6286 
 6287   ins_cost(100); // XXX
 6288   format %{ "setbn$cop $dst\t# unsigned, int" %}
 6289   ins_encode %{
 6290     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6291     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6292   %}
 6293   ins_pipe(ialu_reg);
 6294 %}
 6295 
 6296 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6297   predicate(!UseAPX);
 6298   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6299   ins_cost(200);
 6300   expand %{
 6301     cmovI_regU(cop, cr, dst, src);
 6302   %}
 6303 %}
 6304 
 6305 instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
 6306   predicate(UseAPX);
 6307   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6308   ins_cost(200);
 6309   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6310   ins_encode %{
 6311     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6312   %}
 6313   ins_pipe(pipe_cmov_reg);
 6314 %}
 6315 
 6316 instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6317   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6318   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
 6319 
 6320   ins_cost(200); // XXX
 6321   format %{ "cmovpl  $dst, $src\n\t"
 6322             "cmovnel $dst, $src" %}
 6323   ins_encode %{
 6324     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6325     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6326   %}
 6327   ins_pipe(pipe_cmov_reg);
 6328 %}
 6329 
 6330 instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6331   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6332   match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
 6333   effect(TEMP dst);
 6334 
 6335   ins_cost(200);
 6336   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6337             "cmovnel  $dst, $src2" %}
 6338   ins_encode %{
 6339     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6340     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6341   %}
 6342   ins_pipe(pipe_cmov_reg);
 6343 %}
 6344 
 6345 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6346 // inputs of the CMove
 6347 instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
 6348   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6349   match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
 6350   effect(TEMP dst);
 6351 
 6352   ins_cost(200); // XXX
 6353   format %{ "cmovpl  $dst, $src\n\t"
 6354             "cmovnel $dst, $src" %}
 6355   ins_encode %{
 6356     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6357     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6358   %}
 6359   ins_pipe(pipe_cmov_reg);
 6360 %}
 6361 
 6362 // We need this special handling for only eq / neq comparison since NaN == NaN is false,
 6363 // and parity flag bit is set if any of the operand is a NaN.
 6364 instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
 6365   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6366   match(Set dst (CMoveI (Binary cop cr) (Binary src2 src1)));
 6367   effect(TEMP dst);
 6368 
 6369   ins_cost(200);
 6370   format %{ "ecmovpl  $dst, $src1, $src2\n\t"
 6371             "cmovnel  $dst, $src2" %}
 6372   ins_encode %{
 6373     __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6374     __ cmovl(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6375   %}
 6376   ins_pipe(pipe_cmov_reg);
 6377 %}
 6378 
 6379 // Conditional move
 6380 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
 6381   predicate(!UseAPX);
 6382   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6383 
 6384   ins_cost(250); // XXX
 6385   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
 6386   ins_encode %{
 6387     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6388   %}
 6389   ins_pipe(pipe_cmov_mem);
 6390 %}
 6391 
 6392 // Conditional move
 6393 instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
 6394 %{
 6395   predicate(UseAPX);
 6396   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6397 
 6398   ins_cost(250);
 6399   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
 6400   ins_encode %{
 6401     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6402   %}
 6403   ins_pipe(pipe_cmov_mem);
 6404 %}
 6405 
 6406 // Conditional move
 6407 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
 6408 %{
 6409   predicate(!UseAPX);
 6410   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6411 
 6412   ins_cost(250); // XXX
 6413   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
 6414   ins_encode %{
 6415     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6416   %}
 6417   ins_pipe(pipe_cmov_mem);
 6418 %}
 6419 
 6420 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
 6421   predicate(!UseAPX);
 6422   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
 6423   ins_cost(250);
 6424   expand %{
 6425     cmovI_memU(cop, cr, dst, src);
 6426   %}
 6427 %}
 6428 
 6429 instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
 6430 %{
 6431   predicate(UseAPX);
 6432   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6433 
 6434   ins_cost(250);
 6435   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6436   ins_encode %{
 6437     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6438   %}
 6439   ins_pipe(pipe_cmov_mem);
 6440 %}
 6441 
 6442 instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
 6443 %{
 6444   predicate(UseAPX);
 6445   match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
 6446   ins_cost(250);
 6447   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
 6448   ins_encode %{
 6449     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6450   %}
 6451   ins_pipe(pipe_cmov_mem);
 6452 %}
 6453 
 6454 // Conditional move
 6455 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
 6456 %{
 6457   predicate(!UseAPX);
 6458   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6459 
 6460   ins_cost(200); // XXX
 6461   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
 6462   ins_encode %{
 6463     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6464   %}
 6465   ins_pipe(pipe_cmov_reg);
 6466 %}
 6467 
 6468 // Conditional move ndd
 6469 instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
 6470 %{
 6471   predicate(UseAPX);
 6472   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6473 
 6474   ins_cost(200);
 6475   format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
 6476   ins_encode %{
 6477     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6478   %}
 6479   ins_pipe(pipe_cmov_reg);
 6480 %}
 6481 
 6482 // Conditional move
 6483 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
 6484 %{
 6485   predicate(!UseAPX);
 6486   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6487 
 6488   ins_cost(200); // XXX
 6489   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
 6490   ins_encode %{
 6491     __ cmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6492   %}
 6493   ins_pipe(pipe_cmov_reg);
 6494 %}
 6495 
 6496 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6497   predicate(!UseAPX);
 6498   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6499   ins_cost(200);
 6500   expand %{
 6501     cmovN_regU(cop, cr, dst, src);
 6502   %}
 6503 %}
 6504 
 6505 // Conditional move ndd
 6506 instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
 6507 %{
 6508   predicate(UseAPX);
 6509   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6510 
 6511   ins_cost(200);
 6512   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6513   ins_encode %{
 6514     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6515   %}
 6516   ins_pipe(pipe_cmov_reg);
 6517 %}
 6518 
 6519 instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
 6520   predicate(UseAPX);
 6521   match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
 6522   ins_cost(200);
 6523   format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
 6524   ins_encode %{
 6525     __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6526   %}
 6527   ins_pipe(pipe_cmov_reg);
 6528 %}
 6529 
 6530 instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6531   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6532   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
 6533 
 6534   ins_cost(200); // XXX
 6535   format %{ "cmovpl  $dst, $src\n\t"
 6536             "cmovnel $dst, $src" %}
 6537   ins_encode %{
 6538     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6539     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6540   %}
 6541   ins_pipe(pipe_cmov_reg);
 6542 %}
 6543 
 6544 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6545 // inputs of the CMove
 6546 instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
 6547   predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6548   match(Set dst (CMoveN (Binary cop cr) (Binary src dst)));
 6549 
 6550   ins_cost(200); // XXX
 6551   format %{ "cmovpl  $dst, $src\n\t"
 6552             "cmovnel $dst, $src" %}
 6553   ins_encode %{
 6554     __ cmovl(Assembler::parity, $dst$$Register, $src$$Register);
 6555     __ cmovl(Assembler::notEqual, $dst$$Register, $src$$Register);
 6556   %}
 6557   ins_pipe(pipe_cmov_reg);
 6558 %}
 6559 
 6560 // Conditional move
 6561 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
 6562 %{
 6563   predicate(!UseAPX);
 6564   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6565 
 6566   ins_cost(200); // XXX
 6567   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
 6568   ins_encode %{
 6569     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6570   %}
 6571   ins_pipe(pipe_cmov_reg);  // XXX
 6572 %}
 6573 
 6574 // Conditional move ndd
 6575 instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
 6576 %{
 6577   predicate(UseAPX);
 6578   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6579 
 6580   ins_cost(200);
 6581   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
 6582   ins_encode %{
 6583     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6584   %}
 6585   ins_pipe(pipe_cmov_reg);
 6586 %}
 6587 
 6588 // Conditional move
 6589 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
 6590 %{
 6591   predicate(!UseAPX);
 6592   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6593 
 6594   ins_cost(200); // XXX
 6595   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
 6596   ins_encode %{
 6597     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6598   %}
 6599   ins_pipe(pipe_cmov_reg); // XXX
 6600 %}
 6601 
 6602 // Conditional move ndd
 6603 instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
 6604 %{
 6605   predicate(UseAPX);
 6606   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6607 
 6608   ins_cost(200);
 6609   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6610   ins_encode %{
 6611     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6612   %}
 6613   ins_pipe(pipe_cmov_reg);
 6614 %}
 6615 
 6616 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6617   predicate(!UseAPX);
 6618   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6619   ins_cost(200);
 6620   expand %{
 6621     cmovP_regU(cop, cr, dst, src);
 6622   %}
 6623 %}
 6624 
 6625 instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
 6626   predicate(UseAPX);
 6627   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6628   ins_cost(200);
 6629   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
 6630   ins_encode %{
 6631     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6632   %}
 6633   ins_pipe(pipe_cmov_reg);
 6634 %}
 6635 
 6636 instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6637   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6638   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
 6639 
 6640   ins_cost(200); // XXX
 6641   format %{ "cmovpq  $dst, $src\n\t"
 6642             "cmovneq $dst, $src" %}
 6643   ins_encode %{
 6644     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6645     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6646   %}
 6647   ins_pipe(pipe_cmov_reg);
 6648 %}
 6649 
 6650 instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6651   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6652   match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
 6653   effect(TEMP dst);
 6654 
 6655   ins_cost(200);
 6656   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6657             "cmovneq  $dst, $src2" %}
 6658   ins_encode %{
 6659     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6660     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6661   %}
 6662   ins_pipe(pipe_cmov_reg);
 6663 %}
 6664 
 6665 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6666 // inputs of the CMove
 6667 instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
 6668   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6669   match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
 6670 
 6671   ins_cost(200); // XXX
 6672   format %{ "cmovpq  $dst, $src\n\t"
 6673             "cmovneq $dst, $src" %}
 6674   ins_encode %{
 6675     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6676     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6677   %}
 6678   ins_pipe(pipe_cmov_reg);
 6679 %}
 6680 
 6681 instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
 6682   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6683   match(Set dst (CMoveP (Binary cop cr) (Binary src2 src1)));
 6684   effect(TEMP dst);
 6685 
 6686   ins_cost(200);
 6687   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6688             "cmovneq  $dst, $src2" %}
 6689   ins_encode %{
 6690     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6691     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6692   %}
 6693   ins_pipe(pipe_cmov_reg);
 6694 %}
 6695 
 6696 instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
 6697 %{
 6698   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6699   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6700 
 6701   ins_cost(100); // XXX
 6702   format %{ "setbn$cop $dst\t# signed, long" %}
 6703   ins_encode %{
 6704     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6705     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6706   %}
 6707   ins_pipe(ialu_reg);
 6708 %}
 6709 
 6710 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
 6711 %{
 6712   predicate(!UseAPX);
 6713   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6714 
 6715   ins_cost(200); // XXX
 6716   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6717   ins_encode %{
 6718     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6719   %}
 6720   ins_pipe(pipe_cmov_reg);  // XXX
 6721 %}
 6722 
 6723 instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
 6724 %{
 6725   predicate(UseAPX);
 6726   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6727 
 6728   ins_cost(200);
 6729   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6730   ins_encode %{
 6731     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6732   %}
 6733   ins_pipe(pipe_cmov_reg);
 6734 %}
 6735 
 6736 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
 6737 %{
 6738   predicate(!UseAPX);
 6739   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6740 
 6741   ins_cost(200); // XXX
 6742   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
 6743   ins_encode %{
 6744     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6745   %}
 6746   ins_pipe(pipe_cmov_mem);  // XXX
 6747 %}
 6748 
 6749 instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
 6750 %{
 6751   predicate(UseAPX);
 6752   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6753 
 6754   ins_cost(200);
 6755   format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
 6756   ins_encode %{
 6757     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6758   %}
 6759   ins_pipe(pipe_cmov_mem);
 6760 %}
 6761 
 6762 instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
 6763 %{
 6764   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6765   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6766 
 6767   ins_cost(100); // XXX
 6768   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6769   ins_encode %{
 6770     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6771     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6772   %}
 6773   ins_pipe(ialu_reg);
 6774 %}
 6775 
 6776 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
 6777 %{
 6778   predicate(!UseAPX);
 6779   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6780 
 6781   ins_cost(200); // XXX
 6782   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6783   ins_encode %{
 6784     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Register);
 6785   %}
 6786   ins_pipe(pipe_cmov_reg); // XXX
 6787 %}
 6788 
 6789 instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
 6790 %{
 6791   predicate(UseAPX);
 6792   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6793 
 6794   ins_cost(200);
 6795   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6796   ins_encode %{
 6797     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6798   %}
 6799   ins_pipe(pipe_cmov_reg);
 6800 %}
 6801 
 6802 instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
 6803 %{
 6804   predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
 6805   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6806 
 6807   ins_cost(100); // XXX
 6808   format %{ "setbn$cop $dst\t# unsigned, long" %}
 6809   ins_encode %{
 6810     Assembler::Condition cond = (Assembler::Condition)($cop$$cmpcode);
 6811     __ setb(MacroAssembler::negate_condition(cond), $dst$$Register);
 6812   %}
 6813   ins_pipe(ialu_reg);
 6814 %}
 6815 
 6816 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6817   predicate(!UseAPX);
 6818   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6819   ins_cost(200);
 6820   expand %{
 6821     cmovL_regU(cop, cr, dst, src);
 6822   %}
 6823 %}
 6824 
 6825 instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
 6826 %{
 6827   predicate(UseAPX);
 6828   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6829   ins_cost(200);
 6830   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6831   ins_encode %{
 6832     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
 6833   %}
 6834   ins_pipe(pipe_cmov_reg);
 6835 %}
 6836 
 6837 instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6838   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6839   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
 6840 
 6841   ins_cost(200); // XXX
 6842   format %{ "cmovpq  $dst, $src\n\t"
 6843             "cmovneq $dst, $src" %}
 6844   ins_encode %{
 6845     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6846     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6847   %}
 6848   ins_pipe(pipe_cmov_reg);
 6849 %}
 6850 
 6851 instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6852   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
 6853   match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
 6854   effect(TEMP dst);
 6855 
 6856   ins_cost(200);
 6857   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6858             "cmovneq  $dst, $src2" %}
 6859   ins_encode %{
 6860     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6861     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6862   %}
 6863   ins_pipe(pipe_cmov_reg);
 6864 %}
 6865 
 6866 // Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
 6867 // inputs of the CMove
 6868 instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
 6869   predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6870   match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
 6871 
 6872   ins_cost(200); // XXX
 6873   format %{ "cmovpq  $dst, $src\n\t"
 6874             "cmovneq $dst, $src" %}
 6875   ins_encode %{
 6876     __ cmovq(Assembler::parity, $dst$$Register, $src$$Register);
 6877     __ cmovq(Assembler::notEqual, $dst$$Register, $src$$Register);
 6878   %}
 6879   ins_pipe(pipe_cmov_reg);
 6880 %}
 6881 
 6882 instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
 6883   predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
 6884   match(Set dst (CMoveL (Binary cop cr) (Binary src2 src1)));
 6885   effect(TEMP dst);
 6886 
 6887   ins_cost(200);
 6888   format %{ "ecmovpq  $dst, $src1, $src2\n\t"
 6889             "cmovneq $dst, $src2" %}
 6890   ins_encode %{
 6891     __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
 6892     __ cmovq(Assembler::notEqual, $dst$$Register, $src2$$Register);
 6893   %}
 6894   ins_pipe(pipe_cmov_reg);
 6895 %}
 6896 
 6897 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
 6898 %{
 6899   predicate(!UseAPX);
 6900   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6901 
 6902   ins_cost(200); // XXX
 6903   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
 6904   ins_encode %{
 6905     __ cmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src$$Address);
 6906   %}
 6907   ins_pipe(pipe_cmov_mem); // XXX
 6908 %}
 6909 
 6910 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
 6911   predicate(!UseAPX);
 6912   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
 6913   ins_cost(200);
 6914   expand %{
 6915     cmovL_memU(cop, cr, dst, src);
 6916   %}
 6917 %}
 6918 
 6919 instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
 6920 %{
 6921   predicate(UseAPX);
 6922   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6923 
 6924   ins_cost(200);
 6925   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6926   ins_encode %{
 6927     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6928   %}
 6929   ins_pipe(pipe_cmov_mem);
 6930 %}
 6931 
 6932 instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
 6933 %{
 6934   predicate(UseAPX);
 6935   match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
 6936   ins_cost(200);
 6937   format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
 6938   ins_encode %{
 6939     __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
 6940   %}
 6941   ins_pipe(pipe_cmov_mem);
 6942 %}
 6943 
 6944 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
 6945 %{
 6946   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6947 
 6948   ins_cost(200); // XXX
 6949   format %{ "jn$cop    skip\t# signed cmove float\n\t"
 6950             "movss     $dst, $src\n"
 6951     "skip:" %}
 6952   ins_encode %{
 6953     Label Lskip;
 6954     // Invert sense of branch from sense of CMOV
 6955     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6956     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6957     __ bind(Lskip);
 6958   %}
 6959   ins_pipe(pipe_slow);
 6960 %}
 6961 
 6962 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
 6963 %{
 6964   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6965 
 6966   ins_cost(200); // XXX
 6967   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 6968             "movss     $dst, $src\n"
 6969     "skip:" %}
 6970   ins_encode %{
 6971     Label Lskip;
 6972     // Invert sense of branch from sense of CMOV
 6973     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 6974     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
 6975     __ bind(Lskip);
 6976   %}
 6977   ins_pipe(pipe_slow);
 6978 %}
 6979 
 6980 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 6981   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 6982   ins_cost(200);
 6983   expand %{
 6984     cmovF_regU(cop, cr, dst, src);
 6985   %}
 6986 %}
 6987 
 6988 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
 6989 %{
 6990   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 6991 
 6992   ins_cost(200); // XXX
 6993   format %{ "jn$cop    skip\t# signed cmove double\n\t"
 6994             "movsd     $dst, $src\n"
 6995     "skip:" %}
 6996   ins_encode %{
 6997     Label Lskip;
 6998     // Invert sense of branch from sense of CMOV
 6999     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7000     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7001     __ bind(Lskip);
 7002   %}
 7003   ins_pipe(pipe_slow);
 7004 %}
 7005 
 7006 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 7007 %{
 7008   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7009 
 7010   ins_cost(200); // XXX
 7011   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 7012             "movsd     $dst, $src\n"
 7013     "skip:" %}
 7014   ins_encode %{
 7015     Label Lskip;
 7016     // Invert sense of branch from sense of CMOV
 7017     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
 7018     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
 7019     __ bind(Lskip);
 7020   %}
 7021   ins_pipe(pipe_slow);
 7022 %}
 7023 
 7024 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 7025   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 7026   ins_cost(200);
 7027   expand %{
 7028     cmovD_regU(cop, cr, dst, src);
 7029   %}
 7030 %}
 7031 
 7032 //----------Arithmetic Instructions--------------------------------------------
 7033 //----------Addition Instructions----------------------------------------------
 7034 
 7035 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 7036 %{
 7037   predicate(!UseAPX);
 7038   match(Set dst (AddI dst src));
 7039   effect(KILL cr);
 7040   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7041   format %{ "addl    $dst, $src\t# int" %}
 7042   ins_encode %{
 7043     __ addl($dst$$Register, $src$$Register);
 7044   %}
 7045   ins_pipe(ialu_reg_reg);
 7046 %}
 7047 
 7048 instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 7049 %{
 7050   predicate(UseAPX);
 7051   match(Set dst (AddI src1 src2));
 7052   effect(KILL cr);
 7053   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7054 
 7055   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7056   ins_encode %{
 7057     __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
 7058   %}
 7059   ins_pipe(ialu_reg_reg);
 7060 %}
 7061 
 7062 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 7063 %{
 7064   predicate(!UseAPX);
 7065   match(Set dst (AddI dst src));
 7066   effect(KILL cr);
 7067   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7068 
 7069   format %{ "addl    $dst, $src\t# int" %}
 7070   ins_encode %{
 7071     __ addl($dst$$Register, $src$$constant);
 7072   %}
 7073   ins_pipe( ialu_reg );
 7074 %}
 7075 
 7076 instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 7077 %{
 7078   predicate(UseAPX);
 7079   match(Set dst (AddI src1 src2));
 7080   effect(KILL cr);
 7081   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7082 
 7083   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7084   ins_encode %{
 7085     __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
 7086   %}
 7087   ins_pipe( ialu_reg );
 7088 %}
 7089 
 7090 instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 7091 %{
 7092   predicate(UseAPX);
 7093   match(Set dst (AddI (LoadI src1) src2));
 7094   effect(KILL cr);
 7095   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7096 
 7097   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7098   ins_encode %{
 7099     __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
 7100   %}
 7101   ins_pipe( ialu_reg );
 7102 %}
 7103 
 7104 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 7105 %{
 7106   predicate(!UseAPX);
 7107   match(Set dst (AddI dst (LoadI src)));
 7108   effect(KILL cr);
 7109   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7110 
 7111   ins_cost(150); // XXX
 7112   format %{ "addl    $dst, $src\t# int" %}
 7113   ins_encode %{
 7114     __ addl($dst$$Register, $src$$Address);
 7115   %}
 7116   ins_pipe(ialu_reg_mem);
 7117 %}
 7118 
 7119 instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 7120 %{
 7121   predicate(UseAPX);
 7122   match(Set dst (AddI src1 (LoadI src2)));
 7123   effect(KILL cr);
 7124   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7125 
 7126   ins_cost(150);
 7127   format %{ "eaddl    $dst, $src1, $src2\t# int ndd" %}
 7128   ins_encode %{
 7129     __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
 7130   %}
 7131   ins_pipe(ialu_reg_mem);
 7132 %}
 7133 
 7134 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 7135 %{
 7136   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7137   effect(KILL cr);
 7138   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7139 
 7140   ins_cost(150); // XXX
 7141   format %{ "addl    $dst, $src\t# int" %}
 7142   ins_encode %{
 7143     __ addl($dst$$Address, $src$$Register);
 7144   %}
 7145   ins_pipe(ialu_mem_reg);
 7146 %}
 7147 
 7148 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
 7149 %{
 7150   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7151   effect(KILL cr);
 7152   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7153 
 7154 
 7155   ins_cost(125); // XXX
 7156   format %{ "addl    $dst, $src\t# int" %}
 7157   ins_encode %{
 7158     __ addl($dst$$Address, $src$$constant);
 7159   %}
 7160   ins_pipe(ialu_mem_imm);
 7161 %}
 7162 
 7163 instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
 7164 %{
 7165   predicate(!UseAPX && UseIncDec);
 7166   match(Set dst (AddI dst src));
 7167   effect(KILL cr);
 7168 
 7169   format %{ "incl    $dst\t# int" %}
 7170   ins_encode %{
 7171     __ incrementl($dst$$Register);
 7172   %}
 7173   ins_pipe(ialu_reg);
 7174 %}
 7175 
 7176 instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
 7177 %{
 7178   predicate(UseAPX && UseIncDec);
 7179   match(Set dst (AddI src val));
 7180   effect(KILL cr);
 7181 
 7182   format %{ "eincl    $dst, $src\t# int ndd" %}
 7183   ins_encode %{
 7184     __ eincl($dst$$Register, $src$$Register, false);
 7185   %}
 7186   ins_pipe(ialu_reg);
 7187 %}
 7188 
 7189 instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
 7190 %{
 7191   predicate(UseAPX && UseIncDec);
 7192   match(Set dst (AddI (LoadI src) val));
 7193   effect(KILL cr);
 7194 
 7195   format %{ "eincl    $dst, $src\t# int ndd" %}
 7196   ins_encode %{
 7197     __ eincl($dst$$Register, $src$$Address, false);
 7198   %}
 7199   ins_pipe(ialu_reg);
 7200 %}
 7201 
 7202 instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
 7203 %{
 7204   predicate(UseIncDec);
 7205   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7206   effect(KILL cr);
 7207 
 7208   ins_cost(125); // XXX
 7209   format %{ "incl    $dst\t# int" %}
 7210   ins_encode %{
 7211     __ incrementl($dst$$Address);
 7212   %}
 7213   ins_pipe(ialu_mem_imm);
 7214 %}
 7215 
 7216 // XXX why does that use AddI
 7217 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
 7218 %{
 7219   predicate(!UseAPX && UseIncDec);
 7220   match(Set dst (AddI dst src));
 7221   effect(KILL cr);
 7222 
 7223   format %{ "decl    $dst\t# int" %}
 7224   ins_encode %{
 7225     __ decrementl($dst$$Register);
 7226   %}
 7227   ins_pipe(ialu_reg);
 7228 %}
 7229 
 7230 instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
 7231 %{
 7232   predicate(UseAPX && UseIncDec);
 7233   match(Set dst (AddI src val));
 7234   effect(KILL cr);
 7235 
 7236   format %{ "edecl    $dst, $src\t# int ndd" %}
 7237   ins_encode %{
 7238     __ edecl($dst$$Register, $src$$Register, false);
 7239   %}
 7240   ins_pipe(ialu_reg);
 7241 %}
 7242 
 7243 instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
 7244 %{
 7245   predicate(UseAPX && UseIncDec);
 7246   match(Set dst (AddI (LoadI src) val));
 7247   effect(KILL cr);
 7248 
 7249   format %{ "edecl    $dst, $src\t# int ndd" %}
 7250   ins_encode %{
 7251     __ edecl($dst$$Register, $src$$Address, false);
 7252   %}
 7253   ins_pipe(ialu_reg);
 7254 %}
 7255 
 7256 // XXX why does that use AddI
 7257 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
 7258 %{
 7259   predicate(UseIncDec);
 7260   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
 7261   effect(KILL cr);
 7262 
 7263   ins_cost(125); // XXX
 7264   format %{ "decl    $dst\t# int" %}
 7265   ins_encode %{
 7266     __ decrementl($dst$$Address);
 7267   %}
 7268   ins_pipe(ialu_mem_imm);
 7269 %}
 7270 
 7271 instruct leaI_rReg_immI2_immI(rRegI dst, rRegI index, immI2 scale, immI disp)
 7272 %{
 7273   predicate(VM_Version::supports_fast_2op_lea());
 7274   match(Set dst (AddI (LShiftI index scale) disp));
 7275 
 7276   format %{ "leal $dst, [$index << $scale + $disp]\t# int" %}
 7277   ins_encode %{
 7278     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7279     __ leal($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7280   %}
 7281   ins_pipe(ialu_reg_reg);
 7282 %}
 7283 
 7284 instruct leaI_rReg_rReg_immI(rRegI dst, rRegI base, rRegI index, immI disp)
 7285 %{
 7286   predicate(VM_Version::supports_fast_3op_lea());
 7287   match(Set dst (AddI (AddI base index) disp));
 7288 
 7289   format %{ "leal $dst, [$base + $index + $disp]\t# int" %}
 7290   ins_encode %{
 7291     __ leal($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7292   %}
 7293   ins_pipe(ialu_reg_reg);
 7294 %}
 7295 
 7296 instruct leaI_rReg_rReg_immI2(rRegI dst, no_rbp_r13_RegI base, rRegI index, immI2 scale)
 7297 %{
 7298   predicate(VM_Version::supports_fast_2op_lea());
 7299   match(Set dst (AddI base (LShiftI index scale)));
 7300 
 7301   format %{ "leal $dst, [$base + $index << $scale]\t# int" %}
 7302   ins_encode %{
 7303     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7304     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7305   %}
 7306   ins_pipe(ialu_reg_reg);
 7307 %}
 7308 
 7309 instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 scale, immI disp)
 7310 %{
 7311   predicate(VM_Version::supports_fast_3op_lea());
 7312   match(Set dst (AddI (AddI base (LShiftI index scale)) disp));
 7313 
 7314   format %{ "leal $dst, [$base + $index << $scale + $disp]\t# int" %}
 7315   ins_encode %{
 7316     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7317     __ leal($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7318   %}
 7319   ins_pipe(ialu_reg_reg);
 7320 %}
 7321 
 7322 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 7323 %{
 7324   predicate(!UseAPX);
 7325   match(Set dst (AddL dst src));
 7326   effect(KILL cr);
 7327   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7328 
 7329   format %{ "addq    $dst, $src\t# long" %}
 7330   ins_encode %{
 7331     __ addq($dst$$Register, $src$$Register);
 7332   %}
 7333   ins_pipe(ialu_reg_reg);
 7334 %}
 7335 
 7336 instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 7337 %{
 7338   predicate(UseAPX);
 7339   match(Set dst (AddL src1 src2));
 7340   effect(KILL cr);
 7341   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7342 
 7343   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7344   ins_encode %{
 7345     __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
 7346   %}
 7347   ins_pipe(ialu_reg_reg);
 7348 %}
 7349 
 7350 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
 7351 %{
 7352   predicate(!UseAPX);
 7353   match(Set dst (AddL dst src));
 7354   effect(KILL cr);
 7355   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7356 
 7357   format %{ "addq    $dst, $src\t# long" %}
 7358   ins_encode %{
 7359     __ addq($dst$$Register, $src$$constant);
 7360   %}
 7361   ins_pipe( ialu_reg );
 7362 %}
 7363 
 7364 instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 7365 %{
 7366   predicate(UseAPX);
 7367   match(Set dst (AddL src1 src2));
 7368   effect(KILL cr);
 7369   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7370 
 7371   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7372   ins_encode %{
 7373     __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
 7374   %}
 7375   ins_pipe( ialu_reg );
 7376 %}
 7377 
 7378 instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 7379 %{
 7380   predicate(UseAPX);
 7381   match(Set dst (AddL (LoadL src1) src2));
 7382   effect(KILL cr);
 7383   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7384 
 7385   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7386   ins_encode %{
 7387     __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
 7388   %}
 7389   ins_pipe( ialu_reg );
 7390 %}
 7391 
 7392 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 7393 %{
 7394   predicate(!UseAPX);
 7395   match(Set dst (AddL dst (LoadL src)));
 7396   effect(KILL cr);
 7397   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7398 
 7399   ins_cost(150); // XXX
 7400   format %{ "addq    $dst, $src\t# long" %}
 7401   ins_encode %{
 7402     __ addq($dst$$Register, $src$$Address);
 7403   %}
 7404   ins_pipe(ialu_reg_mem);
 7405 %}
 7406 
 7407 instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 7408 %{
 7409   predicate(UseAPX);
 7410   match(Set dst (AddL src1 (LoadL src2)));
 7411   effect(KILL cr);
 7412   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7413 
 7414   ins_cost(150);
 7415   format %{ "eaddq    $dst, $src1, $src2\t# long ndd" %}
 7416   ins_encode %{
 7417     __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
 7418   %}
 7419   ins_pipe(ialu_reg_mem);
 7420 %}
 7421 
 7422 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 7423 %{
 7424   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7425   effect(KILL cr);
 7426   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7427 
 7428   ins_cost(150); // XXX
 7429   format %{ "addq    $dst, $src\t# long" %}
 7430   ins_encode %{
 7431     __ addq($dst$$Address, $src$$Register);
 7432   %}
 7433   ins_pipe(ialu_mem_reg);
 7434 %}
 7435 
 7436 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
 7437 %{
 7438   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7439   effect(KILL cr);
 7440   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7441 
 7442   ins_cost(125); // XXX
 7443   format %{ "addq    $dst, $src\t# long" %}
 7444   ins_encode %{
 7445     __ addq($dst$$Address, $src$$constant);
 7446   %}
 7447   ins_pipe(ialu_mem_imm);
 7448 %}
 7449 
 7450 instruct incL_rReg(rRegL dst, immL1 src, rFlagsReg cr)
 7451 %{
 7452   predicate(!UseAPX && UseIncDec);
 7453   match(Set dst (AddL dst src));
 7454   effect(KILL cr);
 7455 
 7456   format %{ "incq    $dst\t# long" %}
 7457   ins_encode %{
 7458     __ incrementq($dst$$Register);
 7459   %}
 7460   ins_pipe(ialu_reg);
 7461 %}
 7462 
 7463 instruct incL_rReg_ndd(rRegL dst, rRegI src, immL1 val, rFlagsReg cr)
 7464 %{
 7465   predicate(UseAPX && UseIncDec);
 7466   match(Set dst (AddL src val));
 7467   effect(KILL cr);
 7468 
 7469   format %{ "eincq    $dst, $src\t# long ndd" %}
 7470   ins_encode %{
 7471     __ eincq($dst$$Register, $src$$Register, false);
 7472   %}
 7473   ins_pipe(ialu_reg);
 7474 %}
 7475 
 7476 instruct incL_rReg_mem_ndd(rRegL dst, memory src, immL1 val, rFlagsReg cr)
 7477 %{
 7478   predicate(UseAPX && UseIncDec);
 7479   match(Set dst (AddL (LoadL src) val));
 7480   effect(KILL cr);
 7481 
 7482   format %{ "eincq    $dst, $src\t# long ndd" %}
 7483   ins_encode %{
 7484     __ eincq($dst$$Register, $src$$Address, false);
 7485   %}
 7486   ins_pipe(ialu_reg);
 7487 %}
 7488 
 7489 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
 7490 %{
 7491   predicate(UseIncDec);
 7492   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7493   effect(KILL cr);
 7494 
 7495   ins_cost(125); // XXX
 7496   format %{ "incq    $dst\t# long" %}
 7497   ins_encode %{
 7498     __ incrementq($dst$$Address);
 7499   %}
 7500   ins_pipe(ialu_mem_imm);
 7501 %}
 7502 
 7503 // XXX why does that use AddL
 7504 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
 7505 %{
 7506   predicate(!UseAPX && UseIncDec);
 7507   match(Set dst (AddL dst src));
 7508   effect(KILL cr);
 7509 
 7510   format %{ "decq    $dst\t# long" %}
 7511   ins_encode %{
 7512     __ decrementq($dst$$Register);
 7513   %}
 7514   ins_pipe(ialu_reg);
 7515 %}
 7516 
 7517 instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
 7518 %{
 7519   predicate(UseAPX && UseIncDec);
 7520   match(Set dst (AddL src val));
 7521   effect(KILL cr);
 7522 
 7523   format %{ "edecq    $dst, $src\t# long ndd" %}
 7524   ins_encode %{
 7525     __ edecq($dst$$Register, $src$$Register, false);
 7526   %}
 7527   ins_pipe(ialu_reg);
 7528 %}
 7529 
 7530 instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
 7531 %{
 7532   predicate(UseAPX && UseIncDec);
 7533   match(Set dst (AddL (LoadL src) val));
 7534   effect(KILL cr);
 7535 
 7536   format %{ "edecq    $dst, $src\t# long ndd" %}
 7537   ins_encode %{
 7538     __ edecq($dst$$Register, $src$$Address, false);
 7539   %}
 7540   ins_pipe(ialu_reg);
 7541 %}
 7542 
 7543 // XXX why does that use AddL
 7544 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
 7545 %{
 7546   predicate(UseIncDec);
 7547   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
 7548   effect(KILL cr);
 7549 
 7550   ins_cost(125); // XXX
 7551   format %{ "decq    $dst\t# long" %}
 7552   ins_encode %{
 7553     __ decrementq($dst$$Address);
 7554   %}
 7555   ins_pipe(ialu_mem_imm);
 7556 %}
 7557 
 7558 instruct leaL_rReg_immI2_immL32(rRegL dst, rRegL index, immI2 scale, immL32 disp)
 7559 %{
 7560   predicate(VM_Version::supports_fast_2op_lea());
 7561   match(Set dst (AddL (LShiftL index scale) disp));
 7562 
 7563   format %{ "leaq $dst, [$index << $scale + $disp]\t# long" %}
 7564   ins_encode %{
 7565     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7566     __ leaq($dst$$Register, Address(noreg, $index$$Register, scale, $disp$$constant));
 7567   %}
 7568   ins_pipe(ialu_reg_reg);
 7569 %}
 7570 
 7571 instruct leaL_rReg_rReg_immL32(rRegL dst, rRegL base, rRegL index, immL32 disp)
 7572 %{
 7573   predicate(VM_Version::supports_fast_3op_lea());
 7574   match(Set dst (AddL (AddL base index) disp));
 7575 
 7576   format %{ "leaq $dst, [$base + $index + $disp]\t# long" %}
 7577   ins_encode %{
 7578     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, Address::times_1, $disp$$constant));
 7579   %}
 7580   ins_pipe(ialu_reg_reg);
 7581 %}
 7582 
 7583 instruct leaL_rReg_rReg_immI2(rRegL dst, no_rbp_r13_RegL base, rRegL index, immI2 scale)
 7584 %{
 7585   predicate(VM_Version::supports_fast_2op_lea());
 7586   match(Set dst (AddL base (LShiftL index scale)));
 7587 
 7588   format %{ "leaq $dst, [$base + $index << $scale]\t# long" %}
 7589   ins_encode %{
 7590     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7591     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale));
 7592   %}
 7593   ins_pipe(ialu_reg_reg);
 7594 %}
 7595 
 7596 instruct leaL_rReg_rReg_immI2_immL32(rRegL dst, rRegL base, rRegL index, immI2 scale, immL32 disp)
 7597 %{
 7598   predicate(VM_Version::supports_fast_3op_lea());
 7599   match(Set dst (AddL (AddL base (LShiftL index scale)) disp));
 7600 
 7601   format %{ "leaq $dst, [$base + $index << $scale + $disp]\t# long" %}
 7602   ins_encode %{
 7603     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($scale$$constant);
 7604     __ leaq($dst$$Register, Address($base$$Register, $index$$Register, scale, $disp$$constant));
 7605   %}
 7606   ins_pipe(ialu_reg_reg);
 7607 %}
 7608 
 7609 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
 7610 %{
 7611   match(Set dst (AddP dst src));
 7612   effect(KILL cr);
 7613   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7614 
 7615   format %{ "addq    $dst, $src\t# ptr" %}
 7616   ins_encode %{
 7617     __ addq($dst$$Register, $src$$Register);
 7618   %}
 7619   ins_pipe(ialu_reg_reg);
 7620 %}
 7621 
 7622 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
 7623 %{
 7624   match(Set dst (AddP dst src));
 7625   effect(KILL cr);
 7626   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 7627 
 7628   format %{ "addq    $dst, $src\t# ptr" %}
 7629   ins_encode %{
 7630     __ addq($dst$$Register, $src$$constant);
 7631   %}
 7632   ins_pipe( ialu_reg );
 7633 %}
 7634 
 7635 // XXX addP mem ops ????
 7636 
 7637 instruct checkCastPP(rRegP dst)
 7638 %{
 7639   match(Set dst (CheckCastPP dst));
 7640 
 7641   size(0);
 7642   format %{ "# checkcastPP of $dst" %}
 7643   ins_encode(/* empty encoding */);
 7644   ins_pipe(empty);
 7645 %}
 7646 
 7647 instruct castPP(rRegP dst)
 7648 %{
 7649   match(Set dst (CastPP dst));
 7650 
 7651   size(0);
 7652   format %{ "# castPP of $dst" %}
 7653   ins_encode(/* empty encoding */);
 7654   ins_pipe(empty);
 7655 %}
 7656 
 7657 instruct castII(rRegI dst)
 7658 %{
 7659   predicate(VerifyConstraintCasts == 0);
 7660   match(Set dst (CastII dst));
 7661 
 7662   size(0);
 7663   format %{ "# castII of $dst" %}
 7664   ins_encode(/* empty encoding */);
 7665   ins_cost(0);
 7666   ins_pipe(empty);
 7667 %}
 7668 
 7669 instruct castII_checked(rRegI dst, rFlagsReg cr)
 7670 %{
 7671   predicate(VerifyConstraintCasts > 0);
 7672   match(Set dst (CastII dst));
 7673 
 7674   effect(KILL cr);
 7675   format %{ "# cast_checked_II $dst" %}
 7676   ins_encode %{
 7677     __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register);
 7678   %}
 7679   ins_pipe(pipe_slow);
 7680 %}
 7681 
 7682 instruct castLL(rRegL dst)
 7683 %{
 7684   predicate(VerifyConstraintCasts == 0);
 7685   match(Set dst (CastLL dst));
 7686 
 7687   size(0);
 7688   format %{ "# castLL of $dst" %}
 7689   ins_encode(/* empty encoding */);
 7690   ins_cost(0);
 7691   ins_pipe(empty);
 7692 %}
 7693 
 7694 instruct castLL_checked_L32(rRegL dst, rFlagsReg cr)
 7695 %{
 7696   predicate(VerifyConstraintCasts > 0 && castLL_is_imm32(n));
 7697   match(Set dst (CastLL dst));
 7698 
 7699   effect(KILL cr);
 7700   format %{ "# cast_checked_LL $dst" %}
 7701   ins_encode %{
 7702     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, noreg);
 7703   %}
 7704   ins_pipe(pipe_slow);
 7705 %}
 7706 
 7707 instruct castLL_checked(rRegL dst, rRegL tmp, rFlagsReg cr)
 7708 %{
 7709   predicate(VerifyConstraintCasts > 0 && !castLL_is_imm32(n));
 7710   match(Set dst (CastLL dst));
 7711 
 7712   effect(KILL cr, TEMP tmp);
 7713   format %{ "# cast_checked_LL $dst\tusing $tmp as TEMP" %}
 7714   ins_encode %{
 7715     __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, $tmp$$Register);
 7716   %}
 7717   ins_pipe(pipe_slow);
 7718 %}
 7719 
 7720 instruct castFF(regF dst)
 7721 %{
 7722   match(Set dst (CastFF dst));
 7723 
 7724   size(0);
 7725   format %{ "# castFF of $dst" %}
 7726   ins_encode(/* empty encoding */);
 7727   ins_cost(0);
 7728   ins_pipe(empty);
 7729 %}
 7730 
 7731 instruct castHH(regF dst)
 7732 %{
 7733   match(Set dst (CastHH dst));
 7734 
 7735   size(0);
 7736   format %{ "# castHH of $dst" %}
 7737   ins_encode(/* empty encoding */);
 7738   ins_cost(0);
 7739   ins_pipe(empty);
 7740 %}
 7741 
 7742 instruct castDD(regD dst)
 7743 %{
 7744   match(Set dst (CastDD dst));
 7745 
 7746   size(0);
 7747   format %{ "# castDD of $dst" %}
 7748   ins_encode(/* empty encoding */);
 7749   ins_cost(0);
 7750   ins_pipe(empty);
 7751 %}
 7752 
 7753 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
 7754 instruct compareAndSwapP(rRegI res,
 7755                          memory mem_ptr,
 7756                          rax_RegP oldval, rRegP newval,
 7757                          rFlagsReg cr)
 7758 %{
 7759   predicate(n->as_LoadStore()->barrier_data() == 0);
 7760   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
 7761   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
 7762   effect(KILL cr, KILL oldval);
 7763 
 7764   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7765             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7766             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7767   ins_encode %{
 7768     __ lock();
 7769     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7770     __ setcc(Assembler::equal, $res$$Register);
 7771   %}
 7772   ins_pipe( pipe_cmpxchg );
 7773 %}
 7774 
 7775 instruct compareAndSwapL(rRegI res,
 7776                          memory mem_ptr,
 7777                          rax_RegL oldval, rRegL newval,
 7778                          rFlagsReg cr)
 7779 %{
 7780   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
 7781   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
 7782   effect(KILL cr, KILL oldval);
 7783 
 7784   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7785             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7786             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7787   ins_encode %{
 7788     __ lock();
 7789     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7790     __ setcc(Assembler::equal, $res$$Register);
 7791   %}
 7792   ins_pipe( pipe_cmpxchg );
 7793 %}
 7794 
 7795 instruct compareAndSwapI(rRegI res,
 7796                          memory mem_ptr,
 7797                          rax_RegI oldval, rRegI newval,
 7798                          rFlagsReg cr)
 7799 %{
 7800   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
 7801   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
 7802   effect(KILL cr, KILL oldval);
 7803 
 7804   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7805             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7806             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7807   ins_encode %{
 7808     __ lock();
 7809     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7810     __ setcc(Assembler::equal, $res$$Register);
 7811   %}
 7812   ins_pipe( pipe_cmpxchg );
 7813 %}
 7814 
 7815 instruct compareAndSwapB(rRegI res,
 7816                          memory mem_ptr,
 7817                          rax_RegI oldval, rRegI newval,
 7818                          rFlagsReg cr)
 7819 %{
 7820   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
 7821   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
 7822   effect(KILL cr, KILL oldval);
 7823 
 7824   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7825             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7826             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7827   ins_encode %{
 7828     __ lock();
 7829     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7830     __ setcc(Assembler::equal, $res$$Register);
 7831   %}
 7832   ins_pipe( pipe_cmpxchg );
 7833 %}
 7834 
 7835 instruct compareAndSwapS(rRegI res,
 7836                          memory mem_ptr,
 7837                          rax_RegI oldval, rRegI newval,
 7838                          rFlagsReg cr)
 7839 %{
 7840   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
 7841   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
 7842   effect(KILL cr, KILL oldval);
 7843 
 7844   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7845             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7846             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7847   ins_encode %{
 7848     __ lock();
 7849     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7850     __ setcc(Assembler::equal, $res$$Register);
 7851   %}
 7852   ins_pipe( pipe_cmpxchg );
 7853 %}
 7854 
 7855 instruct compareAndSwapN(rRegI res,
 7856                           memory mem_ptr,
 7857                           rax_RegN oldval, rRegN newval,
 7858                           rFlagsReg cr) %{
 7859   predicate(n->as_LoadStore()->barrier_data() == 0);
 7860   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
 7861   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
 7862   effect(KILL cr, KILL oldval);
 7863 
 7864   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7865             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
 7866             "setcc $res \t# emits sete + movzbl or setzue for APX" %}
 7867   ins_encode %{
 7868     __ lock();
 7869     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7870     __ setcc(Assembler::equal, $res$$Register);
 7871   %}
 7872   ins_pipe( pipe_cmpxchg );
 7873 %}
 7874 
 7875 instruct compareAndExchangeB(
 7876                          memory mem_ptr,
 7877                          rax_RegI oldval, rRegI newval,
 7878                          rFlagsReg cr)
 7879 %{
 7880   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
 7881   effect(KILL cr);
 7882 
 7883   format %{ "cmpxchgb $mem_ptr,$newval\t# "
 7884             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7885   ins_encode %{
 7886     __ lock();
 7887     __ cmpxchgb($newval$$Register, $mem_ptr$$Address);
 7888   %}
 7889   ins_pipe( pipe_cmpxchg );
 7890 %}
 7891 
 7892 instruct compareAndExchangeS(
 7893                          memory mem_ptr,
 7894                          rax_RegI oldval, rRegI newval,
 7895                          rFlagsReg cr)
 7896 %{
 7897   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
 7898   effect(KILL cr);
 7899 
 7900   format %{ "cmpxchgw $mem_ptr,$newval\t# "
 7901             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7902   ins_encode %{
 7903     __ lock();
 7904     __ cmpxchgw($newval$$Register, $mem_ptr$$Address);
 7905   %}
 7906   ins_pipe( pipe_cmpxchg );
 7907 %}
 7908 
 7909 instruct compareAndExchangeI(
 7910                          memory mem_ptr,
 7911                          rax_RegI oldval, rRegI newval,
 7912                          rFlagsReg cr)
 7913 %{
 7914   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
 7915   effect(KILL cr);
 7916 
 7917   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7918             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7919   ins_encode %{
 7920     __ lock();
 7921     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7922   %}
 7923   ins_pipe( pipe_cmpxchg );
 7924 %}
 7925 
 7926 instruct compareAndExchangeL(
 7927                          memory mem_ptr,
 7928                          rax_RegL oldval, rRegL newval,
 7929                          rFlagsReg cr)
 7930 %{
 7931   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
 7932   effect(KILL cr);
 7933 
 7934   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7935             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
 7936   ins_encode %{
 7937     __ lock();
 7938     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7939   %}
 7940   ins_pipe( pipe_cmpxchg );
 7941 %}
 7942 
 7943 instruct compareAndExchangeN(
 7944                           memory mem_ptr,
 7945                           rax_RegN oldval, rRegN newval,
 7946                           rFlagsReg cr) %{
 7947   predicate(n->as_LoadStore()->barrier_data() == 0);
 7948   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
 7949   effect(KILL cr);
 7950 
 7951   format %{ "cmpxchgl $mem_ptr,$newval\t# "
 7952             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7953   ins_encode %{
 7954     __ lock();
 7955     __ cmpxchgl($newval$$Register, $mem_ptr$$Address);
 7956   %}
 7957   ins_pipe( pipe_cmpxchg );
 7958 %}
 7959 
 7960 instruct compareAndExchangeP(
 7961                          memory mem_ptr,
 7962                          rax_RegP oldval, rRegP newval,
 7963                          rFlagsReg cr)
 7964 %{
 7965   predicate(n->as_LoadStore()->barrier_data() == 0);
 7966   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
 7967   effect(KILL cr);
 7968 
 7969   format %{ "cmpxchgq $mem_ptr,$newval\t# "
 7970             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
 7971   ins_encode %{
 7972     __ lock();
 7973     __ cmpxchgq($newval$$Register, $mem_ptr$$Address);
 7974   %}
 7975   ins_pipe( pipe_cmpxchg );
 7976 %}
 7977 
 7978 instruct xaddB_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 7979   predicate(n->as_LoadStore()->result_not_used());
 7980   match(Set dummy (GetAndAddB mem add));
 7981   effect(KILL cr);
 7982   format %{ "addb_lock   $mem, $add" %}
 7983   ins_encode %{
 7984     __ lock();
 7985     __ addb($mem$$Address, $add$$Register);
 7986   %}
 7987   ins_pipe(pipe_cmpxchg);
 7988 %}
 7989 
 7990 instruct xaddB_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 7991   predicate(n->as_LoadStore()->result_not_used());
 7992   match(Set dummy (GetAndAddB mem add));
 7993   effect(KILL cr);
 7994   format %{ "addb_lock   $mem, $add" %}
 7995   ins_encode %{
 7996     __ lock();
 7997     __ addb($mem$$Address, $add$$constant);
 7998   %}
 7999   ins_pipe(pipe_cmpxchg);
 8000 %}
 8001 
 8002 instruct xaddB(memory mem, rRegI newval, rFlagsReg cr) %{
 8003   predicate(!n->as_LoadStore()->result_not_used());
 8004   match(Set newval (GetAndAddB mem newval));
 8005   effect(KILL cr);
 8006   format %{ "xaddb_lock  $mem, $newval" %}
 8007   ins_encode %{
 8008     __ lock();
 8009     __ xaddb($mem$$Address, $newval$$Register);
 8010   %}
 8011   ins_pipe(pipe_cmpxchg);
 8012 %}
 8013 
 8014 instruct xaddS_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8015   predicate(n->as_LoadStore()->result_not_used());
 8016   match(Set dummy (GetAndAddS mem add));
 8017   effect(KILL cr);
 8018   format %{ "addw_lock   $mem, $add" %}
 8019   ins_encode %{
 8020     __ lock();
 8021     __ addw($mem$$Address, $add$$Register);
 8022   %}
 8023   ins_pipe(pipe_cmpxchg);
 8024 %}
 8025 
 8026 instruct xaddS_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8027   predicate(UseStoreImmI16 && n->as_LoadStore()->result_not_used());
 8028   match(Set dummy (GetAndAddS mem add));
 8029   effect(KILL cr);
 8030   format %{ "addw_lock   $mem, $add" %}
 8031   ins_encode %{
 8032     __ lock();
 8033     __ addw($mem$$Address, $add$$constant);
 8034   %}
 8035   ins_pipe(pipe_cmpxchg);
 8036 %}
 8037 
 8038 instruct xaddS(memory mem, rRegI newval, rFlagsReg cr) %{
 8039   predicate(!n->as_LoadStore()->result_not_used());
 8040   match(Set newval (GetAndAddS mem newval));
 8041   effect(KILL cr);
 8042   format %{ "xaddw_lock  $mem, $newval" %}
 8043   ins_encode %{
 8044     __ lock();
 8045     __ xaddw($mem$$Address, $newval$$Register);
 8046   %}
 8047   ins_pipe(pipe_cmpxchg);
 8048 %}
 8049 
 8050 instruct xaddI_reg_no_res(memory mem, Universe dummy, rRegI add, rFlagsReg cr) %{
 8051   predicate(n->as_LoadStore()->result_not_used());
 8052   match(Set dummy (GetAndAddI mem add));
 8053   effect(KILL cr);
 8054   format %{ "addl_lock   $mem, $add" %}
 8055   ins_encode %{
 8056     __ lock();
 8057     __ addl($mem$$Address, $add$$Register);
 8058   %}
 8059   ins_pipe(pipe_cmpxchg);
 8060 %}
 8061 
 8062 instruct xaddI_imm_no_res(memory mem, Universe dummy, immI add, rFlagsReg cr) %{
 8063   predicate(n->as_LoadStore()->result_not_used());
 8064   match(Set dummy (GetAndAddI mem add));
 8065   effect(KILL cr);
 8066   format %{ "addl_lock   $mem, $add" %}
 8067   ins_encode %{
 8068     __ lock();
 8069     __ addl($mem$$Address, $add$$constant);
 8070   %}
 8071   ins_pipe(pipe_cmpxchg);
 8072 %}
 8073 
 8074 instruct xaddI(memory mem, rRegI newval, rFlagsReg cr) %{
 8075   predicate(!n->as_LoadStore()->result_not_used());
 8076   match(Set newval (GetAndAddI mem newval));
 8077   effect(KILL cr);
 8078   format %{ "xaddl_lock  $mem, $newval" %}
 8079   ins_encode %{
 8080     __ lock();
 8081     __ xaddl($mem$$Address, $newval$$Register);
 8082   %}
 8083   ins_pipe(pipe_cmpxchg);
 8084 %}
 8085 
 8086 instruct xaddL_reg_no_res(memory mem, Universe dummy, rRegL add, rFlagsReg cr) %{
 8087   predicate(n->as_LoadStore()->result_not_used());
 8088   match(Set dummy (GetAndAddL mem add));
 8089   effect(KILL cr);
 8090   format %{ "addq_lock   $mem, $add" %}
 8091   ins_encode %{
 8092     __ lock();
 8093     __ addq($mem$$Address, $add$$Register);
 8094   %}
 8095   ins_pipe(pipe_cmpxchg);
 8096 %}
 8097 
 8098 instruct xaddL_imm_no_res(memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
 8099   predicate(n->as_LoadStore()->result_not_used());
 8100   match(Set dummy (GetAndAddL mem add));
 8101   effect(KILL cr);
 8102   format %{ "addq_lock   $mem, $add" %}
 8103   ins_encode %{
 8104     __ lock();
 8105     __ addq($mem$$Address, $add$$constant);
 8106   %}
 8107   ins_pipe(pipe_cmpxchg);
 8108 %}
 8109 
 8110 instruct xaddL(memory mem, rRegL newval, rFlagsReg cr) %{
 8111   predicate(!n->as_LoadStore()->result_not_used());
 8112   match(Set newval (GetAndAddL mem newval));
 8113   effect(KILL cr);
 8114   format %{ "xaddq_lock  $mem, $newval" %}
 8115   ins_encode %{
 8116     __ lock();
 8117     __ xaddq($mem$$Address, $newval$$Register);
 8118   %}
 8119   ins_pipe(pipe_cmpxchg);
 8120 %}
 8121 
 8122 instruct xchgB( memory mem, rRegI newval) %{
 8123   match(Set newval (GetAndSetB mem newval));
 8124   format %{ "XCHGB  $newval,[$mem]" %}
 8125   ins_encode %{
 8126     __ xchgb($newval$$Register, $mem$$Address);
 8127   %}
 8128   ins_pipe( pipe_cmpxchg );
 8129 %}
 8130 
 8131 instruct xchgS( memory mem, rRegI newval) %{
 8132   match(Set newval (GetAndSetS mem newval));
 8133   format %{ "XCHGW  $newval,[$mem]" %}
 8134   ins_encode %{
 8135     __ xchgw($newval$$Register, $mem$$Address);
 8136   %}
 8137   ins_pipe( pipe_cmpxchg );
 8138 %}
 8139 
 8140 instruct xchgI( memory mem, rRegI newval) %{
 8141   match(Set newval (GetAndSetI mem newval));
 8142   format %{ "XCHGL  $newval,[$mem]" %}
 8143   ins_encode %{
 8144     __ xchgl($newval$$Register, $mem$$Address);
 8145   %}
 8146   ins_pipe( pipe_cmpxchg );
 8147 %}
 8148 
 8149 instruct xchgL( memory mem, rRegL newval) %{
 8150   match(Set newval (GetAndSetL mem newval));
 8151   format %{ "XCHGL  $newval,[$mem]" %}
 8152   ins_encode %{
 8153     __ xchgq($newval$$Register, $mem$$Address);
 8154   %}
 8155   ins_pipe( pipe_cmpxchg );
 8156 %}
 8157 
 8158 instruct xchgP( memory mem, rRegP newval) %{
 8159   match(Set newval (GetAndSetP mem newval));
 8160   predicate(n->as_LoadStore()->barrier_data() == 0);
 8161   format %{ "XCHGQ  $newval,[$mem]" %}
 8162   ins_encode %{
 8163     __ xchgq($newval$$Register, $mem$$Address);
 8164   %}
 8165   ins_pipe( pipe_cmpxchg );
 8166 %}
 8167 
 8168 instruct xchgN( memory mem, rRegN newval) %{
 8169   predicate(n->as_LoadStore()->barrier_data() == 0);
 8170   match(Set newval (GetAndSetN mem newval));
 8171   format %{ "XCHGL  $newval,$mem]" %}
 8172   ins_encode %{
 8173     __ xchgl($newval$$Register, $mem$$Address);
 8174   %}
 8175   ins_pipe( pipe_cmpxchg );
 8176 %}
 8177 
 8178 //----------Abs Instructions-------------------------------------------
 8179 
 8180 // Integer Absolute Instructions
 8181 instruct absI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8182 %{
 8183   match(Set dst (AbsI src));
 8184   effect(TEMP dst, KILL cr);
 8185   format %{ "xorl    $dst, $dst\t# abs int\n\t"
 8186             "subl    $dst, $src\n\t"
 8187             "cmovll  $dst, $src" %}
 8188   ins_encode %{
 8189     __ xorl($dst$$Register, $dst$$Register);
 8190     __ subl($dst$$Register, $src$$Register);
 8191     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
 8192   %}
 8193 
 8194   ins_pipe(ialu_reg_reg);
 8195 %}
 8196 
 8197 // Long Absolute Instructions
 8198 instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8199 %{
 8200   match(Set dst (AbsL src));
 8201   effect(TEMP dst, KILL cr);
 8202   format %{ "xorl    $dst, $dst\t# abs long\n\t"
 8203             "subq    $dst, $src\n\t"
 8204             "cmovlq  $dst, $src" %}
 8205   ins_encode %{
 8206     __ xorl($dst$$Register, $dst$$Register);
 8207     __ subq($dst$$Register, $src$$Register);
 8208     __ cmovq(Assembler::less, $dst$$Register, $src$$Register);
 8209   %}
 8210 
 8211   ins_pipe(ialu_reg_reg);
 8212 %}
 8213 
 8214 //----------Subtraction Instructions-------------------------------------------
 8215 
 8216 // Integer Subtraction Instructions
 8217 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8218 %{
 8219   predicate(!UseAPX);
 8220   match(Set dst (SubI dst src));
 8221   effect(KILL cr);
 8222   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8223 
 8224   format %{ "subl    $dst, $src\t# int" %}
 8225   ins_encode %{
 8226     __ subl($dst$$Register, $src$$Register);
 8227   %}
 8228   ins_pipe(ialu_reg_reg);
 8229 %}
 8230 
 8231 instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8232 %{
 8233   predicate(UseAPX);
 8234   match(Set dst (SubI src1 src2));
 8235   effect(KILL cr);
 8236   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8237 
 8238   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8239   ins_encode %{
 8240     __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
 8241   %}
 8242   ins_pipe(ialu_reg_reg);
 8243 %}
 8244 
 8245 instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
 8246 %{
 8247   predicate(UseAPX);
 8248   match(Set dst (SubI src1 src2));
 8249   effect(KILL cr);
 8250   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8251 
 8252   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8253   ins_encode %{
 8254     __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
 8255   %}
 8256   ins_pipe(ialu_reg_reg);
 8257 %}
 8258 
 8259 instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
 8260 %{
 8261   predicate(UseAPX);
 8262   match(Set dst (SubI (LoadI src1) src2));
 8263   effect(KILL cr);
 8264   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8265 
 8266   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8267   ins_encode %{
 8268     __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
 8269   %}
 8270   ins_pipe(ialu_reg_reg);
 8271 %}
 8272 
 8273 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
 8274 %{
 8275   predicate(!UseAPX);
 8276   match(Set dst (SubI dst (LoadI src)));
 8277   effect(KILL cr);
 8278   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8279 
 8280   ins_cost(150);
 8281   format %{ "subl    $dst, $src\t# int" %}
 8282   ins_encode %{
 8283     __ subl($dst$$Register, $src$$Address);
 8284   %}
 8285   ins_pipe(ialu_reg_mem);
 8286 %}
 8287 
 8288 instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8289 %{
 8290   predicate(UseAPX);
 8291   match(Set dst (SubI src1 (LoadI src2)));
 8292   effect(KILL cr);
 8293   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8294 
 8295   ins_cost(150);
 8296   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8297   ins_encode %{
 8298     __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
 8299   %}
 8300   ins_pipe(ialu_reg_mem);
 8301 %}
 8302 
 8303 instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
 8304 %{
 8305   predicate(UseAPX);
 8306   match(Set dst (SubI (LoadI src1) src2));
 8307   effect(KILL cr);
 8308   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8309 
 8310   ins_cost(150);
 8311   format %{ "esubl    $dst, $src1, $src2\t# int ndd" %}
 8312   ins_encode %{
 8313     __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
 8314   %}
 8315   ins_pipe(ialu_reg_mem);
 8316 %}
 8317 
 8318 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
 8319 %{
 8320   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
 8321   effect(KILL cr);
 8322   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8323 
 8324   ins_cost(150);
 8325   format %{ "subl    $dst, $src\t# int" %}
 8326   ins_encode %{
 8327     __ subl($dst$$Address, $src$$Register);
 8328   %}
 8329   ins_pipe(ialu_mem_reg);
 8330 %}
 8331 
 8332 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8333 %{
 8334   predicate(!UseAPX);
 8335   match(Set dst (SubL dst src));
 8336   effect(KILL cr);
 8337   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8338 
 8339   format %{ "subq    $dst, $src\t# long" %}
 8340   ins_encode %{
 8341     __ subq($dst$$Register, $src$$Register);
 8342   %}
 8343   ins_pipe(ialu_reg_reg);
 8344 %}
 8345 
 8346 instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8347 %{
 8348   predicate(UseAPX);
 8349   match(Set dst (SubL src1 src2));
 8350   effect(KILL cr);
 8351   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8352 
 8353   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8354   ins_encode %{
 8355     __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8356   %}
 8357   ins_pipe(ialu_reg_reg);
 8358 %}
 8359 
 8360 instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
 8361 %{
 8362   predicate(UseAPX);
 8363   match(Set dst (SubL src1 src2));
 8364   effect(KILL cr);
 8365   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8366 
 8367   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8368   ins_encode %{
 8369     __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
 8370   %}
 8371   ins_pipe(ialu_reg_reg);
 8372 %}
 8373 
 8374 instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
 8375 %{
 8376   predicate(UseAPX);
 8377   match(Set dst (SubL (LoadL src1) src2));
 8378   effect(KILL cr);
 8379   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8380 
 8381   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8382   ins_encode %{
 8383     __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
 8384   %}
 8385   ins_pipe(ialu_reg_reg);
 8386 %}
 8387 
 8388 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
 8389 %{
 8390   predicate(!UseAPX);
 8391   match(Set dst (SubL dst (LoadL src)));
 8392   effect(KILL cr);
 8393   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8394 
 8395   ins_cost(150);
 8396   format %{ "subq    $dst, $src\t# long" %}
 8397   ins_encode %{
 8398     __ subq($dst$$Register, $src$$Address);
 8399   %}
 8400   ins_pipe(ialu_reg_mem);
 8401 %}
 8402 
 8403 instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8404 %{
 8405   predicate(UseAPX);
 8406   match(Set dst (SubL src1 (LoadL src2)));
 8407   effect(KILL cr);
 8408   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8409 
 8410   ins_cost(150);
 8411   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8412   ins_encode %{
 8413     __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8414   %}
 8415   ins_pipe(ialu_reg_mem);
 8416 %}
 8417 
 8418 instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
 8419 %{
 8420   predicate(UseAPX);
 8421   match(Set dst (SubL (LoadL src1) src2));
 8422   effect(KILL cr);
 8423   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8424 
 8425   ins_cost(150);
 8426   format %{ "esubq    $dst, $src1, $src2\t# long ndd" %}
 8427   ins_encode %{
 8428     __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
 8429   %}
 8430   ins_pipe(ialu_reg_mem);
 8431 %}
 8432 
 8433 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
 8434 %{
 8435   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
 8436   effect(KILL cr);
 8437   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
 8438 
 8439   ins_cost(150);
 8440   format %{ "subq    $dst, $src\t# long" %}
 8441   ins_encode %{
 8442     __ subq($dst$$Address, $src$$Register);
 8443   %}
 8444   ins_pipe(ialu_mem_reg);
 8445 %}
 8446 
 8447 // Subtract from a pointer
 8448 // XXX hmpf???
 8449 instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8450 %{
 8451   match(Set dst (AddP dst (SubI zero src)));
 8452   effect(KILL cr);
 8453 
 8454   format %{ "subq    $dst, $src\t# ptr - int" %}
 8455   ins_encode %{
 8456     __ subq($dst$$Register, $src$$Register);
 8457   %}
 8458   ins_pipe(ialu_reg_reg);
 8459 %}
 8460 
 8461 instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
 8462 %{
 8463   predicate(!UseAPX);
 8464   match(Set dst (SubI zero dst));
 8465   effect(KILL cr);
 8466   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8467 
 8468   format %{ "negl    $dst\t# int" %}
 8469   ins_encode %{
 8470     __ negl($dst$$Register);
 8471   %}
 8472   ins_pipe(ialu_reg);
 8473 %}
 8474 
 8475 instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
 8476 %{
 8477   predicate(UseAPX);
 8478   match(Set dst (SubI zero src));
 8479   effect(KILL cr);
 8480   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8481 
 8482   format %{ "enegl    $dst, $src\t# int ndd" %}
 8483   ins_encode %{
 8484     __ enegl($dst$$Register, $src$$Register, false);
 8485   %}
 8486   ins_pipe(ialu_reg);
 8487 %}
 8488 
 8489 instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
 8490 %{
 8491   predicate(!UseAPX);
 8492   match(Set dst (NegI dst));
 8493   effect(KILL cr);
 8494   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8495 
 8496   format %{ "negl    $dst\t# int" %}
 8497   ins_encode %{
 8498     __ negl($dst$$Register);
 8499   %}
 8500   ins_pipe(ialu_reg);
 8501 %}
 8502 
 8503 instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
 8504 %{
 8505   predicate(UseAPX);
 8506   match(Set dst (NegI src));
 8507   effect(KILL cr);
 8508   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8509 
 8510   format %{ "enegl    $dst, $src\t# int ndd" %}
 8511   ins_encode %{
 8512     __ enegl($dst$$Register, $src$$Register, false);
 8513   %}
 8514   ins_pipe(ialu_reg);
 8515 %}
 8516 
 8517 instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
 8518 %{
 8519   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
 8520   effect(KILL cr);
 8521   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8522 
 8523   format %{ "negl    $dst\t# int" %}
 8524   ins_encode %{
 8525     __ negl($dst$$Address);
 8526   %}
 8527   ins_pipe(ialu_reg);
 8528 %}
 8529 
 8530 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
 8531 %{
 8532   predicate(!UseAPX);
 8533   match(Set dst (SubL zero dst));
 8534   effect(KILL cr);
 8535   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8536 
 8537   format %{ "negq    $dst\t# long" %}
 8538   ins_encode %{
 8539     __ negq($dst$$Register);
 8540   %}
 8541   ins_pipe(ialu_reg);
 8542 %}
 8543 
 8544 instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
 8545 %{
 8546   predicate(UseAPX);
 8547   match(Set dst (SubL zero src));
 8548   effect(KILL cr);
 8549   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8550 
 8551   format %{ "enegq    $dst, $src\t# long ndd" %}
 8552   ins_encode %{
 8553     __ enegq($dst$$Register, $src$$Register, false);
 8554   %}
 8555   ins_pipe(ialu_reg);
 8556 %}
 8557 
 8558 instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
 8559 %{
 8560   predicate(!UseAPX);
 8561   match(Set dst (NegL dst));
 8562   effect(KILL cr);
 8563   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8564 
 8565   format %{ "negq    $dst\t# int" %}
 8566   ins_encode %{
 8567     __ negq($dst$$Register);
 8568   %}
 8569   ins_pipe(ialu_reg);
 8570 %}
 8571 
 8572 instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
 8573 %{
 8574   predicate(UseAPX);
 8575   match(Set dst (NegL src));
 8576   effect(KILL cr);
 8577   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8578 
 8579   format %{ "enegq    $dst, $src\t# long ndd" %}
 8580   ins_encode %{
 8581     __ enegq($dst$$Register, $src$$Register, false);
 8582   %}
 8583   ins_pipe(ialu_reg);
 8584 %}
 8585 
 8586 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
 8587 %{
 8588   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
 8589   effect(KILL cr);
 8590   flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
 8591 
 8592   format %{ "negq    $dst\t# long" %}
 8593   ins_encode %{
 8594     __ negq($dst$$Address);
 8595   %}
 8596   ins_pipe(ialu_reg);
 8597 %}
 8598 
 8599 //----------Multiplication/Division Instructions-------------------------------
 8600 // Integer Multiplication Instructions
 8601 // Multiply Register
 8602 
 8603 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
 8604 %{
 8605   predicate(!UseAPX);
 8606   match(Set dst (MulI dst src));
 8607   effect(KILL cr);
 8608 
 8609   ins_cost(300);
 8610   format %{ "imull   $dst, $src\t# int" %}
 8611   ins_encode %{
 8612     __ imull($dst$$Register, $src$$Register);
 8613   %}
 8614   ins_pipe(ialu_reg_reg_alu0);
 8615 %}
 8616 
 8617 instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
 8618 %{
 8619   predicate(UseAPX);
 8620   match(Set dst (MulI src1 src2));
 8621   effect(KILL cr);
 8622 
 8623   ins_cost(300);
 8624   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8625   ins_encode %{
 8626     __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
 8627   %}
 8628   ins_pipe(ialu_reg_reg_alu0);
 8629 %}
 8630 
 8631 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
 8632 %{
 8633   match(Set dst (MulI src imm));
 8634   effect(KILL cr);
 8635 
 8636   ins_cost(300);
 8637   format %{ "imull   $dst, $src, $imm\t# int" %}
 8638   ins_encode %{
 8639     __ imull($dst$$Register, $src$$Register, $imm$$constant);
 8640   %}
 8641   ins_pipe(ialu_reg_reg_alu0);
 8642 %}
 8643 
 8644 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
 8645 %{
 8646   predicate(!UseAPX);
 8647   match(Set dst (MulI dst (LoadI src)));
 8648   effect(KILL cr);
 8649 
 8650   ins_cost(350);
 8651   format %{ "imull   $dst, $src\t# int" %}
 8652   ins_encode %{
 8653     __ imull($dst$$Register, $src$$Address);
 8654   %}
 8655   ins_pipe(ialu_reg_mem_alu0);
 8656 %}
 8657 
 8658 instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
 8659 %{
 8660   predicate(UseAPX);
 8661   match(Set dst (MulI src1 (LoadI src2)));
 8662   effect(KILL cr);
 8663 
 8664   ins_cost(350);
 8665   format %{ "eimull   $dst, $src1, $src2\t# int ndd" %}
 8666   ins_encode %{
 8667     __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
 8668   %}
 8669   ins_pipe(ialu_reg_mem_alu0);
 8670 %}
 8671 
 8672 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
 8673 %{
 8674   match(Set dst (MulI (LoadI src) imm));
 8675   effect(KILL cr);
 8676 
 8677   ins_cost(300);
 8678   format %{ "imull   $dst, $src, $imm\t# int" %}
 8679   ins_encode %{
 8680     __ imull($dst$$Register, $src$$Address, $imm$$constant);
 8681   %}
 8682   ins_pipe(ialu_reg_mem_alu0);
 8683 %}
 8684 
 8685 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
 8686 %{
 8687   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
 8688   effect(KILL cr, KILL src2);
 8689 
 8690   expand %{ mulI_rReg(dst, src1, cr);
 8691            mulI_rReg(src2, src3, cr);
 8692            addI_rReg(dst, src2, cr); %}
 8693 %}
 8694 
 8695 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
 8696 %{
 8697   predicate(!UseAPX);
 8698   match(Set dst (MulL dst src));
 8699   effect(KILL cr);
 8700 
 8701   ins_cost(300);
 8702   format %{ "imulq   $dst, $src\t# long" %}
 8703   ins_encode %{
 8704     __ imulq($dst$$Register, $src$$Register);
 8705   %}
 8706   ins_pipe(ialu_reg_reg_alu0);
 8707 %}
 8708 
 8709 instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
 8710 %{
 8711   predicate(UseAPX);
 8712   match(Set dst (MulL src1 src2));
 8713   effect(KILL cr);
 8714 
 8715   ins_cost(300);
 8716   format %{ "eimulq   $dst, $src1, $src2\t# long ndd" %}
 8717   ins_encode %{
 8718     __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
 8719   %}
 8720   ins_pipe(ialu_reg_reg_alu0);
 8721 %}
 8722 
 8723 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
 8724 %{
 8725   match(Set dst (MulL src imm));
 8726   effect(KILL cr);
 8727 
 8728   ins_cost(300);
 8729   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8730   ins_encode %{
 8731     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
 8732   %}
 8733   ins_pipe(ialu_reg_reg_alu0);
 8734 %}
 8735 
 8736 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
 8737 %{
 8738   predicate(!UseAPX);
 8739   match(Set dst (MulL dst (LoadL src)));
 8740   effect(KILL cr);
 8741 
 8742   ins_cost(350);
 8743   format %{ "imulq   $dst, $src\t# long" %}
 8744   ins_encode %{
 8745     __ imulq($dst$$Register, $src$$Address);
 8746   %}
 8747   ins_pipe(ialu_reg_mem_alu0);
 8748 %}
 8749 
 8750 instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
 8751 %{
 8752   predicate(UseAPX);
 8753   match(Set dst (MulL src1 (LoadL src2)));
 8754   effect(KILL cr);
 8755 
 8756   ins_cost(350);
 8757   format %{ "eimulq   $dst, $src1, $src2 \t# long" %}
 8758   ins_encode %{
 8759     __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
 8760   %}
 8761   ins_pipe(ialu_reg_mem_alu0);
 8762 %}
 8763 
 8764 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
 8765 %{
 8766   match(Set dst (MulL (LoadL src) imm));
 8767   effect(KILL cr);
 8768 
 8769   ins_cost(300);
 8770   format %{ "imulq   $dst, $src, $imm\t# long" %}
 8771   ins_encode %{
 8772     __ imulq($dst$$Register, $src$$Address, $imm$$constant);
 8773   %}
 8774   ins_pipe(ialu_reg_mem_alu0);
 8775 %}
 8776 
 8777 instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8778 %{
 8779   match(Set dst (MulHiL src rax));
 8780   effect(USE_KILL rax, KILL cr);
 8781 
 8782   ins_cost(300);
 8783   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
 8784   ins_encode %{
 8785     __ imulq($src$$Register);
 8786   %}
 8787   ins_pipe(ialu_reg_reg_alu0);
 8788 %}
 8789 
 8790 instruct umulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
 8791 %{
 8792   match(Set dst (UMulHiL src rax));
 8793   effect(USE_KILL rax, KILL cr);
 8794 
 8795   ins_cost(300);
 8796   format %{ "mulq   RDX:RAX, RAX, $src\t# umulhi" %}
 8797   ins_encode %{
 8798     __ mulq($src$$Register);
 8799   %}
 8800   ins_pipe(ialu_reg_reg_alu0);
 8801 %}
 8802 
 8803 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8804                    rFlagsReg cr)
 8805 %{
 8806   match(Set rax (DivI rax div));
 8807   effect(KILL rdx, KILL cr);
 8808 
 8809   ins_cost(30*100+10*100); // XXX
 8810   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8811             "jne,s   normal\n\t"
 8812             "xorl    rdx, rdx\n\t"
 8813             "cmpl    $div, -1\n\t"
 8814             "je,s    done\n"
 8815     "normal: cdql\n\t"
 8816             "idivl   $div\n"
 8817     "done:"        %}
 8818   ins_encode(cdql_enc(div));
 8819   ins_pipe(ialu_reg_reg_alu0);
 8820 %}
 8821 
 8822 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8823                    rFlagsReg cr)
 8824 %{
 8825   match(Set rax (DivL rax div));
 8826   effect(KILL rdx, KILL cr);
 8827 
 8828   ins_cost(30*100+10*100); // XXX
 8829   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8830             "cmpq    rax, rdx\n\t"
 8831             "jne,s   normal\n\t"
 8832             "xorl    rdx, rdx\n\t"
 8833             "cmpq    $div, -1\n\t"
 8834             "je,s    done\n"
 8835     "normal: cdqq\n\t"
 8836             "idivq   $div\n"
 8837     "done:"        %}
 8838   ins_encode(cdqq_enc(div));
 8839   ins_pipe(ialu_reg_reg_alu0);
 8840 %}
 8841 
 8842 instruct udivI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div, rFlagsReg cr)
 8843 %{
 8844   match(Set rax (UDivI rax div));
 8845   effect(KILL rdx, KILL cr);
 8846 
 8847   ins_cost(300);
 8848   format %{ "udivl $rax,$rax,$div\t# UDivI\n" %}
 8849   ins_encode %{
 8850     __ udivI($rax$$Register, $div$$Register, $rdx$$Register);
 8851   %}
 8852   ins_pipe(ialu_reg_reg_alu0);
 8853 %}
 8854 
 8855 instruct udivL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div, rFlagsReg cr)
 8856 %{
 8857   match(Set rax (UDivL rax div));
 8858   effect(KILL rdx, KILL cr);
 8859 
 8860   ins_cost(300);
 8861   format %{ "udivq $rax,$rax,$div\t# UDivL\n" %}
 8862   ins_encode %{
 8863      __ udivL($rax$$Register, $div$$Register, $rdx$$Register);
 8864   %}
 8865   ins_pipe(ialu_reg_reg_alu0);
 8866 %}
 8867 
 8868 // Integer DIVMOD with Register, both quotient and mod results
 8869 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
 8870                              rFlagsReg cr)
 8871 %{
 8872   match(DivModI rax div);
 8873   effect(KILL cr);
 8874 
 8875   ins_cost(30*100+10*100); // XXX
 8876   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
 8877             "jne,s   normal\n\t"
 8878             "xorl    rdx, rdx\n\t"
 8879             "cmpl    $div, -1\n\t"
 8880             "je,s    done\n"
 8881     "normal: cdql\n\t"
 8882             "idivl   $div\n"
 8883     "done:"        %}
 8884   ins_encode(cdql_enc(div));
 8885   ins_pipe(pipe_slow);
 8886 %}
 8887 
 8888 // Long DIVMOD with Register, both quotient and mod results
 8889 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
 8890                              rFlagsReg cr)
 8891 %{
 8892   match(DivModL rax div);
 8893   effect(KILL cr);
 8894 
 8895   ins_cost(30*100+10*100); // XXX
 8896   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
 8897             "cmpq    rax, rdx\n\t"
 8898             "jne,s   normal\n\t"
 8899             "xorl    rdx, rdx\n\t"
 8900             "cmpq    $div, -1\n\t"
 8901             "je,s    done\n"
 8902     "normal: cdqq\n\t"
 8903             "idivq   $div\n"
 8904     "done:"        %}
 8905   ins_encode(cdqq_enc(div));
 8906   ins_pipe(pipe_slow);
 8907 %}
 8908 
 8909 // Unsigned integer DIVMOD with Register, both quotient and mod results
 8910 instruct udivModI_rReg_divmod(rax_RegI rax, no_rax_rdx_RegI tmp, rdx_RegI rdx,
 8911                               no_rax_rdx_RegI div, rFlagsReg cr)
 8912 %{
 8913   match(UDivModI rax div);
 8914   effect(TEMP tmp, KILL cr);
 8915 
 8916   ins_cost(300);
 8917   format %{ "udivl $rax,$rax,$div\t# begin UDivModI\n\t"
 8918             "umodl $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModI\n"
 8919           %}
 8920   ins_encode %{
 8921     __ udivmodI($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8922   %}
 8923   ins_pipe(pipe_slow);
 8924 %}
 8925 
 8926 // Unsigned long DIVMOD with Register, both quotient and mod results
 8927 instruct udivModL_rReg_divmod(rax_RegL rax, no_rax_rdx_RegL tmp, rdx_RegL rdx,
 8928                               no_rax_rdx_RegL div, rFlagsReg cr)
 8929 %{
 8930   match(UDivModL rax div);
 8931   effect(TEMP tmp, KILL cr);
 8932 
 8933   ins_cost(300);
 8934   format %{ "udivq $rax,$rax,$div\t# begin UDivModL\n\t"
 8935             "umodq $rdx,$rax,$div\t! using $tmp as TEMP # end UDivModL\n"
 8936           %}
 8937   ins_encode %{
 8938     __ udivmodL($rax$$Register, $div$$Register, $rdx$$Register, $tmp$$Register);
 8939   %}
 8940   ins_pipe(pipe_slow);
 8941 %}
 8942 
 8943 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
 8944                    rFlagsReg cr)
 8945 %{
 8946   match(Set rdx (ModI rax div));
 8947   effect(KILL rax, KILL cr);
 8948 
 8949   ins_cost(300); // XXX
 8950   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
 8951             "jne,s   normal\n\t"
 8952             "xorl    rdx, rdx\n\t"
 8953             "cmpl    $div, -1\n\t"
 8954             "je,s    done\n"
 8955     "normal: cdql\n\t"
 8956             "idivl   $div\n"
 8957     "done:"        %}
 8958   ins_encode(cdql_enc(div));
 8959   ins_pipe(ialu_reg_reg_alu0);
 8960 %}
 8961 
 8962 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
 8963                    rFlagsReg cr)
 8964 %{
 8965   match(Set rdx (ModL rax div));
 8966   effect(KILL rax, KILL cr);
 8967 
 8968   ins_cost(300); // XXX
 8969   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
 8970             "cmpq    rax, rdx\n\t"
 8971             "jne,s   normal\n\t"
 8972             "xorl    rdx, rdx\n\t"
 8973             "cmpq    $div, -1\n\t"
 8974             "je,s    done\n"
 8975     "normal: cdqq\n\t"
 8976             "idivq   $div\n"
 8977     "done:"        %}
 8978   ins_encode(cdqq_enc(div));
 8979   ins_pipe(ialu_reg_reg_alu0);
 8980 %}
 8981 
 8982 instruct umodI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div, rFlagsReg cr)
 8983 %{
 8984   match(Set rdx (UModI rax div));
 8985   effect(KILL rax, KILL cr);
 8986 
 8987   ins_cost(300);
 8988   format %{ "umodl $rdx,$rax,$div\t# UModI\n" %}
 8989   ins_encode %{
 8990     __ umodI($rax$$Register, $div$$Register, $rdx$$Register);
 8991   %}
 8992   ins_pipe(ialu_reg_reg_alu0);
 8993 %}
 8994 
 8995 instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg cr)
 8996 %{
 8997   match(Set rdx (UModL rax div));
 8998   effect(KILL rax, KILL cr);
 8999 
 9000   ins_cost(300);
 9001   format %{ "umodq $rdx,$rax,$div\t# UModL\n" %}
 9002   ins_encode %{
 9003     __ umodL($rax$$Register, $div$$Register, $rdx$$Register);
 9004   %}
 9005   ins_pipe(ialu_reg_reg_alu0);
 9006 %}
 9007 
 9008 // Integer Shift Instructions
 9009 // Shift Left by one, two, three
 9010 instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
 9011 %{
 9012   predicate(!UseAPX);
 9013   match(Set dst (LShiftI dst shift));
 9014   effect(KILL cr);
 9015 
 9016   format %{ "sall    $dst, $shift" %}
 9017   ins_encode %{
 9018     __ sall($dst$$Register, $shift$$constant);
 9019   %}
 9020   ins_pipe(ialu_reg);
 9021 %}
 9022 
 9023 // Shift Left by one, two, three
 9024 instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
 9025 %{
 9026   predicate(UseAPX);
 9027   match(Set dst (LShiftI src shift));
 9028   effect(KILL cr);
 9029 
 9030   format %{ "esall    $dst, $src, $shift\t# int(ndd)" %}
 9031   ins_encode %{
 9032     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9033   %}
 9034   ins_pipe(ialu_reg);
 9035 %}
 9036 
 9037 // Shift Left by 8-bit immediate
 9038 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9039 %{
 9040   predicate(!UseAPX);
 9041   match(Set dst (LShiftI dst shift));
 9042   effect(KILL cr);
 9043 
 9044   format %{ "sall    $dst, $shift" %}
 9045   ins_encode %{
 9046     __ sall($dst$$Register, $shift$$constant);
 9047   %}
 9048   ins_pipe(ialu_reg);
 9049 %}
 9050 
 9051 // Shift Left by 8-bit immediate
 9052 instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9053 %{
 9054   predicate(UseAPX);
 9055   match(Set dst (LShiftI src shift));
 9056   effect(KILL cr);
 9057 
 9058   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9059   ins_encode %{
 9060     __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
 9061   %}
 9062   ins_pipe(ialu_reg);
 9063 %}
 9064 
 9065 instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9066 %{
 9067   predicate(UseAPX);
 9068   match(Set dst (LShiftI (LoadI src) shift));
 9069   effect(KILL cr);
 9070 
 9071   format %{ "esall    $dst, $src, $shift\t# int (ndd)" %}
 9072   ins_encode %{
 9073     __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
 9074   %}
 9075   ins_pipe(ialu_reg);
 9076 %}
 9077 
 9078 // Shift Left by 8-bit immediate
 9079 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9080 %{
 9081   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9082   effect(KILL cr);
 9083 
 9084   format %{ "sall    $dst, $shift" %}
 9085   ins_encode %{
 9086     __ sall($dst$$Address, $shift$$constant);
 9087   %}
 9088   ins_pipe(ialu_mem_imm);
 9089 %}
 9090 
 9091 // Shift Left by variable
 9092 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9093 %{
 9094   predicate(!VM_Version::supports_bmi2());
 9095   match(Set dst (LShiftI dst shift));
 9096   effect(KILL cr);
 9097 
 9098   format %{ "sall    $dst, $shift" %}
 9099   ins_encode %{
 9100     __ sall($dst$$Register);
 9101   %}
 9102   ins_pipe(ialu_reg_reg);
 9103 %}
 9104 
 9105 // Shift Left by variable
 9106 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9107 %{
 9108   predicate(!VM_Version::supports_bmi2());
 9109   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
 9110   effect(KILL cr);
 9111 
 9112   format %{ "sall    $dst, $shift" %}
 9113   ins_encode %{
 9114     __ sall($dst$$Address);
 9115   %}
 9116   ins_pipe(ialu_mem_reg);
 9117 %}
 9118 
 9119 instruct salI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9120 %{
 9121   predicate(VM_Version::supports_bmi2());
 9122   match(Set dst (LShiftI src shift));
 9123 
 9124   format %{ "shlxl   $dst, $src, $shift" %}
 9125   ins_encode %{
 9126     __ shlxl($dst$$Register, $src$$Register, $shift$$Register);
 9127   %}
 9128   ins_pipe(ialu_reg_reg);
 9129 %}
 9130 
 9131 instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9132 %{
 9133   predicate(VM_Version::supports_bmi2());
 9134   match(Set dst (LShiftI (LoadI src) shift));
 9135   ins_cost(175);
 9136   format %{ "shlxl   $dst, $src, $shift" %}
 9137   ins_encode %{
 9138     __ shlxl($dst$$Register, $src$$Address, $shift$$Register);
 9139   %}
 9140   ins_pipe(ialu_reg_mem);
 9141 %}
 9142 
 9143 // Arithmetic Shift Right by 8-bit immediate
 9144 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9145 %{
 9146   predicate(!UseAPX);
 9147   match(Set dst (RShiftI dst shift));
 9148   effect(KILL cr);
 9149 
 9150   format %{ "sarl    $dst, $shift" %}
 9151   ins_encode %{
 9152     __ sarl($dst$$Register, $shift$$constant);
 9153   %}
 9154   ins_pipe(ialu_mem_imm);
 9155 %}
 9156 
 9157 // Arithmetic Shift Right by 8-bit immediate
 9158 instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9159 %{
 9160   predicate(UseAPX);
 9161   match(Set dst (RShiftI src shift));
 9162   effect(KILL cr);
 9163 
 9164   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9165   ins_encode %{
 9166     __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
 9167   %}
 9168   ins_pipe(ialu_mem_imm);
 9169 %}
 9170 
 9171 instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9172 %{
 9173   predicate(UseAPX);
 9174   match(Set dst (RShiftI (LoadI src) shift));
 9175   effect(KILL cr);
 9176 
 9177   format %{ "esarl    $dst, $src, $shift\t# int (ndd)" %}
 9178   ins_encode %{
 9179     __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
 9180   %}
 9181   ins_pipe(ialu_mem_imm);
 9182 %}
 9183 
 9184 // Arithmetic Shift Right by 8-bit immediate
 9185 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9186 %{
 9187   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9188   effect(KILL cr);
 9189 
 9190   format %{ "sarl    $dst, $shift" %}
 9191   ins_encode %{
 9192     __ sarl($dst$$Address, $shift$$constant);
 9193   %}
 9194   ins_pipe(ialu_mem_imm);
 9195 %}
 9196 
 9197 // Arithmetic Shift Right by variable
 9198 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9199 %{
 9200   predicate(!VM_Version::supports_bmi2());
 9201   match(Set dst (RShiftI dst shift));
 9202   effect(KILL cr);
 9203 
 9204   format %{ "sarl    $dst, $shift" %}
 9205   ins_encode %{
 9206     __ sarl($dst$$Register);
 9207   %}
 9208   ins_pipe(ialu_reg_reg);
 9209 %}
 9210 
 9211 // Arithmetic Shift Right by variable
 9212 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9213 %{
 9214   predicate(!VM_Version::supports_bmi2());
 9215   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
 9216   effect(KILL cr);
 9217 
 9218   format %{ "sarl    $dst, $shift" %}
 9219   ins_encode %{
 9220     __ sarl($dst$$Address);
 9221   %}
 9222   ins_pipe(ialu_mem_reg);
 9223 %}
 9224 
 9225 instruct sarI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9226 %{
 9227   predicate(VM_Version::supports_bmi2());
 9228   match(Set dst (RShiftI src shift));
 9229 
 9230   format %{ "sarxl   $dst, $src, $shift" %}
 9231   ins_encode %{
 9232     __ sarxl($dst$$Register, $src$$Register, $shift$$Register);
 9233   %}
 9234   ins_pipe(ialu_reg_reg);
 9235 %}
 9236 
 9237 instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9238 %{
 9239   predicate(VM_Version::supports_bmi2());
 9240   match(Set dst (RShiftI (LoadI src) shift));
 9241   ins_cost(175);
 9242   format %{ "sarxl   $dst, $src, $shift" %}
 9243   ins_encode %{
 9244     __ sarxl($dst$$Register, $src$$Address, $shift$$Register);
 9245   %}
 9246   ins_pipe(ialu_reg_mem);
 9247 %}
 9248 
 9249 // Logical Shift Right by 8-bit immediate
 9250 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
 9251 %{
 9252   predicate(!UseAPX);
 9253   match(Set dst (URShiftI dst shift));
 9254   effect(KILL cr);
 9255 
 9256   format %{ "shrl    $dst, $shift" %}
 9257   ins_encode %{
 9258     __ shrl($dst$$Register, $shift$$constant);
 9259   %}
 9260   ins_pipe(ialu_reg);
 9261 %}
 9262 
 9263 // Logical Shift Right by 8-bit immediate
 9264 instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
 9265 %{
 9266   predicate(UseAPX);
 9267   match(Set dst (URShiftI src shift));
 9268   effect(KILL cr);
 9269 
 9270   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9271   ins_encode %{
 9272     __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
 9273   %}
 9274   ins_pipe(ialu_reg);
 9275 %}
 9276 
 9277 instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
 9278 %{
 9279   predicate(UseAPX);
 9280   match(Set dst (URShiftI (LoadI src) shift));
 9281   effect(KILL cr);
 9282 
 9283   format %{ "eshrl    $dst, $src, $shift\t # int (ndd)" %}
 9284   ins_encode %{
 9285     __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
 9286   %}
 9287   ins_pipe(ialu_reg);
 9288 %}
 9289 
 9290 // Logical Shift Right by 8-bit immediate
 9291 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9292 %{
 9293   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9294   effect(KILL cr);
 9295 
 9296   format %{ "shrl    $dst, $shift" %}
 9297   ins_encode %{
 9298     __ shrl($dst$$Address, $shift$$constant);
 9299   %}
 9300   ins_pipe(ialu_mem_imm);
 9301 %}
 9302 
 9303 // Logical Shift Right by variable
 9304 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9305 %{
 9306   predicate(!VM_Version::supports_bmi2());
 9307   match(Set dst (URShiftI dst shift));
 9308   effect(KILL cr);
 9309 
 9310   format %{ "shrl    $dst, $shift" %}
 9311   ins_encode %{
 9312     __ shrl($dst$$Register);
 9313   %}
 9314   ins_pipe(ialu_reg_reg);
 9315 %}
 9316 
 9317 // Logical Shift Right by variable
 9318 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9319 %{
 9320   predicate(!VM_Version::supports_bmi2());
 9321   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
 9322   effect(KILL cr);
 9323 
 9324   format %{ "shrl    $dst, $shift" %}
 9325   ins_encode %{
 9326     __ shrl($dst$$Address);
 9327   %}
 9328   ins_pipe(ialu_mem_reg);
 9329 %}
 9330 
 9331 instruct shrI_rReg_rReg(rRegI dst, rRegI src, rRegI shift)
 9332 %{
 9333   predicate(VM_Version::supports_bmi2());
 9334   match(Set dst (URShiftI src shift));
 9335 
 9336   format %{ "shrxl   $dst, $src, $shift" %}
 9337   ins_encode %{
 9338     __ shrxl($dst$$Register, $src$$Register, $shift$$Register);
 9339   %}
 9340   ins_pipe(ialu_reg_reg);
 9341 %}
 9342 
 9343 instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
 9344 %{
 9345   predicate(VM_Version::supports_bmi2());
 9346   match(Set dst (URShiftI (LoadI src) shift));
 9347   ins_cost(175);
 9348   format %{ "shrxl   $dst, $src, $shift" %}
 9349   ins_encode %{
 9350     __ shrxl($dst$$Register, $src$$Address, $shift$$Register);
 9351   %}
 9352   ins_pipe(ialu_reg_mem);
 9353 %}
 9354 
 9355 // Long Shift Instructions
 9356 // Shift Left by one, two, three
 9357 instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
 9358 %{
 9359   predicate(!UseAPX);
 9360   match(Set dst (LShiftL dst shift));
 9361   effect(KILL cr);
 9362 
 9363   format %{ "salq    $dst, $shift" %}
 9364   ins_encode %{
 9365     __ salq($dst$$Register, $shift$$constant);
 9366   %}
 9367   ins_pipe(ialu_reg);
 9368 %}
 9369 
 9370 // Shift Left by one, two, three
 9371 instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
 9372 %{
 9373   predicate(UseAPX);
 9374   match(Set dst (LShiftL src shift));
 9375   effect(KILL cr);
 9376 
 9377   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9378   ins_encode %{
 9379     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9380   %}
 9381   ins_pipe(ialu_reg);
 9382 %}
 9383 
 9384 // Shift Left by 8-bit immediate
 9385 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9386 %{
 9387   predicate(!UseAPX);
 9388   match(Set dst (LShiftL dst shift));
 9389   effect(KILL cr);
 9390 
 9391   format %{ "salq    $dst, $shift" %}
 9392   ins_encode %{
 9393     __ salq($dst$$Register, $shift$$constant);
 9394   %}
 9395   ins_pipe(ialu_reg);
 9396 %}
 9397 
 9398 // Shift Left by 8-bit immediate
 9399 instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9400 %{
 9401   predicate(UseAPX);
 9402   match(Set dst (LShiftL src shift));
 9403   effect(KILL cr);
 9404 
 9405   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9406   ins_encode %{
 9407     __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
 9408   %}
 9409   ins_pipe(ialu_reg);
 9410 %}
 9411 
 9412 instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9413 %{
 9414   predicate(UseAPX);
 9415   match(Set dst (LShiftL (LoadL src) shift));
 9416   effect(KILL cr);
 9417 
 9418   format %{ "esalq    $dst, $src, $shift\t# long (ndd)" %}
 9419   ins_encode %{
 9420     __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
 9421   %}
 9422   ins_pipe(ialu_reg);
 9423 %}
 9424 
 9425 // Shift Left by 8-bit immediate
 9426 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9427 %{
 9428   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9429   effect(KILL cr);
 9430 
 9431   format %{ "salq    $dst, $shift" %}
 9432   ins_encode %{
 9433     __ salq($dst$$Address, $shift$$constant);
 9434   %}
 9435   ins_pipe(ialu_mem_imm);
 9436 %}
 9437 
 9438 // Shift Left by variable
 9439 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9440 %{
 9441   predicate(!VM_Version::supports_bmi2());
 9442   match(Set dst (LShiftL dst shift));
 9443   effect(KILL cr);
 9444 
 9445   format %{ "salq    $dst, $shift" %}
 9446   ins_encode %{
 9447     __ salq($dst$$Register);
 9448   %}
 9449   ins_pipe(ialu_reg_reg);
 9450 %}
 9451 
 9452 // Shift Left by variable
 9453 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9454 %{
 9455   predicate(!VM_Version::supports_bmi2());
 9456   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
 9457   effect(KILL cr);
 9458 
 9459   format %{ "salq    $dst, $shift" %}
 9460   ins_encode %{
 9461     __ salq($dst$$Address);
 9462   %}
 9463   ins_pipe(ialu_mem_reg);
 9464 %}
 9465 
 9466 instruct salL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9467 %{
 9468   predicate(VM_Version::supports_bmi2());
 9469   match(Set dst (LShiftL src shift));
 9470 
 9471   format %{ "shlxq   $dst, $src, $shift" %}
 9472   ins_encode %{
 9473     __ shlxq($dst$$Register, $src$$Register, $shift$$Register);
 9474   %}
 9475   ins_pipe(ialu_reg_reg);
 9476 %}
 9477 
 9478 instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9479 %{
 9480   predicate(VM_Version::supports_bmi2());
 9481   match(Set dst (LShiftL (LoadL src) shift));
 9482   ins_cost(175);
 9483   format %{ "shlxq   $dst, $src, $shift" %}
 9484   ins_encode %{
 9485     __ shlxq($dst$$Register, $src$$Address, $shift$$Register);
 9486   %}
 9487   ins_pipe(ialu_reg_mem);
 9488 %}
 9489 
 9490 // Arithmetic Shift Right by 8-bit immediate
 9491 instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
 9492 %{
 9493   predicate(!UseAPX);
 9494   match(Set dst (RShiftL dst shift));
 9495   effect(KILL cr);
 9496 
 9497   format %{ "sarq    $dst, $shift" %}
 9498   ins_encode %{
 9499     __ sarq($dst$$Register, (unsigned char)($shift$$constant & 0x3F));
 9500   %}
 9501   ins_pipe(ialu_mem_imm);
 9502 %}
 9503 
 9504 // Arithmetic Shift Right by 8-bit immediate
 9505 instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
 9506 %{
 9507   predicate(UseAPX);
 9508   match(Set dst (RShiftL src shift));
 9509   effect(KILL cr);
 9510 
 9511   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9512   ins_encode %{
 9513     __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
 9514   %}
 9515   ins_pipe(ialu_mem_imm);
 9516 %}
 9517 
 9518 instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
 9519 %{
 9520   predicate(UseAPX);
 9521   match(Set dst (RShiftL (LoadL src) shift));
 9522   effect(KILL cr);
 9523 
 9524   format %{ "esarq    $dst, $src, $shift\t# long (ndd)" %}
 9525   ins_encode %{
 9526     __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
 9527   %}
 9528   ins_pipe(ialu_mem_imm);
 9529 %}
 9530 
 9531 // Arithmetic Shift Right by 8-bit immediate
 9532 instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
 9533 %{
 9534   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9535   effect(KILL cr);
 9536 
 9537   format %{ "sarq    $dst, $shift" %}
 9538   ins_encode %{
 9539     __ sarq($dst$$Address, (unsigned char)($shift$$constant & 0x3F));
 9540   %}
 9541   ins_pipe(ialu_mem_imm);
 9542 %}
 9543 
 9544 // Arithmetic Shift Right by variable
 9545 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9546 %{
 9547   predicate(!VM_Version::supports_bmi2());
 9548   match(Set dst (RShiftL dst shift));
 9549   effect(KILL cr);
 9550 
 9551   format %{ "sarq    $dst, $shift" %}
 9552   ins_encode %{
 9553     __ sarq($dst$$Register);
 9554   %}
 9555   ins_pipe(ialu_reg_reg);
 9556 %}
 9557 
 9558 // Arithmetic Shift Right by variable
 9559 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9560 %{
 9561   predicate(!VM_Version::supports_bmi2());
 9562   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
 9563   effect(KILL cr);
 9564 
 9565   format %{ "sarq    $dst, $shift" %}
 9566   ins_encode %{
 9567     __ sarq($dst$$Address);
 9568   %}
 9569   ins_pipe(ialu_mem_reg);
 9570 %}
 9571 
 9572 instruct sarL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9573 %{
 9574   predicate(VM_Version::supports_bmi2());
 9575   match(Set dst (RShiftL src shift));
 9576 
 9577   format %{ "sarxq   $dst, $src, $shift" %}
 9578   ins_encode %{
 9579     __ sarxq($dst$$Register, $src$$Register, $shift$$Register);
 9580   %}
 9581   ins_pipe(ialu_reg_reg);
 9582 %}
 9583 
 9584 instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9585 %{
 9586   predicate(VM_Version::supports_bmi2());
 9587   match(Set dst (RShiftL (LoadL src) shift));
 9588   ins_cost(175);
 9589   format %{ "sarxq   $dst, $src, $shift" %}
 9590   ins_encode %{
 9591     __ sarxq($dst$$Register, $src$$Address, $shift$$Register);
 9592   %}
 9593   ins_pipe(ialu_reg_mem);
 9594 %}
 9595 
 9596 // Logical Shift Right by 8-bit immediate
 9597 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
 9598 %{
 9599   predicate(!UseAPX);
 9600   match(Set dst (URShiftL dst shift));
 9601   effect(KILL cr);
 9602 
 9603   format %{ "shrq    $dst, $shift" %}
 9604   ins_encode %{
 9605     __ shrq($dst$$Register, $shift$$constant);
 9606   %}
 9607   ins_pipe(ialu_reg);
 9608 %}
 9609 
 9610 // Logical Shift Right by 8-bit immediate
 9611 instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
 9612 %{
 9613   predicate(UseAPX);
 9614   match(Set dst (URShiftL src shift));
 9615   effect(KILL cr);
 9616 
 9617   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9618   ins_encode %{
 9619     __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
 9620   %}
 9621   ins_pipe(ialu_reg);
 9622 %}
 9623 
 9624 instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
 9625 %{
 9626   predicate(UseAPX);
 9627   match(Set dst (URShiftL (LoadL src) shift));
 9628   effect(KILL cr);
 9629 
 9630   format %{ "eshrq    $dst, $src, $shift\t# long (ndd)" %}
 9631   ins_encode %{
 9632     __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
 9633   %}
 9634   ins_pipe(ialu_reg);
 9635 %}
 9636 
 9637 // Logical Shift Right by 8-bit immediate
 9638 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
 9639 %{
 9640   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9641   effect(KILL cr);
 9642 
 9643   format %{ "shrq    $dst, $shift" %}
 9644   ins_encode %{
 9645     __ shrq($dst$$Address, $shift$$constant);
 9646   %}
 9647   ins_pipe(ialu_mem_imm);
 9648 %}
 9649 
 9650 // Logical Shift Right by variable
 9651 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9652 %{
 9653   predicate(!VM_Version::supports_bmi2());
 9654   match(Set dst (URShiftL dst shift));
 9655   effect(KILL cr);
 9656 
 9657   format %{ "shrq    $dst, $shift" %}
 9658   ins_encode %{
 9659     __ shrq($dst$$Register);
 9660   %}
 9661   ins_pipe(ialu_reg_reg);
 9662 %}
 9663 
 9664 // Logical Shift Right by variable
 9665 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
 9666 %{
 9667   predicate(!VM_Version::supports_bmi2());
 9668   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
 9669   effect(KILL cr);
 9670 
 9671   format %{ "shrq    $dst, $shift" %}
 9672   ins_encode %{
 9673     __ shrq($dst$$Address);
 9674   %}
 9675   ins_pipe(ialu_mem_reg);
 9676 %}
 9677 
 9678 instruct shrL_rReg_rReg(rRegL dst, rRegL src, rRegI shift)
 9679 %{
 9680   predicate(VM_Version::supports_bmi2());
 9681   match(Set dst (URShiftL src shift));
 9682 
 9683   format %{ "shrxq   $dst, $src, $shift" %}
 9684   ins_encode %{
 9685     __ shrxq($dst$$Register, $src$$Register, $shift$$Register);
 9686   %}
 9687   ins_pipe(ialu_reg_reg);
 9688 %}
 9689 
 9690 instruct shrL_mem_rReg(rRegL dst, memory src, rRegI shift)
 9691 %{
 9692   predicate(VM_Version::supports_bmi2());
 9693   match(Set dst (URShiftL (LoadL src) shift));
 9694   ins_cost(175);
 9695   format %{ "shrxq   $dst, $src, $shift" %}
 9696   ins_encode %{
 9697     __ shrxq($dst$$Register, $src$$Address, $shift$$Register);
 9698   %}
 9699   ins_pipe(ialu_reg_mem);
 9700 %}
 9701 
 9702 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
 9703 // This idiom is used by the compiler for the i2b bytecode.
 9704 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
 9705 %{
 9706   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
 9707 
 9708   format %{ "movsbl  $dst, $src\t# i2b" %}
 9709   ins_encode %{
 9710     __ movsbl($dst$$Register, $src$$Register);
 9711   %}
 9712   ins_pipe(ialu_reg_reg);
 9713 %}
 9714 
 9715 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
 9716 // This idiom is used by the compiler the i2s bytecode.
 9717 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
 9718 %{
 9719   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
 9720 
 9721   format %{ "movswl  $dst, $src\t# i2s" %}
 9722   ins_encode %{
 9723     __ movswl($dst$$Register, $src$$Register);
 9724   %}
 9725   ins_pipe(ialu_reg_reg);
 9726 %}
 9727 
 9728 // ROL/ROR instructions
 9729 
 9730 // Rotate left by constant.
 9731 instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9732 %{
 9733   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9734   match(Set dst (RotateLeft dst shift));
 9735   effect(KILL cr);
 9736   format %{ "roll    $dst, $shift" %}
 9737   ins_encode %{
 9738     __ roll($dst$$Register, $shift$$constant);
 9739   %}
 9740   ins_pipe(ialu_reg);
 9741 %}
 9742 
 9743 instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
 9744 %{
 9745   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9746   match(Set dst (RotateLeft src shift));
 9747   format %{ "rolxl   $dst, $src, $shift" %}
 9748   ins_encode %{
 9749     int shift = 32 - ($shift$$constant & 31);
 9750     __ rorxl($dst$$Register, $src$$Register, shift);
 9751   %}
 9752   ins_pipe(ialu_reg_reg);
 9753 %}
 9754 
 9755 instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9756 %{
 9757   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9758   match(Set dst (RotateLeft (LoadI src) shift));
 9759   ins_cost(175);
 9760   format %{ "rolxl   $dst, $src, $shift" %}
 9761   ins_encode %{
 9762     int shift = 32 - ($shift$$constant & 31);
 9763     __ rorxl($dst$$Register, $src$$Address, shift);
 9764   %}
 9765   ins_pipe(ialu_reg_mem);
 9766 %}
 9767 
 9768 // Rotate Left by variable
 9769 instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9770 %{
 9771   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9772   match(Set dst (RotateLeft dst shift));
 9773   effect(KILL cr);
 9774   format %{ "roll    $dst, $shift" %}
 9775   ins_encode %{
 9776     __ roll($dst$$Register);
 9777   %}
 9778   ins_pipe(ialu_reg_reg);
 9779 %}
 9780 
 9781 // Rotate Left by variable
 9782 instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9783 %{
 9784   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9785   match(Set dst (RotateLeft src shift));
 9786   effect(KILL cr);
 9787 
 9788   format %{ "eroll    $dst, $src, $shift\t# rotate left (int ndd)" %}
 9789   ins_encode %{
 9790     __ eroll($dst$$Register, $src$$Register, false);
 9791   %}
 9792   ins_pipe(ialu_reg_reg);
 9793 %}
 9794 
 9795 // Rotate Right by constant.
 9796 instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
 9797 %{
 9798   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9799   match(Set dst (RotateRight dst shift));
 9800   effect(KILL cr);
 9801   format %{ "rorl    $dst, $shift" %}
 9802   ins_encode %{
 9803     __ rorl($dst$$Register, $shift$$constant);
 9804   %}
 9805   ins_pipe(ialu_reg);
 9806 %}
 9807 
 9808 // Rotate Right by constant.
 9809 instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
 9810 %{
 9811   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9812   match(Set dst (RotateRight src shift));
 9813   format %{ "rorxl   $dst, $src, $shift" %}
 9814   ins_encode %{
 9815     __ rorxl($dst$$Register, $src$$Register, $shift$$constant);
 9816   %}
 9817   ins_pipe(ialu_reg_reg);
 9818 %}
 9819 
 9820 instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
 9821 %{
 9822   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
 9823   match(Set dst (RotateRight (LoadI src) shift));
 9824   ins_cost(175);
 9825   format %{ "rorxl   $dst, $src, $shift" %}
 9826   ins_encode %{
 9827     __ rorxl($dst$$Register, $src$$Address, $shift$$constant);
 9828   %}
 9829   ins_pipe(ialu_reg_mem);
 9830 %}
 9831 
 9832 // Rotate Right by variable
 9833 instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
 9834 %{
 9835   predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
 9836   match(Set dst (RotateRight dst shift));
 9837   effect(KILL cr);
 9838   format %{ "rorl    $dst, $shift" %}
 9839   ins_encode %{
 9840     __ rorl($dst$$Register);
 9841   %}
 9842   ins_pipe(ialu_reg_reg);
 9843 %}
 9844 
 9845 // Rotate Right by variable
 9846 instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
 9847 %{
 9848   predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
 9849   match(Set dst (RotateRight src shift));
 9850   effect(KILL cr);
 9851 
 9852   format %{ "erorl    $dst, $src, $shift\t# rotate right(int ndd)" %}
 9853   ins_encode %{
 9854     __ erorl($dst$$Register, $src$$Register, false);
 9855   %}
 9856   ins_pipe(ialu_reg_reg);
 9857 %}
 9858 
 9859 // Rotate Left by constant.
 9860 instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9861 %{
 9862   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9863   match(Set dst (RotateLeft dst shift));
 9864   effect(KILL cr);
 9865   format %{ "rolq    $dst, $shift" %}
 9866   ins_encode %{
 9867     __ rolq($dst$$Register, $shift$$constant);
 9868   %}
 9869   ins_pipe(ialu_reg);
 9870 %}
 9871 
 9872 instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
 9873 %{
 9874   predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9875   match(Set dst (RotateLeft src shift));
 9876   format %{ "rolxq   $dst, $src, $shift" %}
 9877   ins_encode %{
 9878     int shift = 64 - ($shift$$constant & 63);
 9879     __ rorxq($dst$$Register, $src$$Register, shift);
 9880   %}
 9881   ins_pipe(ialu_reg_reg);
 9882 %}
 9883 
 9884 instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9885 %{
 9886   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9887   match(Set dst (RotateLeft (LoadL src) shift));
 9888   ins_cost(175);
 9889   format %{ "rolxq   $dst, $src, $shift" %}
 9890   ins_encode %{
 9891     int shift = 64 - ($shift$$constant & 63);
 9892     __ rorxq($dst$$Register, $src$$Address, shift);
 9893   %}
 9894   ins_pipe(ialu_reg_mem);
 9895 %}
 9896 
 9897 // Rotate Left by variable
 9898 instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9899 %{
 9900   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9901   match(Set dst (RotateLeft dst shift));
 9902   effect(KILL cr);
 9903   format %{ "rolq    $dst, $shift" %}
 9904   ins_encode %{
 9905     __ rolq($dst$$Register);
 9906   %}
 9907   ins_pipe(ialu_reg_reg);
 9908 %}
 9909 
 9910 // Rotate Left by variable
 9911 instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9912 %{
 9913   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9914   match(Set dst (RotateLeft src shift));
 9915   effect(KILL cr);
 9916 
 9917   format %{ "erolq    $dst, $src, $shift\t# rotate left(long ndd)" %}
 9918   ins_encode %{
 9919     __ erolq($dst$$Register, $src$$Register, false);
 9920   %}
 9921   ins_pipe(ialu_reg_reg);
 9922 %}
 9923 
 9924 // Rotate Right by constant.
 9925 instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
 9926 %{
 9927   predicate(!VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9928   match(Set dst (RotateRight dst shift));
 9929   effect(KILL cr);
 9930   format %{ "rorq    $dst, $shift" %}
 9931   ins_encode %{
 9932     __ rorq($dst$$Register, $shift$$constant);
 9933   %}
 9934   ins_pipe(ialu_reg);
 9935 %}
 9936 
 9937 // Rotate Right by constant
 9938 instruct rorL_immI8(rRegL dst, rRegL src, immI8 shift)
 9939 %{
 9940   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9941   match(Set dst (RotateRight src shift));
 9942   format %{ "rorxq   $dst, $src, $shift" %}
 9943   ins_encode %{
 9944     __ rorxq($dst$$Register, $src$$Register, $shift$$constant);
 9945   %}
 9946   ins_pipe(ialu_reg_reg);
 9947 %}
 9948 
 9949 instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
 9950 %{
 9951   predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
 9952   match(Set dst (RotateRight (LoadL src) shift));
 9953   ins_cost(175);
 9954   format %{ "rorxq   $dst, $src, $shift" %}
 9955   ins_encode %{
 9956     __ rorxq($dst$$Register, $src$$Address, $shift$$constant);
 9957   %}
 9958   ins_pipe(ialu_reg_mem);
 9959 %}
 9960 
 9961 // Rotate Right by variable
 9962 instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
 9963 %{
 9964   predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9965   match(Set dst (RotateRight dst shift));
 9966   effect(KILL cr);
 9967   format %{ "rorq    $dst, $shift" %}
 9968   ins_encode %{
 9969     __ rorq($dst$$Register);
 9970   %}
 9971   ins_pipe(ialu_reg_reg);
 9972 %}
 9973 
 9974 // Rotate Right by variable
 9975 instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
 9976 %{
 9977   predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
 9978   match(Set dst (RotateRight src shift));
 9979   effect(KILL cr);
 9980 
 9981   format %{ "erorq    $dst, $src, $shift\t# rotate right(long ndd)" %}
 9982   ins_encode %{
 9983     __ erorq($dst$$Register, $src$$Register, false);
 9984   %}
 9985   ins_pipe(ialu_reg_reg);
 9986 %}
 9987 
 9988 //----------------------------- CompressBits/ExpandBits ------------------------
 9989 
 9990 instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
 9991   predicate(n->bottom_type()->isa_long());
 9992   match(Set dst (CompressBits src mask));
 9993   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
 9994   ins_encode %{
 9995     __ pextq($dst$$Register, $src$$Register, $mask$$Register);
 9996   %}
 9997   ins_pipe( pipe_slow );
 9998 %}
 9999 
10000 instruct expandBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
10001   predicate(n->bottom_type()->isa_long());
10002   match(Set dst (ExpandBits src mask));
10003   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10004   ins_encode %{
10005     __ pdepq($dst$$Register, $src$$Register, $mask$$Register);
10006   %}
10007   ins_pipe( pipe_slow );
10008 %}
10009 
10010 instruct compressBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10011   predicate(n->bottom_type()->isa_long());
10012   match(Set dst (CompressBits src (LoadL mask)));
10013   format %{ "pextq  $dst, $src, $mask\t! parallel bit extract" %}
10014   ins_encode %{
10015     __ pextq($dst$$Register, $src$$Register, $mask$$Address);
10016   %}
10017   ins_pipe( pipe_slow );
10018 %}
10019 
10020 instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
10021   predicate(n->bottom_type()->isa_long());
10022   match(Set dst (ExpandBits src (LoadL mask)));
10023   format %{ "pdepq  $dst, $src, $mask\t! parallel bit deposit" %}
10024   ins_encode %{
10025     __ pdepq($dst$$Register, $src$$Register, $mask$$Address);
10026   %}
10027   ins_pipe( pipe_slow );
10028 %}
10029 
10030 
10031 // Logical Instructions
10032 
10033 // Integer Logical Instructions
10034 
10035 // And Instructions
10036 // And Register with Register
10037 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10038 %{
10039   predicate(!UseAPX);
10040   match(Set dst (AndI dst src));
10041   effect(KILL cr);
10042   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10043 
10044   format %{ "andl    $dst, $src\t# int" %}
10045   ins_encode %{
10046     __ andl($dst$$Register, $src$$Register);
10047   %}
10048   ins_pipe(ialu_reg_reg);
10049 %}
10050 
10051 // And Register with Register using New Data Destination (NDD)
10052 instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10053 %{
10054   predicate(UseAPX);
10055   match(Set dst (AndI src1 src2));
10056   effect(KILL cr);
10057   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10058 
10059   format %{ "eandl     $dst, $src1, $src2\t# int ndd" %}
10060   ins_encode %{
10061     __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
10062 
10063   %}
10064   ins_pipe(ialu_reg_reg);
10065 %}
10066 
10067 // And Register with Immediate 255
10068 instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
10069 %{
10070   match(Set dst (AndI src mask));
10071 
10072   format %{ "movzbl  $dst, $src\t# int & 0xFF" %}
10073   ins_encode %{
10074     __ movzbl($dst$$Register, $src$$Register);
10075   %}
10076   ins_pipe(ialu_reg);
10077 %}
10078 
10079 // And Register with Immediate 255 and promote to long
10080 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
10081 %{
10082   match(Set dst (ConvI2L (AndI src mask)));
10083 
10084   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
10085   ins_encode %{
10086     __ movzbl($dst$$Register, $src$$Register);
10087   %}
10088   ins_pipe(ialu_reg);
10089 %}
10090 
10091 // And Register with Immediate 65535
10092 instruct andI_rReg_imm65535(rRegI dst, rRegI src, immI_65535 mask)
10093 %{
10094   match(Set dst (AndI src mask));
10095 
10096   format %{ "movzwl  $dst, $src\t# int & 0xFFFF" %}
10097   ins_encode %{
10098     __ movzwl($dst$$Register, $src$$Register);
10099   %}
10100   ins_pipe(ialu_reg);
10101 %}
10102 
10103 // And Register with Immediate 65535 and promote to long
10104 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
10105 %{
10106   match(Set dst (ConvI2L (AndI src mask)));
10107 
10108   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
10109   ins_encode %{
10110     __ movzwl($dst$$Register, $src$$Register);
10111   %}
10112   ins_pipe(ialu_reg);
10113 %}
10114 
10115 // Can skip int2long conversions after AND with small bitmask
10116 instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rRegI tmp, rFlagsReg cr)
10117 %{
10118   predicate(VM_Version::supports_bmi2());
10119   ins_cost(125);
10120   effect(TEMP tmp, KILL cr);
10121   match(Set dst (ConvI2L (AndI src mask)));
10122   format %{ "bzhiq $dst, $src, $mask \t# using $tmp as TEMP, int &  immI_Pow2M1 -> long" %}
10123   ins_encode %{
10124     __ movl($tmp$$Register, exact_log2($mask$$constant + 1));
10125     __ bzhiq($dst$$Register, $src$$Register, $tmp$$Register);
10126   %}
10127   ins_pipe(ialu_reg_reg);
10128 %}
10129 
10130 // And Register with Immediate
10131 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10132 %{
10133   predicate(!UseAPX);
10134   match(Set dst (AndI dst src));
10135   effect(KILL cr);
10136   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10137 
10138   format %{ "andl    $dst, $src\t# int" %}
10139   ins_encode %{
10140     __ andl($dst$$Register, $src$$constant);
10141   %}
10142   ins_pipe(ialu_reg);
10143 %}
10144 
10145 instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10146 %{
10147   predicate(UseAPX);
10148   match(Set dst (AndI src1 src2));
10149   effect(KILL cr);
10150   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10151 
10152   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10153   ins_encode %{
10154     __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
10155   %}
10156   ins_pipe(ialu_reg);
10157 %}
10158 
10159 instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10160 %{
10161   predicate(UseAPX);
10162   match(Set dst (AndI (LoadI src1) src2));
10163   effect(KILL cr);
10164   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10165 
10166   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10167   ins_encode %{
10168     __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
10169   %}
10170   ins_pipe(ialu_reg);
10171 %}
10172 
10173 // And Register with Memory
10174 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10175 %{
10176   predicate(!UseAPX);
10177   match(Set dst (AndI dst (LoadI src)));
10178   effect(KILL cr);
10179   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10180 
10181   ins_cost(150);
10182   format %{ "andl    $dst, $src\t# int" %}
10183   ins_encode %{
10184     __ andl($dst$$Register, $src$$Address);
10185   %}
10186   ins_pipe(ialu_reg_mem);
10187 %}
10188 
10189 instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10190 %{
10191   predicate(UseAPX);
10192   match(Set dst (AndI src1 (LoadI src2)));
10193   effect(KILL cr);
10194   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10195 
10196   ins_cost(150);
10197   format %{ "eandl    $dst, $src1, $src2\t# int ndd" %}
10198   ins_encode %{
10199     __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
10200   %}
10201   ins_pipe(ialu_reg_mem);
10202 %}
10203 
10204 // And Memory with Register
10205 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10206 %{
10207   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
10208   effect(KILL cr);
10209   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10210 
10211   ins_cost(150);
10212   format %{ "andb    $dst, $src\t# byte" %}
10213   ins_encode %{
10214     __ andb($dst$$Address, $src$$Register);
10215   %}
10216   ins_pipe(ialu_mem_reg);
10217 %}
10218 
10219 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10220 %{
10221   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10222   effect(KILL cr);
10223   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10224 
10225   ins_cost(150);
10226   format %{ "andl    $dst, $src\t# int" %}
10227   ins_encode %{
10228     __ andl($dst$$Address, $src$$Register);
10229   %}
10230   ins_pipe(ialu_mem_reg);
10231 %}
10232 
10233 // And Memory with Immediate
10234 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
10235 %{
10236   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
10237   effect(KILL cr);
10238   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10239 
10240   ins_cost(125);
10241   format %{ "andl    $dst, $src\t# int" %}
10242   ins_encode %{
10243     __ andl($dst$$Address, $src$$constant);
10244   %}
10245   ins_pipe(ialu_mem_imm);
10246 %}
10247 
10248 // BMI1 instructions
10249 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
10250   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
10251   predicate(UseBMI1Instructions);
10252   effect(KILL cr);
10253   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10254 
10255   ins_cost(125);
10256   format %{ "andnl  $dst, $src1, $src2" %}
10257 
10258   ins_encode %{
10259     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
10260   %}
10261   ins_pipe(ialu_reg_mem);
10262 %}
10263 
10264 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
10265   match(Set dst (AndI (XorI src1 minus_1) src2));
10266   predicate(UseBMI1Instructions);
10267   effect(KILL cr);
10268   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10269 
10270   format %{ "andnl  $dst, $src1, $src2" %}
10271 
10272   ins_encode %{
10273     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
10274   %}
10275   ins_pipe(ialu_reg);
10276 %}
10277 
10278 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, rFlagsReg cr) %{
10279   match(Set dst (AndI (SubI imm_zero src) src));
10280   predicate(UseBMI1Instructions);
10281   effect(KILL cr);
10282   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10283 
10284   format %{ "blsil  $dst, $src" %}
10285 
10286   ins_encode %{
10287     __ blsil($dst$$Register, $src$$Register);
10288   %}
10289   ins_pipe(ialu_reg);
10290 %}
10291 
10292 instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, rFlagsReg cr) %{
10293   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
10294   predicate(UseBMI1Instructions);
10295   effect(KILL cr);
10296   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10297 
10298   ins_cost(125);
10299   format %{ "blsil  $dst, $src" %}
10300 
10301   ins_encode %{
10302     __ blsil($dst$$Register, $src$$Address);
10303   %}
10304   ins_pipe(ialu_reg_mem);
10305 %}
10306 
10307 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10308 %{
10309   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
10310   predicate(UseBMI1Instructions);
10311   effect(KILL cr);
10312   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10313 
10314   ins_cost(125);
10315   format %{ "blsmskl $dst, $src" %}
10316 
10317   ins_encode %{
10318     __ blsmskl($dst$$Register, $src$$Address);
10319   %}
10320   ins_pipe(ialu_reg_mem);
10321 %}
10322 
10323 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10324 %{
10325   match(Set dst (XorI (AddI src minus_1) src));
10326   predicate(UseBMI1Instructions);
10327   effect(KILL cr);
10328   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10329 
10330   format %{ "blsmskl $dst, $src" %}
10331 
10332   ins_encode %{
10333     __ blsmskl($dst$$Register, $src$$Register);
10334   %}
10335 
10336   ins_pipe(ialu_reg);
10337 %}
10338 
10339 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
10340 %{
10341   match(Set dst (AndI (AddI src minus_1) src) );
10342   predicate(UseBMI1Instructions);
10343   effect(KILL cr);
10344   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10345 
10346   format %{ "blsrl  $dst, $src" %}
10347 
10348   ins_encode %{
10349     __ blsrl($dst$$Register, $src$$Register);
10350   %}
10351 
10352   ins_pipe(ialu_reg_mem);
10353 %}
10354 
10355 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
10356 %{
10357   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
10358   predicate(UseBMI1Instructions);
10359   effect(KILL cr);
10360   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10361 
10362   ins_cost(125);
10363   format %{ "blsrl  $dst, $src" %}
10364 
10365   ins_encode %{
10366     __ blsrl($dst$$Register, $src$$Address);
10367   %}
10368 
10369   ins_pipe(ialu_reg);
10370 %}
10371 
10372 // Or Instructions
10373 // Or Register with Register
10374 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10375 %{
10376   predicate(!UseAPX);
10377   match(Set dst (OrI dst src));
10378   effect(KILL cr);
10379   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10380 
10381   format %{ "orl     $dst, $src\t# int" %}
10382   ins_encode %{
10383     __ orl($dst$$Register, $src$$Register);
10384   %}
10385   ins_pipe(ialu_reg_reg);
10386 %}
10387 
10388 // Or Register with Register using New Data Destination (NDD)
10389 instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10390 %{
10391   predicate(UseAPX);
10392   match(Set dst (OrI src1 src2));
10393   effect(KILL cr);
10394   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10395 
10396   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10397   ins_encode %{
10398     __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10399   %}
10400   ins_pipe(ialu_reg_reg);
10401 %}
10402 
10403 // Or Register with Immediate
10404 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10405 %{
10406   predicate(!UseAPX);
10407   match(Set dst (OrI dst src));
10408   effect(KILL cr);
10409   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10410 
10411   format %{ "orl     $dst, $src\t# int" %}
10412   ins_encode %{
10413     __ orl($dst$$Register, $src$$constant);
10414   %}
10415   ins_pipe(ialu_reg);
10416 %}
10417 
10418 instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10419 %{
10420   predicate(UseAPX);
10421   match(Set dst (OrI src1 src2));
10422   effect(KILL cr);
10423   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10424 
10425   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10426   ins_encode %{
10427     __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10428   %}
10429   ins_pipe(ialu_reg);
10430 %}
10431 
10432 instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
10433 %{
10434   predicate(UseAPX);
10435   match(Set dst (OrI src1 src2));
10436   effect(KILL cr);
10437   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10438 
10439   format %{ "eorl     $dst, $src2, $src1\t# int ndd" %}
10440   ins_encode %{
10441     __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
10442   %}
10443   ins_pipe(ialu_reg);
10444 %}
10445 
10446 instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10447 %{
10448   predicate(UseAPX);
10449   match(Set dst (OrI (LoadI src1) src2));
10450   effect(KILL cr);
10451   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10452 
10453   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10454   ins_encode %{
10455     __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10456   %}
10457   ins_pipe(ialu_reg);
10458 %}
10459 
10460 // Or Register with Memory
10461 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10462 %{
10463   predicate(!UseAPX);
10464   match(Set dst (OrI dst (LoadI src)));
10465   effect(KILL cr);
10466   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10467 
10468   ins_cost(150);
10469   format %{ "orl     $dst, $src\t# int" %}
10470   ins_encode %{
10471     __ orl($dst$$Register, $src$$Address);
10472   %}
10473   ins_pipe(ialu_reg_mem);
10474 %}
10475 
10476 instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10477 %{
10478   predicate(UseAPX);
10479   match(Set dst (OrI src1 (LoadI src2)));
10480   effect(KILL cr);
10481   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10482 
10483   ins_cost(150);
10484   format %{ "eorl     $dst, $src1, $src2\t# int ndd" %}
10485   ins_encode %{
10486     __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10487   %}
10488   ins_pipe(ialu_reg_mem);
10489 %}
10490 
10491 // Or Memory with Register
10492 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10493 %{
10494   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
10495   effect(KILL cr);
10496   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10497 
10498   ins_cost(150);
10499   format %{ "orb    $dst, $src\t# byte" %}
10500   ins_encode %{
10501     __ orb($dst$$Address, $src$$Register);
10502   %}
10503   ins_pipe(ialu_mem_reg);
10504 %}
10505 
10506 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10507 %{
10508   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10509   effect(KILL cr);
10510   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10511 
10512   ins_cost(150);
10513   format %{ "orl     $dst, $src\t# int" %}
10514   ins_encode %{
10515     __ orl($dst$$Address, $src$$Register);
10516   %}
10517   ins_pipe(ialu_mem_reg);
10518 %}
10519 
10520 // Or Memory with Immediate
10521 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
10522 %{
10523   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
10524   effect(KILL cr);
10525   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10526 
10527   ins_cost(125);
10528   format %{ "orl     $dst, $src\t# int" %}
10529   ins_encode %{
10530     __ orl($dst$$Address, $src$$constant);
10531   %}
10532   ins_pipe(ialu_mem_imm);
10533 %}
10534 
10535 // Xor Instructions
10536 // Xor Register with Register
10537 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
10538 %{
10539   predicate(!UseAPX);
10540   match(Set dst (XorI dst src));
10541   effect(KILL cr);
10542   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10543 
10544   format %{ "xorl    $dst, $src\t# int" %}
10545   ins_encode %{
10546     __ xorl($dst$$Register, $src$$Register);
10547   %}
10548   ins_pipe(ialu_reg_reg);
10549 %}
10550 
10551 // Xor Register with Register using New Data Destination (NDD)
10552 instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
10553 %{
10554   predicate(UseAPX);
10555   match(Set dst (XorI src1 src2));
10556   effect(KILL cr);
10557   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10558 
10559   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10560   ins_encode %{
10561     __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
10562   %}
10563   ins_pipe(ialu_reg_reg);
10564 %}
10565 
10566 // Xor Register with Immediate -1
10567 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
10568 %{
10569   predicate(!UseAPX);
10570   match(Set dst (XorI dst imm));
10571 
10572   format %{ "notl    $dst" %}
10573   ins_encode %{
10574      __ notl($dst$$Register);
10575   %}
10576   ins_pipe(ialu_reg);
10577 %}
10578 
10579 instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
10580 %{
10581   match(Set dst (XorI src imm));
10582   predicate(UseAPX);
10583 
10584   format %{ "enotl    $dst, $src" %}
10585   ins_encode %{
10586      __ enotl($dst$$Register, $src$$Register);
10587   %}
10588   ins_pipe(ialu_reg);
10589 %}
10590 
10591 // Xor Register with Immediate
10592 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
10593 %{
10594   // Strict predicate check to make selection of xorI_rReg_im1 cost agnostic if immI src is -1.
10595   predicate(!UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10596   match(Set dst (XorI dst src));
10597   effect(KILL cr);
10598   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10599 
10600   format %{ "xorl    $dst, $src\t# int" %}
10601   ins_encode %{
10602     __ xorl($dst$$Register, $src$$constant);
10603   %}
10604   ins_pipe(ialu_reg);
10605 %}
10606 
10607 instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
10608 %{
10609   // Strict predicate check to make selection of xorI_rReg_im1_ndd cost agnostic if immI src2 is -1.
10610   predicate(UseAPX && n->in(2)->bottom_type()->is_int()->get_con() != -1);
10611   match(Set dst (XorI src1 src2));
10612   effect(KILL cr);
10613   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10614 
10615   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10616   ins_encode %{
10617     __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
10618   %}
10619   ins_pipe(ialu_reg);
10620 %}
10621 
10622 // Xor Memory with Immediate
10623 instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
10624 %{
10625   predicate(UseAPX);
10626   match(Set dst (XorI (LoadI src1) src2));
10627   effect(KILL cr);
10628   ins_cost(150);
10629   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10630 
10631   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10632   ins_encode %{
10633     __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
10634   %}
10635   ins_pipe(ialu_reg);
10636 %}
10637 
10638 // Xor Register with Memory
10639 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
10640 %{
10641   predicate(!UseAPX);
10642   match(Set dst (XorI dst (LoadI src)));
10643   effect(KILL cr);
10644   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10645 
10646   ins_cost(150);
10647   format %{ "xorl    $dst, $src\t# int" %}
10648   ins_encode %{
10649     __ xorl($dst$$Register, $src$$Address);
10650   %}
10651   ins_pipe(ialu_reg_mem);
10652 %}
10653 
10654 instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
10655 %{
10656   predicate(UseAPX);
10657   match(Set dst (XorI src1 (LoadI src2)));
10658   effect(KILL cr);
10659   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10660 
10661   ins_cost(150);
10662   format %{ "exorl    $dst, $src1, $src2\t# int ndd" %}
10663   ins_encode %{
10664     __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
10665   %}
10666   ins_pipe(ialu_reg_mem);
10667 %}
10668 
10669 // Xor Memory with Register
10670 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10671 %{
10672   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
10673   effect(KILL cr);
10674   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10675 
10676   ins_cost(150);
10677   format %{ "xorb    $dst, $src\t# byte" %}
10678   ins_encode %{
10679     __ xorb($dst$$Address, $src$$Register);
10680   %}
10681   ins_pipe(ialu_mem_reg);
10682 %}
10683 
10684 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
10685 %{
10686   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10687   effect(KILL cr);
10688   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10689 
10690   ins_cost(150);
10691   format %{ "xorl    $dst, $src\t# int" %}
10692   ins_encode %{
10693     __ xorl($dst$$Address, $src$$Register);
10694   %}
10695   ins_pipe(ialu_mem_reg);
10696 %}
10697 
10698 // Xor Memory with Immediate
10699 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
10700 %{
10701   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
10702   effect(KILL cr);
10703   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10704 
10705   ins_cost(125);
10706   format %{ "xorl    $dst, $src\t# int" %}
10707   ins_encode %{
10708     __ xorl($dst$$Address, $src$$constant);
10709   %}
10710   ins_pipe(ialu_mem_imm);
10711 %}
10712 
10713 
10714 // Long Logical Instructions
10715 
10716 // And Instructions
10717 // And Register with Register
10718 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10719 %{
10720   predicate(!UseAPX);
10721   match(Set dst (AndL dst src));
10722   effect(KILL cr);
10723   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10724 
10725   format %{ "andq    $dst, $src\t# long" %}
10726   ins_encode %{
10727     __ andq($dst$$Register, $src$$Register);
10728   %}
10729   ins_pipe(ialu_reg_reg);
10730 %}
10731 
10732 // And Register with Register using New Data Destination (NDD)
10733 instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
10734 %{
10735   predicate(UseAPX);
10736   match(Set dst (AndL src1 src2));
10737   effect(KILL cr);
10738   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10739 
10740   format %{ "eandq     $dst, $src1, $src2\t# long ndd" %}
10741   ins_encode %{
10742     __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
10743 
10744   %}
10745   ins_pipe(ialu_reg_reg);
10746 %}
10747 
10748 // And Register with Immediate 255
10749 instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
10750 %{
10751   match(Set dst (AndL src mask));
10752 
10753   format %{ "movzbl  $dst, $src\t# long & 0xFF" %}
10754   ins_encode %{
10755     // movzbl zeroes out the upper 32-bit and does not need REX.W
10756     __ movzbl($dst$$Register, $src$$Register);
10757   %}
10758   ins_pipe(ialu_reg);
10759 %}
10760 
10761 // And Register with Immediate 65535
10762 instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
10763 %{
10764   match(Set dst (AndL src mask));
10765 
10766   format %{ "movzwl  $dst, $src\t# long & 0xFFFF" %}
10767   ins_encode %{
10768     // movzwl zeroes out the upper 32-bit and does not need REX.W
10769     __ movzwl($dst$$Register, $src$$Register);
10770   %}
10771   ins_pipe(ialu_reg);
10772 %}
10773 
10774 // And Register with Immediate
10775 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10776 %{
10777   predicate(!UseAPX);
10778   match(Set dst (AndL dst src));
10779   effect(KILL cr);
10780   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10781 
10782   format %{ "andq    $dst, $src\t# long" %}
10783   ins_encode %{
10784     __ andq($dst$$Register, $src$$constant);
10785   %}
10786   ins_pipe(ialu_reg);
10787 %}
10788 
10789 instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
10790 %{
10791   predicate(UseAPX);
10792   match(Set dst (AndL src1 src2));
10793   effect(KILL cr);
10794   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10795 
10796   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10797   ins_encode %{
10798     __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
10799   %}
10800   ins_pipe(ialu_reg);
10801 %}
10802 
10803 instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
10804 %{
10805   predicate(UseAPX);
10806   match(Set dst (AndL (LoadL src1) src2));
10807   effect(KILL cr);
10808   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10809 
10810   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10811   ins_encode %{
10812     __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
10813   %}
10814   ins_pipe(ialu_reg);
10815 %}
10816 
10817 // And Register with Memory
10818 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10819 %{
10820   predicate(!UseAPX);
10821   match(Set dst (AndL dst (LoadL src)));
10822   effect(KILL cr);
10823   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10824 
10825   ins_cost(150);
10826   format %{ "andq    $dst, $src\t# long" %}
10827   ins_encode %{
10828     __ andq($dst$$Register, $src$$Address);
10829   %}
10830   ins_pipe(ialu_reg_mem);
10831 %}
10832 
10833 instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
10834 %{
10835   predicate(UseAPX);
10836   match(Set dst (AndL src1 (LoadL src2)));
10837   effect(KILL cr);
10838   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10839 
10840   ins_cost(150);
10841   format %{ "eandq    $dst, $src1, $src2\t# long ndd" %}
10842   ins_encode %{
10843     __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
10844   %}
10845   ins_pipe(ialu_reg_mem);
10846 %}
10847 
10848 // And Memory with Register
10849 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10850 %{
10851   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10852   effect(KILL cr);
10853   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10854 
10855   ins_cost(150);
10856   format %{ "andq    $dst, $src\t# long" %}
10857   ins_encode %{
10858     __ andq($dst$$Address, $src$$Register);
10859   %}
10860   ins_pipe(ialu_mem_reg);
10861 %}
10862 
10863 // And Memory with Immediate
10864 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10865 %{
10866   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10867   effect(KILL cr);
10868   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10869 
10870   ins_cost(125);
10871   format %{ "andq    $dst, $src\t# long" %}
10872   ins_encode %{
10873     __ andq($dst$$Address, $src$$constant);
10874   %}
10875   ins_pipe(ialu_mem_imm);
10876 %}
10877 
10878 instruct btrL_mem_imm(memory dst, immL_NotPow2 con, rFlagsReg cr)
10879 %{
10880   // con should be a pure 64-bit immediate given that not(con) is a power of 2
10881   // because AND/OR works well enough for 8/32-bit values.
10882   predicate(log2i_graceful(~n->in(3)->in(2)->get_long()) > 30);
10883 
10884   match(Set dst (StoreL dst (AndL (LoadL dst) con)));
10885   effect(KILL cr);
10886 
10887   ins_cost(125);
10888   format %{ "btrq    $dst, log2(not($con))\t# long" %}
10889   ins_encode %{
10890     __ btrq($dst$$Address, log2i_exact((julong)~$con$$constant));
10891   %}
10892   ins_pipe(ialu_mem_imm);
10893 %}
10894 
10895 // BMI1 instructions
10896 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10897   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10898   predicate(UseBMI1Instructions);
10899   effect(KILL cr);
10900   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10901 
10902   ins_cost(125);
10903   format %{ "andnq  $dst, $src1, $src2" %}
10904 
10905   ins_encode %{
10906     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10907   %}
10908   ins_pipe(ialu_reg_mem);
10909 %}
10910 
10911 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10912   match(Set dst (AndL (XorL src1 minus_1) src2));
10913   predicate(UseBMI1Instructions);
10914   effect(KILL cr);
10915   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
10916 
10917   format %{ "andnq  $dst, $src1, $src2" %}
10918 
10919   ins_encode %{
10920   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10921   %}
10922   ins_pipe(ialu_reg_mem);
10923 %}
10924 
10925 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10926   match(Set dst (AndL (SubL imm_zero src) src));
10927   predicate(UseBMI1Instructions);
10928   effect(KILL cr);
10929   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10930 
10931   format %{ "blsiq  $dst, $src" %}
10932 
10933   ins_encode %{
10934     __ blsiq($dst$$Register, $src$$Register);
10935   %}
10936   ins_pipe(ialu_reg);
10937 %}
10938 
10939 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10940   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10941   predicate(UseBMI1Instructions);
10942   effect(KILL cr);
10943   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10944 
10945   ins_cost(125);
10946   format %{ "blsiq  $dst, $src" %}
10947 
10948   ins_encode %{
10949     __ blsiq($dst$$Register, $src$$Address);
10950   %}
10951   ins_pipe(ialu_reg_mem);
10952 %}
10953 
10954 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10955 %{
10956   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10957   predicate(UseBMI1Instructions);
10958   effect(KILL cr);
10959   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10960 
10961   ins_cost(125);
10962   format %{ "blsmskq $dst, $src" %}
10963 
10964   ins_encode %{
10965     __ blsmskq($dst$$Register, $src$$Address);
10966   %}
10967   ins_pipe(ialu_reg_mem);
10968 %}
10969 
10970 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10971 %{
10972   match(Set dst (XorL (AddL src minus_1) src));
10973   predicate(UseBMI1Instructions);
10974   effect(KILL cr);
10975   flag(PD::Flag_sets_sign_flag, PD::Flag_clears_zero_flag, PD::Flag_clears_overflow_flag);
10976 
10977   format %{ "blsmskq $dst, $src" %}
10978 
10979   ins_encode %{
10980     __ blsmskq($dst$$Register, $src$$Register);
10981   %}
10982 
10983   ins_pipe(ialu_reg);
10984 %}
10985 
10986 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10987 %{
10988   match(Set dst (AndL (AddL src minus_1) src) );
10989   predicate(UseBMI1Instructions);
10990   effect(KILL cr);
10991   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
10992 
10993   format %{ "blsrq  $dst, $src" %}
10994 
10995   ins_encode %{
10996     __ blsrq($dst$$Register, $src$$Register);
10997   %}
10998 
10999   ins_pipe(ialu_reg);
11000 %}
11001 
11002 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
11003 %{
11004   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
11005   predicate(UseBMI1Instructions);
11006   effect(KILL cr);
11007   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_clears_overflow_flag);
11008 
11009   ins_cost(125);
11010   format %{ "blsrq  $dst, $src" %}
11011 
11012   ins_encode %{
11013     __ blsrq($dst$$Register, $src$$Address);
11014   %}
11015 
11016   ins_pipe(ialu_reg);
11017 %}
11018 
11019 // Or Instructions
11020 // Or Register with Register
11021 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11022 %{
11023   predicate(!UseAPX);
11024   match(Set dst (OrL dst src));
11025   effect(KILL cr);
11026   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11027 
11028   format %{ "orq     $dst, $src\t# long" %}
11029   ins_encode %{
11030     __ orq($dst$$Register, $src$$Register);
11031   %}
11032   ins_pipe(ialu_reg_reg);
11033 %}
11034 
11035 // Or Register with Register using New Data Destination (NDD)
11036 instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11037 %{
11038   predicate(UseAPX);
11039   match(Set dst (OrL src1 src2));
11040   effect(KILL cr);
11041   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11042 
11043   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11044   ins_encode %{
11045     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11046 
11047   %}
11048   ins_pipe(ialu_reg_reg);
11049 %}
11050 
11051 // Use any_RegP to match R15 (TLS register) without spilling.
11052 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
11053   match(Set dst (OrL dst (CastP2X src)));
11054   effect(KILL cr);
11055   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11056 
11057   format %{ "orq     $dst, $src\t# long" %}
11058   ins_encode %{
11059     __ orq($dst$$Register, $src$$Register);
11060   %}
11061   ins_pipe(ialu_reg_reg);
11062 %}
11063 
11064 instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
11065   match(Set dst (OrL src1 (CastP2X src2)));
11066   effect(KILL cr);
11067   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11068 
11069   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11070   ins_encode %{
11071     __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11072   %}
11073   ins_pipe(ialu_reg_reg);
11074 %}
11075 
11076 // Or Register with Immediate
11077 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11078 %{
11079   predicate(!UseAPX);
11080   match(Set dst (OrL dst src));
11081   effect(KILL cr);
11082   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11083 
11084   format %{ "orq     $dst, $src\t# long" %}
11085   ins_encode %{
11086     __ orq($dst$$Register, $src$$constant);
11087   %}
11088   ins_pipe(ialu_reg);
11089 %}
11090 
11091 instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11092 %{
11093   predicate(UseAPX);
11094   match(Set dst (OrL src1 src2));
11095   effect(KILL cr);
11096   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11097 
11098   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11099   ins_encode %{
11100     __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11101   %}
11102   ins_pipe(ialu_reg);
11103 %}
11104 
11105 instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
11106 %{
11107   predicate(UseAPX);
11108   match(Set dst (OrL src1 src2));
11109   effect(KILL cr);
11110   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11111 
11112   format %{ "eorq     $dst, $src2, $src1\t# long ndd" %}
11113   ins_encode %{
11114     __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
11115   %}
11116   ins_pipe(ialu_reg);
11117 %}
11118 
11119 // Or Memory with Immediate
11120 instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11121 %{
11122   predicate(UseAPX);
11123   match(Set dst (OrL (LoadL src1) src2));
11124   effect(KILL cr);
11125   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11126 
11127   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11128   ins_encode %{
11129     __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11130   %}
11131   ins_pipe(ialu_reg);
11132 %}
11133 
11134 // Or Register with Memory
11135 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11136 %{
11137   predicate(!UseAPX);
11138   match(Set dst (OrL dst (LoadL src)));
11139   effect(KILL cr);
11140   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11141 
11142   ins_cost(150);
11143   format %{ "orq     $dst, $src\t# long" %}
11144   ins_encode %{
11145     __ orq($dst$$Register, $src$$Address);
11146   %}
11147   ins_pipe(ialu_reg_mem);
11148 %}
11149 
11150 instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11151 %{
11152   predicate(UseAPX);
11153   match(Set dst (OrL src1 (LoadL src2)));
11154   effect(KILL cr);
11155   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11156 
11157   ins_cost(150);
11158   format %{ "eorq     $dst, $src1, $src2\t# long ndd" %}
11159   ins_encode %{
11160     __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11161   %}
11162   ins_pipe(ialu_reg_mem);
11163 %}
11164 
11165 // Or Memory with Register
11166 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11167 %{
11168   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11169   effect(KILL cr);
11170   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11171 
11172   ins_cost(150);
11173   format %{ "orq     $dst, $src\t# long" %}
11174   ins_encode %{
11175     __ orq($dst$$Address, $src$$Register);
11176   %}
11177   ins_pipe(ialu_mem_reg);
11178 %}
11179 
11180 // Or Memory with Immediate
11181 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11182 %{
11183   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
11184   effect(KILL cr);
11185   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11186 
11187   ins_cost(125);
11188   format %{ "orq     $dst, $src\t# long" %}
11189   ins_encode %{
11190     __ orq($dst$$Address, $src$$constant);
11191   %}
11192   ins_pipe(ialu_mem_imm);
11193 %}
11194 
11195 instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
11196 %{
11197   // con should be a pure 64-bit power of 2 immediate
11198   // because AND/OR works well enough for 8/32-bit values.
11199   predicate(log2i_graceful(n->in(3)->in(2)->get_long()) > 31);
11200 
11201   match(Set dst (StoreL dst (OrL (LoadL dst) con)));
11202   effect(KILL cr);
11203 
11204   ins_cost(125);
11205   format %{ "btsq    $dst, log2($con)\t# long" %}
11206   ins_encode %{
11207     __ btsq($dst$$Address, log2i_exact((julong)$con$$constant));
11208   %}
11209   ins_pipe(ialu_mem_imm);
11210 %}
11211 
11212 // Xor Instructions
11213 // Xor Register with Register
11214 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
11215 %{
11216   predicate(!UseAPX);
11217   match(Set dst (XorL dst src));
11218   effect(KILL cr);
11219   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11220 
11221   format %{ "xorq    $dst, $src\t# long" %}
11222   ins_encode %{
11223     __ xorq($dst$$Register, $src$$Register);
11224   %}
11225   ins_pipe(ialu_reg_reg);
11226 %}
11227 
11228 // Xor Register with Register using New Data Destination (NDD)
11229 instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
11230 %{
11231   predicate(UseAPX);
11232   match(Set dst (XorL src1 src2));
11233   effect(KILL cr);
11234   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11235 
11236   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11237   ins_encode %{
11238     __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
11239   %}
11240   ins_pipe(ialu_reg_reg);
11241 %}
11242 
11243 // Xor Register with Immediate -1
11244 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
11245 %{
11246   predicate(!UseAPX);
11247   match(Set dst (XorL dst imm));
11248 
11249   format %{ "notq   $dst" %}
11250   ins_encode %{
11251      __ notq($dst$$Register);
11252   %}
11253   ins_pipe(ialu_reg);
11254 %}
11255 
11256 instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
11257 %{
11258   predicate(UseAPX);
11259   match(Set dst (XorL src imm));
11260 
11261   format %{ "enotq   $dst, $src" %}
11262   ins_encode %{
11263     __ enotq($dst$$Register, $src$$Register);
11264   %}
11265   ins_pipe(ialu_reg);
11266 %}
11267 
11268 // Xor Register with Immediate
11269 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
11270 %{
11271   // Strict predicate check to make selection of xorL_rReg_im1 cost agnostic if immL32 src is -1.
11272   predicate(!UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11273   match(Set dst (XorL dst src));
11274   effect(KILL cr);
11275   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11276 
11277   format %{ "xorq    $dst, $src\t# long" %}
11278   ins_encode %{
11279     __ xorq($dst$$Register, $src$$constant);
11280   %}
11281   ins_pipe(ialu_reg);
11282 %}
11283 
11284 instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
11285 %{
11286   // Strict predicate check to make selection of xorL_rReg_im1_ndd cost agnostic if immL32 src2 is -1.
11287   predicate(UseAPX && n->in(2)->bottom_type()->is_long()->get_con() != -1L);
11288   match(Set dst (XorL src1 src2));
11289   effect(KILL cr);
11290   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11291 
11292   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11293   ins_encode %{
11294     __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
11295   %}
11296   ins_pipe(ialu_reg);
11297 %}
11298 
11299 // Xor Memory with Immediate
11300 instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
11301 %{
11302   predicate(UseAPX);
11303   match(Set dst (XorL (LoadL src1) src2));
11304   effect(KILL cr);
11305   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11306   ins_cost(150);
11307 
11308   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11309   ins_encode %{
11310     __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
11311   %}
11312   ins_pipe(ialu_reg);
11313 %}
11314 
11315 // Xor Register with Memory
11316 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
11317 %{
11318   predicate(!UseAPX);
11319   match(Set dst (XorL dst (LoadL src)));
11320   effect(KILL cr);
11321   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11322 
11323   ins_cost(150);
11324   format %{ "xorq    $dst, $src\t# long" %}
11325   ins_encode %{
11326     __ xorq($dst$$Register, $src$$Address);
11327   %}
11328   ins_pipe(ialu_reg_mem);
11329 %}
11330 
11331 instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
11332 %{
11333   predicate(UseAPX);
11334   match(Set dst (XorL src1 (LoadL src2)));
11335   effect(KILL cr);
11336   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11337 
11338   ins_cost(150);
11339   format %{ "exorq    $dst, $src1, $src2\t# long ndd" %}
11340   ins_encode %{
11341     __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
11342   %}
11343   ins_pipe(ialu_reg_mem);
11344 %}
11345 
11346 // Xor Memory with Register
11347 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
11348 %{
11349   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11350   effect(KILL cr);
11351   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11352 
11353   ins_cost(150);
11354   format %{ "xorq    $dst, $src\t# long" %}
11355   ins_encode %{
11356     __ xorq($dst$$Address, $src$$Register);
11357   %}
11358   ins_pipe(ialu_mem_reg);
11359 %}
11360 
11361 // Xor Memory with Immediate
11362 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
11363 %{
11364   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
11365   effect(KILL cr);
11366   flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
11367 
11368   ins_cost(125);
11369   format %{ "xorq    $dst, $src\t# long" %}
11370   ins_encode %{
11371     __ xorq($dst$$Address, $src$$constant);
11372   %}
11373   ins_pipe(ialu_mem_imm);
11374 %}
11375 
11376 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
11377 %{
11378   match(Set dst (CmpLTMask p q));
11379   effect(KILL cr);
11380 
11381   ins_cost(400);
11382   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
11383             "setcc   $dst \t# emits setlt + movzbl or setzul for APX"
11384             "negl    $dst" %}
11385   ins_encode %{
11386     __ cmpl($p$$Register, $q$$Register);
11387     __ setcc(Assembler::less, $dst$$Register);
11388     __ negl($dst$$Register);
11389   %}
11390   ins_pipe(pipe_slow);
11391 %}
11392 
11393 instruct cmpLTMask0(rRegI dst, immI_0 zero, rFlagsReg cr)
11394 %{
11395   match(Set dst (CmpLTMask dst zero));
11396   effect(KILL cr);
11397 
11398   ins_cost(100);
11399   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
11400   ins_encode %{
11401     __ sarl($dst$$Register, 31);
11402   %}
11403   ins_pipe(ialu_reg);
11404 %}
11405 
11406 /* Better to save a register than avoid a branch */
11407 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11408 %{
11409   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
11410   effect(KILL cr);
11411   ins_cost(300);
11412   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
11413             "jge     done\n\t"
11414             "addl    $p,$y\n"
11415             "done:   " %}
11416   ins_encode %{
11417     Register Rp = $p$$Register;
11418     Register Rq = $q$$Register;
11419     Register Ry = $y$$Register;
11420     Label done;
11421     __ subl(Rp, Rq);
11422     __ jccb(Assembler::greaterEqual, done);
11423     __ addl(Rp, Ry);
11424     __ bind(done);
11425   %}
11426   ins_pipe(pipe_cmplt);
11427 %}
11428 
11429 /* Better to save a register than avoid a branch */
11430 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
11431 %{
11432   match(Set y (AndI (CmpLTMask p q) y));
11433   effect(KILL cr);
11434 
11435   ins_cost(300);
11436 
11437   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
11438             "jlt     done\n\t"
11439             "xorl    $y, $y\n"
11440             "done:   " %}
11441   ins_encode %{
11442     Register Rp = $p$$Register;
11443     Register Rq = $q$$Register;
11444     Register Ry = $y$$Register;
11445     Label done;
11446     __ cmpl(Rp, Rq);
11447     __ jccb(Assembler::less, done);
11448     __ xorl(Ry, Ry);
11449     __ bind(done);
11450   %}
11451   ins_pipe(pipe_cmplt);
11452 %}
11453 
11454 
11455 //---------- FP Instructions------------------------------------------------
11456 
11457 // Really expensive, avoid
11458 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
11459 %{
11460   match(Set cr (CmpF src1 src2));
11461 
11462   ins_cost(500);
11463   format %{ "ucomiss $src1, $src2\n\t"
11464             "jnp,s   exit\n\t"
11465             "pushfq\t# saw NaN, set CF\n\t"
11466             "andq    [rsp], #0xffffff2b\n\t"
11467             "popfq\n"
11468     "exit:" %}
11469   ins_encode %{
11470     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11471     emit_cmpfp_fixup(masm);
11472   %}
11473   ins_pipe(pipe_slow);
11474 %}
11475 
11476 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
11477   match(Set cr (CmpF src1 src2));
11478 
11479   ins_cost(100);
11480   format %{ "ucomiss $src1, $src2" %}
11481   ins_encode %{
11482     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11483   %}
11484   ins_pipe(pipe_slow);
11485 %}
11486 
11487 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
11488   match(Set cr (CmpF src1 (LoadF src2)));
11489 
11490   ins_cost(100);
11491   format %{ "ucomiss $src1, $src2" %}
11492   ins_encode %{
11493     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11494   %}
11495   ins_pipe(pipe_slow);
11496 %}
11497 
11498 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
11499   match(Set cr (CmpF src con));
11500   ins_cost(100);
11501   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
11502   ins_encode %{
11503     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11504   %}
11505   ins_pipe(pipe_slow);
11506 %}
11507 
11508 // Really expensive, avoid
11509 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
11510 %{
11511   match(Set cr (CmpD src1 src2));
11512 
11513   ins_cost(500);
11514   format %{ "ucomisd $src1, $src2\n\t"
11515             "jnp,s   exit\n\t"
11516             "pushfq\t# saw NaN, set CF\n\t"
11517             "andq    [rsp], #0xffffff2b\n\t"
11518             "popfq\n"
11519     "exit:" %}
11520   ins_encode %{
11521     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11522     emit_cmpfp_fixup(masm);
11523   %}
11524   ins_pipe(pipe_slow);
11525 %}
11526 
11527 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
11528   match(Set cr (CmpD src1 src2));
11529 
11530   ins_cost(100);
11531   format %{ "ucomisd $src1, $src2 test" %}
11532   ins_encode %{
11533     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11534   %}
11535   ins_pipe(pipe_slow);
11536 %}
11537 
11538 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
11539   match(Set cr (CmpD src1 (LoadD src2)));
11540 
11541   ins_cost(100);
11542   format %{ "ucomisd $src1, $src2" %}
11543   ins_encode %{
11544     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11545   %}
11546   ins_pipe(pipe_slow);
11547 %}
11548 
11549 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
11550   match(Set cr (CmpD src con));
11551   ins_cost(100);
11552   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
11553   ins_encode %{
11554     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11555   %}
11556   ins_pipe(pipe_slow);
11557 %}
11558 
11559 // Compare into -1,0,1
11560 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
11561 %{
11562   match(Set dst (CmpF3 src1 src2));
11563   effect(KILL cr);
11564 
11565   ins_cost(275);
11566   format %{ "ucomiss $src1, $src2\n\t"
11567             "movl    $dst, #-1\n\t"
11568             "jp,s    done\n\t"
11569             "jb,s    done\n\t"
11570             "setne   $dst\n\t"
11571             "movzbl  $dst, $dst\n"
11572     "done:" %}
11573   ins_encode %{
11574     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
11575     emit_cmpfp3(masm, $dst$$Register);
11576   %}
11577   ins_pipe(pipe_slow);
11578 %}
11579 
11580 // Compare into -1,0,1
11581 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
11582 %{
11583   match(Set dst (CmpF3 src1 (LoadF src2)));
11584   effect(KILL cr);
11585 
11586   ins_cost(275);
11587   format %{ "ucomiss $src1, $src2\n\t"
11588             "movl    $dst, #-1\n\t"
11589             "jp,s    done\n\t"
11590             "jb,s    done\n\t"
11591             "setne   $dst\n\t"
11592             "movzbl  $dst, $dst\n"
11593     "done:" %}
11594   ins_encode %{
11595     __ ucomiss($src1$$XMMRegister, $src2$$Address);
11596     emit_cmpfp3(masm, $dst$$Register);
11597   %}
11598   ins_pipe(pipe_slow);
11599 %}
11600 
11601 // Compare into -1,0,1
11602 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
11603   match(Set dst (CmpF3 src con));
11604   effect(KILL cr);
11605 
11606   ins_cost(275);
11607   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
11608             "movl    $dst, #-1\n\t"
11609             "jp,s    done\n\t"
11610             "jb,s    done\n\t"
11611             "setne   $dst\n\t"
11612             "movzbl  $dst, $dst\n"
11613     "done:" %}
11614   ins_encode %{
11615     __ ucomiss($src$$XMMRegister, $constantaddress($con));
11616     emit_cmpfp3(masm, $dst$$Register);
11617   %}
11618   ins_pipe(pipe_slow);
11619 %}
11620 
11621 // Compare into -1,0,1
11622 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
11623 %{
11624   match(Set dst (CmpD3 src1 src2));
11625   effect(KILL cr);
11626 
11627   ins_cost(275);
11628   format %{ "ucomisd $src1, $src2\n\t"
11629             "movl    $dst, #-1\n\t"
11630             "jp,s    done\n\t"
11631             "jb,s    done\n\t"
11632             "setne   $dst\n\t"
11633             "movzbl  $dst, $dst\n"
11634     "done:" %}
11635   ins_encode %{
11636     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
11637     emit_cmpfp3(masm, $dst$$Register);
11638   %}
11639   ins_pipe(pipe_slow);
11640 %}
11641 
11642 // Compare into -1,0,1
11643 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
11644 %{
11645   match(Set dst (CmpD3 src1 (LoadD src2)));
11646   effect(KILL cr);
11647 
11648   ins_cost(275);
11649   format %{ "ucomisd $src1, $src2\n\t"
11650             "movl    $dst, #-1\n\t"
11651             "jp,s    done\n\t"
11652             "jb,s    done\n\t"
11653             "setne   $dst\n\t"
11654             "movzbl  $dst, $dst\n"
11655     "done:" %}
11656   ins_encode %{
11657     __ ucomisd($src1$$XMMRegister, $src2$$Address);
11658     emit_cmpfp3(masm, $dst$$Register);
11659   %}
11660   ins_pipe(pipe_slow);
11661 %}
11662 
11663 // Compare into -1,0,1
11664 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
11665   match(Set dst (CmpD3 src con));
11666   effect(KILL cr);
11667 
11668   ins_cost(275);
11669   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
11670             "movl    $dst, #-1\n\t"
11671             "jp,s    done\n\t"
11672             "jb,s    done\n\t"
11673             "setne   $dst\n\t"
11674             "movzbl  $dst, $dst\n"
11675     "done:" %}
11676   ins_encode %{
11677     __ ucomisd($src$$XMMRegister, $constantaddress($con));
11678     emit_cmpfp3(masm, $dst$$Register);
11679   %}
11680   ins_pipe(pipe_slow);
11681 %}
11682 
11683 //----------Arithmetic Conversion Instructions---------------------------------
11684 
11685 instruct convF2D_reg_reg(regD dst, regF src)
11686 %{
11687   match(Set dst (ConvF2D src));
11688 
11689   format %{ "cvtss2sd $dst, $src" %}
11690   ins_encode %{
11691     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
11692   %}
11693   ins_pipe(pipe_slow); // XXX
11694 %}
11695 
11696 instruct convF2D_reg_mem(regD dst, memory src)
11697 %{
11698   predicate(UseAVX == 0);
11699   match(Set dst (ConvF2D (LoadF src)));
11700 
11701   format %{ "cvtss2sd $dst, $src" %}
11702   ins_encode %{
11703     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
11704   %}
11705   ins_pipe(pipe_slow); // XXX
11706 %}
11707 
11708 instruct convD2F_reg_reg(regF dst, regD src)
11709 %{
11710   match(Set dst (ConvD2F src));
11711 
11712   format %{ "cvtsd2ss $dst, $src" %}
11713   ins_encode %{
11714     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
11715   %}
11716   ins_pipe(pipe_slow); // XXX
11717 %}
11718 
11719 instruct convD2F_reg_mem(regF dst, memory src)
11720 %{
11721   predicate(UseAVX == 0);
11722   match(Set dst (ConvD2F (LoadD src)));
11723 
11724   format %{ "cvtsd2ss $dst, $src" %}
11725   ins_encode %{
11726     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
11727   %}
11728   ins_pipe(pipe_slow); // XXX
11729 %}
11730 
11731 // XXX do mem variants
11732 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
11733 %{
11734   predicate(!VM_Version::supports_avx10_2());
11735   match(Set dst (ConvF2I src));
11736   effect(KILL cr);
11737   format %{ "convert_f2i $dst, $src" %}
11738   ins_encode %{
11739     __ convertF2I(T_INT, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11740   %}
11741   ins_pipe(pipe_slow);
11742 %}
11743 
11744 instruct convF2I_reg_reg_avx10(rRegI dst, regF src)
11745 %{
11746   predicate(VM_Version::supports_avx10_2());
11747   match(Set dst (ConvF2I src));
11748   format %{ "evcvttss2sisl $dst, $src" %}
11749   ins_encode %{
11750     __ evcvttss2sisl($dst$$Register, $src$$XMMRegister);
11751   %}
11752   ins_pipe(pipe_slow);
11753 %}
11754 
11755 instruct convF2I_reg_mem_avx10(rRegI dst, memory src)
11756 %{
11757   predicate(VM_Version::supports_avx10_2());
11758   match(Set dst (ConvF2I (LoadF src)));
11759   format %{ "evcvttss2sisl $dst, $src" %}
11760   ins_encode %{
11761     __ evcvttss2sisl($dst$$Register, $src$$Address);
11762   %}
11763   ins_pipe(pipe_slow);
11764 %}
11765 
11766 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
11767 %{
11768   predicate(!VM_Version::supports_avx10_2());
11769   match(Set dst (ConvF2L src));
11770   effect(KILL cr);
11771   format %{ "convert_f2l $dst, $src"%}
11772   ins_encode %{
11773     __ convertF2I(T_LONG, T_FLOAT, $dst$$Register, $src$$XMMRegister);
11774   %}
11775   ins_pipe(pipe_slow);
11776 %}
11777 
11778 instruct convF2L_reg_reg_avx10(rRegL dst, regF src)
11779 %{
11780   predicate(VM_Version::supports_avx10_2());
11781   match(Set dst (ConvF2L src));
11782   format %{ "evcvttss2sisq $dst, $src" %}
11783   ins_encode %{
11784     __ evcvttss2sisq($dst$$Register, $src$$XMMRegister);
11785   %}
11786   ins_pipe(pipe_slow);
11787 %}
11788 
11789 instruct convF2L_reg_mem_avx10(rRegL dst, memory src)
11790 %{
11791   predicate(VM_Version::supports_avx10_2());
11792   match(Set dst (ConvF2L (LoadF src)));
11793   format %{ "evcvttss2sisq $dst, $src" %}
11794   ins_encode %{
11795     __ evcvttss2sisq($dst$$Register, $src$$Address);
11796   %}
11797   ins_pipe(pipe_slow);
11798 %}
11799 
11800 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11801 %{
11802   predicate(!VM_Version::supports_avx10_2());
11803   match(Set dst (ConvD2I src));
11804   effect(KILL cr);
11805   format %{ "convert_d2i $dst, $src"%}
11806   ins_encode %{
11807     __ convertF2I(T_INT, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11808   %}
11809   ins_pipe(pipe_slow);
11810 %}
11811 
11812 instruct convD2I_reg_reg_avx10(rRegI dst, regD src)
11813 %{
11814   predicate(VM_Version::supports_avx10_2());
11815   match(Set dst (ConvD2I src));
11816   format %{ "evcvttsd2sisl $dst, $src" %}
11817   ins_encode %{
11818     __ evcvttsd2sisl($dst$$Register, $src$$XMMRegister);
11819   %}
11820   ins_pipe(pipe_slow);
11821 %}
11822 
11823 instruct convD2I_reg_mem_avx10(rRegI dst, memory src)
11824 %{
11825   predicate(VM_Version::supports_avx10_2());
11826   match(Set dst (ConvD2I (LoadD src)));
11827   format %{ "evcvttsd2sisl $dst, $src" %}
11828   ins_encode %{
11829     __ evcvttsd2sisl($dst$$Register, $src$$Address);
11830   %}
11831   ins_pipe(pipe_slow);
11832 %}
11833 
11834 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11835 %{
11836   predicate(!VM_Version::supports_avx10_2());
11837   match(Set dst (ConvD2L src));
11838   effect(KILL cr);
11839   format %{ "convert_d2l $dst, $src"%}
11840   ins_encode %{
11841     __ convertF2I(T_LONG, T_DOUBLE, $dst$$Register, $src$$XMMRegister);
11842   %}
11843   ins_pipe(pipe_slow);
11844 %}
11845 
11846 instruct convD2L_reg_reg_avx10(rRegL dst, regD src)
11847 %{
11848   predicate(VM_Version::supports_avx10_2());
11849   match(Set dst (ConvD2L src));
11850   format %{ "evcvttsd2sisq $dst, $src" %}
11851   ins_encode %{
11852     __ evcvttsd2sisq($dst$$Register, $src$$XMMRegister);
11853   %}
11854   ins_pipe(pipe_slow);
11855 %}
11856 
11857 instruct convD2L_reg_mem_avx10(rRegL dst, memory src)
11858 %{
11859   predicate(VM_Version::supports_avx10_2());
11860   match(Set dst (ConvD2L (LoadD src)));
11861   format %{ "evcvttsd2sisq $dst, $src" %}
11862   ins_encode %{
11863     __ evcvttsd2sisq($dst$$Register, $src$$Address);
11864   %}
11865   ins_pipe(pipe_slow);
11866 %}
11867 
11868 instruct round_double_reg(rRegL dst, regD src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11869 %{
11870   match(Set dst (RoundD src));
11871   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11872   format %{ "round_double $dst,$src \t! using $rtmp and $rcx as TEMP"%}
11873   ins_encode %{
11874     __ round_double($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11875   %}
11876   ins_pipe(pipe_slow);
11877 %}
11878 
11879 instruct round_float_reg(rRegI dst, regF src, rRegL rtmp, rcx_RegL rcx, rFlagsReg cr)
11880 %{
11881   match(Set dst (RoundF src));
11882   effect(TEMP dst, TEMP rtmp, TEMP rcx, KILL cr);
11883   format %{ "round_float $dst,$src" %}
11884   ins_encode %{
11885     __ round_float($dst$$Register, $src$$XMMRegister, $rtmp$$Register, $rcx$$Register);
11886   %}
11887   ins_pipe(pipe_slow);
11888 %}
11889 
11890 instruct convI2F_reg_reg(vlRegF dst, rRegI src)
11891 %{
11892   predicate(!UseXmmI2F);
11893   match(Set dst (ConvI2F src));
11894 
11895   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11896   ins_encode %{
11897     if (UseAVX > 0) {
11898       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11899     }
11900     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
11901   %}
11902   ins_pipe(pipe_slow); // XXX
11903 %}
11904 
11905 instruct convI2F_reg_mem(regF dst, memory src)
11906 %{
11907   predicate(UseAVX == 0);
11908   match(Set dst (ConvI2F (LoadI src)));
11909 
11910   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11911   ins_encode %{
11912     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
11913   %}
11914   ins_pipe(pipe_slow); // XXX
11915 %}
11916 
11917 instruct convI2D_reg_reg(vlRegD dst, rRegI src)
11918 %{
11919   predicate(!UseXmmI2D);
11920   match(Set dst (ConvI2D src));
11921 
11922   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11923   ins_encode %{
11924     if (UseAVX > 0) {
11925       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11926     }
11927     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
11928   %}
11929   ins_pipe(pipe_slow); // XXX
11930 %}
11931 
11932 instruct convI2D_reg_mem(regD dst, memory src)
11933 %{
11934   predicate(UseAVX == 0);
11935   match(Set dst (ConvI2D (LoadI src)));
11936 
11937   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11938   ins_encode %{
11939     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
11940   %}
11941   ins_pipe(pipe_slow); // XXX
11942 %}
11943 
11944 instruct convXI2F_reg(regF dst, rRegI src)
11945 %{
11946   predicate(UseXmmI2F);
11947   match(Set dst (ConvI2F src));
11948 
11949   format %{ "movdl $dst, $src\n\t"
11950             "cvtdq2psl $dst, $dst\t# i2f" %}
11951   ins_encode %{
11952     __ movdl($dst$$XMMRegister, $src$$Register);
11953     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11954   %}
11955   ins_pipe(pipe_slow); // XXX
11956 %}
11957 
11958 instruct convXI2D_reg(regD dst, rRegI src)
11959 %{
11960   predicate(UseXmmI2D);
11961   match(Set dst (ConvI2D src));
11962 
11963   format %{ "movdl $dst, $src\n\t"
11964             "cvtdq2pdl $dst, $dst\t# i2d" %}
11965   ins_encode %{
11966     __ movdl($dst$$XMMRegister, $src$$Register);
11967     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11968   %}
11969   ins_pipe(pipe_slow); // XXX
11970 %}
11971 
11972 instruct convL2F_reg_reg(vlRegF dst, rRegL src)
11973 %{
11974   match(Set dst (ConvL2F src));
11975 
11976   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11977   ins_encode %{
11978     if (UseAVX > 0) {
11979       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
11980     }
11981     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11982   %}
11983   ins_pipe(pipe_slow); // XXX
11984 %}
11985 
11986 instruct convL2F_reg_mem(regF dst, memory src)
11987 %{
11988   predicate(UseAVX == 0);
11989   match(Set dst (ConvL2F (LoadL src)));
11990 
11991   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11992   ins_encode %{
11993     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11994   %}
11995   ins_pipe(pipe_slow); // XXX
11996 %}
11997 
11998 instruct convL2D_reg_reg(vlRegD dst, rRegL src)
11999 %{
12000   match(Set dst (ConvL2D src));
12001 
12002   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
12003   ins_encode %{
12004     if (UseAVX > 0) {
12005       __ pxor($dst$$XMMRegister, $dst$$XMMRegister);
12006     }
12007     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
12008   %}
12009   ins_pipe(pipe_slow); // XXX
12010 %}
12011 
12012 instruct convL2D_reg_mem(regD dst, memory src)
12013 %{
12014   predicate(UseAVX == 0);
12015   match(Set dst (ConvL2D (LoadL src)));
12016 
12017   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
12018   ins_encode %{
12019     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
12020   %}
12021   ins_pipe(pipe_slow); // XXX
12022 %}
12023 
12024 instruct convI2L_reg_reg(rRegL dst, rRegI src)
12025 %{
12026   match(Set dst (ConvI2L src));
12027 
12028   ins_cost(125);
12029   format %{ "movslq  $dst, $src\t# i2l" %}
12030   ins_encode %{
12031     __ movslq($dst$$Register, $src$$Register);
12032   %}
12033   ins_pipe(ialu_reg_reg);
12034 %}
12035 
12036 // Zero-extend convert int to long
12037 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
12038 %{
12039   match(Set dst (AndL (ConvI2L src) mask));
12040 
12041   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12042   ins_encode %{
12043     if ($dst$$reg != $src$$reg) {
12044       __ movl($dst$$Register, $src$$Register);
12045     }
12046   %}
12047   ins_pipe(ialu_reg_reg);
12048 %}
12049 
12050 // Zero-extend convert int to long
12051 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
12052 %{
12053   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
12054 
12055   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
12056   ins_encode %{
12057     __ movl($dst$$Register, $src$$Address);
12058   %}
12059   ins_pipe(ialu_reg_mem);
12060 %}
12061 
12062 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
12063 %{
12064   match(Set dst (AndL src mask));
12065 
12066   format %{ "movl    $dst, $src\t# zero-extend long" %}
12067   ins_encode %{
12068     __ movl($dst$$Register, $src$$Register);
12069   %}
12070   ins_pipe(ialu_reg_reg);
12071 %}
12072 
12073 instruct convL2I_reg_reg(rRegI dst, rRegL src)
12074 %{
12075   match(Set dst (ConvL2I src));
12076 
12077   format %{ "movl    $dst, $src\t# l2i" %}
12078   ins_encode %{
12079     __ movl($dst$$Register, $src$$Register);
12080   %}
12081   ins_pipe(ialu_reg_reg);
12082 %}
12083 
12084 
12085 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
12086   match(Set dst (MoveF2I src));
12087   effect(DEF dst, USE src);
12088 
12089   ins_cost(125);
12090   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
12091   ins_encode %{
12092     __ movl($dst$$Register, Address(rsp, $src$$disp));
12093   %}
12094   ins_pipe(ialu_reg_mem);
12095 %}
12096 
12097 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12098   match(Set dst (MoveI2F src));
12099   effect(DEF dst, USE src);
12100 
12101   ins_cost(125);
12102   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
12103   ins_encode %{
12104     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
12105   %}
12106   ins_pipe(pipe_slow);
12107 %}
12108 
12109 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
12110   match(Set dst (MoveD2L src));
12111   effect(DEF dst, USE src);
12112 
12113   ins_cost(125);
12114   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
12115   ins_encode %{
12116     __ movq($dst$$Register, Address(rsp, $src$$disp));
12117   %}
12118   ins_pipe(ialu_reg_mem);
12119 %}
12120 
12121 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
12122   predicate(!UseXmmLoadAndClearUpper);
12123   match(Set dst (MoveL2D src));
12124   effect(DEF dst, USE src);
12125 
12126   ins_cost(125);
12127   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
12128   ins_encode %{
12129     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12130   %}
12131   ins_pipe(pipe_slow);
12132 %}
12133 
12134 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12135   predicate(UseXmmLoadAndClearUpper);
12136   match(Set dst (MoveL2D src));
12137   effect(DEF dst, USE src);
12138 
12139   ins_cost(125);
12140   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
12141   ins_encode %{
12142     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
12143   %}
12144   ins_pipe(pipe_slow);
12145 %}
12146 
12147 
12148 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12149   match(Set dst (MoveF2I src));
12150   effect(DEF dst, USE src);
12151 
12152   ins_cost(95); // XXX
12153   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
12154   ins_encode %{
12155     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
12156   %}
12157   ins_pipe(pipe_slow);
12158 %}
12159 
12160 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
12161   match(Set dst (MoveI2F src));
12162   effect(DEF dst, USE src);
12163 
12164   ins_cost(100);
12165   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
12166   ins_encode %{
12167     __ movl(Address(rsp, $dst$$disp), $src$$Register);
12168   %}
12169   ins_pipe( ialu_mem_reg );
12170 %}
12171 
12172 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12173   match(Set dst (MoveD2L src));
12174   effect(DEF dst, USE src);
12175 
12176   ins_cost(95); // XXX
12177   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
12178   ins_encode %{
12179     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
12180   %}
12181   ins_pipe(pipe_slow);
12182 %}
12183 
12184 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
12185   match(Set dst (MoveL2D src));
12186   effect(DEF dst, USE src);
12187 
12188   ins_cost(100);
12189   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
12190   ins_encode %{
12191     __ movq(Address(rsp, $dst$$disp), $src$$Register);
12192   %}
12193   ins_pipe(ialu_mem_reg);
12194 %}
12195 
12196 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
12197   match(Set dst (MoveF2I src));
12198   effect(DEF dst, USE src);
12199   ins_cost(85);
12200   format %{ "movd    $dst,$src\t# MoveF2I" %}
12201   ins_encode %{
12202     __ movdl($dst$$Register, $src$$XMMRegister);
12203   %}
12204   ins_pipe( pipe_slow );
12205 %}
12206 
12207 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
12208   match(Set dst (MoveD2L src));
12209   effect(DEF dst, USE src);
12210   ins_cost(85);
12211   format %{ "movd    $dst,$src\t# MoveD2L" %}
12212   ins_encode %{
12213     __ movdq($dst$$Register, $src$$XMMRegister);
12214   %}
12215   ins_pipe( pipe_slow );
12216 %}
12217 
12218 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
12219   match(Set dst (MoveI2F src));
12220   effect(DEF dst, USE src);
12221   ins_cost(100);
12222   format %{ "movd    $dst,$src\t# MoveI2F" %}
12223   ins_encode %{
12224     __ movdl($dst$$XMMRegister, $src$$Register);
12225   %}
12226   ins_pipe( pipe_slow );
12227 %}
12228 
12229 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
12230   match(Set dst (MoveL2D src));
12231   effect(DEF dst, USE src);
12232   ins_cost(100);
12233   format %{ "movd    $dst,$src\t# MoveL2D" %}
12234   ins_encode %{
12235      __ movdq($dst$$XMMRegister, $src$$Register);
12236   %}
12237   ins_pipe( pipe_slow );
12238 %}
12239 
12240 // Fast clearing of an array
12241 // Small non-constant lenght ClearArray for non-AVX512 targets.
12242 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12243                   Universe dummy, rFlagsReg cr)
12244 %{
12245   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
12246   match(Set dummy (ClearArray cnt base));
12247   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12248 
12249   format %{ $$template
12250     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12251     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12252     $$emit$$"jg      LARGE\n\t"
12253     $$emit$$"dec     rcx\n\t"
12254     $$emit$$"js      DONE\t# Zero length\n\t"
12255     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12256     $$emit$$"dec     rcx\n\t"
12257     $$emit$$"jge     LOOP\n\t"
12258     $$emit$$"jmp     DONE\n\t"
12259     $$emit$$"# LARGE:\n\t"
12260     if (UseFastStosb) {
12261        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12262        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12263     } else if (UseXMMForObjInit) {
12264        $$emit$$"mov     rdi,rax\n\t"
12265        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12266        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12267        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12268        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12269        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12270        $$emit$$"add     0x40,rax\n\t"
12271        $$emit$$"# L_zero_64_bytes:\n\t"
12272        $$emit$$"sub     0x8,rcx\n\t"
12273        $$emit$$"jge     L_loop\n\t"
12274        $$emit$$"add     0x4,rcx\n\t"
12275        $$emit$$"jl      L_tail\n\t"
12276        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12277        $$emit$$"add     0x20,rax\n\t"
12278        $$emit$$"sub     0x4,rcx\n\t"
12279        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12280        $$emit$$"add     0x4,rcx\n\t"
12281        $$emit$$"jle     L_end\n\t"
12282        $$emit$$"dec     rcx\n\t"
12283        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12284        $$emit$$"vmovq   xmm0,(rax)\n\t"
12285        $$emit$$"add     0x8,rax\n\t"
12286        $$emit$$"dec     rcx\n\t"
12287        $$emit$$"jge     L_sloop\n\t"
12288        $$emit$$"# L_end:\n\t"
12289     } else {
12290        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12291     }
12292     $$emit$$"# DONE"
12293   %}
12294   ins_encode %{
12295     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12296                  $tmp$$XMMRegister, false, knoreg);
12297   %}
12298   ins_pipe(pipe_slow);
12299 %}
12300 
12301 // Small non-constant length ClearArray for AVX512 targets.
12302 instruct rep_stos_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12303                        Universe dummy, rFlagsReg cr)
12304 %{
12305   predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
12306   match(Set dummy (ClearArray cnt base));
12307   ins_cost(125);
12308   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12309 
12310   format %{ $$template
12311     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12312     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
12313     $$emit$$"jg      LARGE\n\t"
12314     $$emit$$"dec     rcx\n\t"
12315     $$emit$$"js      DONE\t# Zero length\n\t"
12316     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
12317     $$emit$$"dec     rcx\n\t"
12318     $$emit$$"jge     LOOP\n\t"
12319     $$emit$$"jmp     DONE\n\t"
12320     $$emit$$"# LARGE:\n\t"
12321     if (UseFastStosb) {
12322        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12323        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
12324     } else if (UseXMMForObjInit) {
12325        $$emit$$"mov     rdi,rax\n\t"
12326        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12327        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12328        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12329        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12330        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12331        $$emit$$"add     0x40,rax\n\t"
12332        $$emit$$"# L_zero_64_bytes:\n\t"
12333        $$emit$$"sub     0x8,rcx\n\t"
12334        $$emit$$"jge     L_loop\n\t"
12335        $$emit$$"add     0x4,rcx\n\t"
12336        $$emit$$"jl      L_tail\n\t"
12337        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12338        $$emit$$"add     0x20,rax\n\t"
12339        $$emit$$"sub     0x4,rcx\n\t"
12340        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12341        $$emit$$"add     0x4,rcx\n\t"
12342        $$emit$$"jle     L_end\n\t"
12343        $$emit$$"dec     rcx\n\t"
12344        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12345        $$emit$$"vmovq   xmm0,(rax)\n\t"
12346        $$emit$$"add     0x8,rax\n\t"
12347        $$emit$$"dec     rcx\n\t"
12348        $$emit$$"jge     L_sloop\n\t"
12349        $$emit$$"# L_end:\n\t"
12350     } else {
12351        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
12352     }
12353     $$emit$$"# DONE"
12354   %}
12355   ins_encode %{
12356     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12357                  $tmp$$XMMRegister, false, $ktmp$$KRegister);
12358   %}
12359   ins_pipe(pipe_slow);
12360 %}
12361 
12362 // Large non-constant length ClearArray for non-AVX512 targets.
12363 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
12364                         Universe dummy, rFlagsReg cr)
12365 %{
12366   predicate((UseAVX <=2) && ((ClearArrayNode*)n)->is_large());
12367   match(Set dummy (ClearArray cnt base));
12368   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
12369 
12370   format %{ $$template
12371     if (UseFastStosb) {
12372        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12373        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12374        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12375     } else if (UseXMMForObjInit) {
12376        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12377        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12378        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12379        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12380        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12381        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12382        $$emit$$"add     0x40,rax\n\t"
12383        $$emit$$"# L_zero_64_bytes:\n\t"
12384        $$emit$$"sub     0x8,rcx\n\t"
12385        $$emit$$"jge     L_loop\n\t"
12386        $$emit$$"add     0x4,rcx\n\t"
12387        $$emit$$"jl      L_tail\n\t"
12388        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12389        $$emit$$"add     0x20,rax\n\t"
12390        $$emit$$"sub     0x4,rcx\n\t"
12391        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12392        $$emit$$"add     0x4,rcx\n\t"
12393        $$emit$$"jle     L_end\n\t"
12394        $$emit$$"dec     rcx\n\t"
12395        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12396        $$emit$$"vmovq   xmm0,(rax)\n\t"
12397        $$emit$$"add     0x8,rax\n\t"
12398        $$emit$$"dec     rcx\n\t"
12399        $$emit$$"jge     L_sloop\n\t"
12400        $$emit$$"# L_end:\n\t"
12401     } else {
12402        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12403        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12404     }
12405   %}
12406   ins_encode %{
12407     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12408                  $tmp$$XMMRegister, true, knoreg);
12409   %}
12410   ins_pipe(pipe_slow);
12411 %}
12412 
12413 // Large non-constant length ClearArray for AVX512 targets.
12414 instruct rep_stos_large_evex(rcx_RegL cnt, rdi_RegP base, legRegD tmp, kReg ktmp, rax_RegI zero,
12415                              Universe dummy, rFlagsReg cr)
12416 %{
12417   predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
12418   match(Set dummy (ClearArray cnt base));
12419   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
12420 
12421   format %{ $$template
12422     if (UseFastStosb) {
12423        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12424        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
12425        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
12426     } else if (UseXMMForObjInit) {
12427        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
12428        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
12429        $$emit$$"jmpq    L_zero_64_bytes\n\t"
12430        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
12431        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12432        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
12433        $$emit$$"add     0x40,rax\n\t"
12434        $$emit$$"# L_zero_64_bytes:\n\t"
12435        $$emit$$"sub     0x8,rcx\n\t"
12436        $$emit$$"jge     L_loop\n\t"
12437        $$emit$$"add     0x4,rcx\n\t"
12438        $$emit$$"jl      L_tail\n\t"
12439        $$emit$$"vmovdqu ymm0,(rax)\n\t"
12440        $$emit$$"add     0x20,rax\n\t"
12441        $$emit$$"sub     0x4,rcx\n\t"
12442        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
12443        $$emit$$"add     0x4,rcx\n\t"
12444        $$emit$$"jle     L_end\n\t"
12445        $$emit$$"dec     rcx\n\t"
12446        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
12447        $$emit$$"vmovq   xmm0,(rax)\n\t"
12448        $$emit$$"add     0x8,rax\n\t"
12449        $$emit$$"dec     rcx\n\t"
12450        $$emit$$"jge     L_sloop\n\t"
12451        $$emit$$"# L_end:\n\t"
12452     } else {
12453        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
12454        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
12455     }
12456   %}
12457   ins_encode %{
12458     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
12459                  $tmp$$XMMRegister, true, $ktmp$$KRegister);
12460   %}
12461   ins_pipe(pipe_slow);
12462 %}
12463 
12464 // Small constant length ClearArray for AVX512 targets.
12465 instruct rep_stos_im(immL cnt, rRegP base, regD tmp, rRegI zero, kReg ktmp, Universe dummy, rFlagsReg cr)
12466 %{
12467   predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
12468   match(Set dummy (ClearArray cnt base));
12469   ins_cost(100);
12470   effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
12471   format %{ "clear_mem_imm $base , $cnt  \n\t" %}
12472   ins_encode %{
12473    __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
12474   %}
12475   ins_pipe(pipe_slow);
12476 %}
12477 
12478 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12479                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12480 %{
12481   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12482   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12483   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12484 
12485   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12486   ins_encode %{
12487     __ string_compare($str1$$Register, $str2$$Register,
12488                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12489                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
12490   %}
12491   ins_pipe( pipe_slow );
12492 %}
12493 
12494 instruct string_compareL_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12495                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12496 %{
12497   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
12498   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12499   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12500 
12501   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12502   ins_encode %{
12503     __ string_compare($str1$$Register, $str2$$Register,
12504                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12505                       $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
12506   %}
12507   ins_pipe( pipe_slow );
12508 %}
12509 
12510 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12511                          rax_RegI result, legRegD tmp1, rFlagsReg cr)
12512 %{
12513   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12514   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12515   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12516 
12517   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12518   ins_encode %{
12519     __ string_compare($str1$$Register, $str2$$Register,
12520                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12521                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
12522   %}
12523   ins_pipe( pipe_slow );
12524 %}
12525 
12526 instruct string_compareU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12527                               rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12528 %{
12529   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
12530   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12531   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12532 
12533   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12534   ins_encode %{
12535     __ string_compare($str1$$Register, $str2$$Register,
12536                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12537                       $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
12538   %}
12539   ins_pipe( pipe_slow );
12540 %}
12541 
12542 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12543                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12544 %{
12545   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12546   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12547   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12548 
12549   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12550   ins_encode %{
12551     __ string_compare($str1$$Register, $str2$$Register,
12552                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12553                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
12554   %}
12555   ins_pipe( pipe_slow );
12556 %}
12557 
12558 instruct string_compareLU_evex(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
12559                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12560 %{
12561   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
12562   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12563   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12564 
12565   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12566   ins_encode %{
12567     __ string_compare($str1$$Register, $str2$$Register,
12568                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12569                       $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
12570   %}
12571   ins_pipe( pipe_slow );
12572 %}
12573 
12574 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12575                           rax_RegI result, legRegD tmp1, rFlagsReg cr)
12576 %{
12577   predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12578   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12579   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12580 
12581   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12582   ins_encode %{
12583     __ string_compare($str2$$Register, $str1$$Register,
12584                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12585                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
12586   %}
12587   ins_pipe( pipe_slow );
12588 %}
12589 
12590 instruct string_compareUL_evex(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
12591                                rax_RegI result, legRegD tmp1, kReg ktmp, rFlagsReg cr)
12592 %{
12593   predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
12594   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12595   effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12596 
12597   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12598   ins_encode %{
12599     __ string_compare($str2$$Register, $str1$$Register,
12600                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
12601                       $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
12602   %}
12603   ins_pipe( pipe_slow );
12604 %}
12605 
12606 // fast search of substring with known size.
12607 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12608                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12609 %{
12610   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12611   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12612   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12613 
12614   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12615   ins_encode %{
12616     int icnt2 = (int)$int_cnt2$$constant;
12617     if (icnt2 >= 16) {
12618       // IndexOf for constant substrings with size >= 16 elements
12619       // which don't need to be loaded through stack.
12620       __ string_indexofC8($str1$$Register, $str2$$Register,
12621                           $cnt1$$Register, $cnt2$$Register,
12622                           icnt2, $result$$Register,
12623                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12624     } else {
12625       // Small strings are loaded through stack if they cross page boundary.
12626       __ string_indexof($str1$$Register, $str2$$Register,
12627                         $cnt1$$Register, $cnt2$$Register,
12628                         icnt2, $result$$Register,
12629                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12630     }
12631   %}
12632   ins_pipe( pipe_slow );
12633 %}
12634 
12635 // fast search of substring with known size.
12636 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12637                              rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12638 %{
12639   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12640   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12641   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12642 
12643   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12644   ins_encode %{
12645     int icnt2 = (int)$int_cnt2$$constant;
12646     if (icnt2 >= 8) {
12647       // IndexOf for constant substrings with size >= 8 elements
12648       // which don't need to be loaded through stack.
12649       __ string_indexofC8($str1$$Register, $str2$$Register,
12650                           $cnt1$$Register, $cnt2$$Register,
12651                           icnt2, $result$$Register,
12652                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12653     } else {
12654       // Small strings are loaded through stack if they cross page boundary.
12655       __ string_indexof($str1$$Register, $str2$$Register,
12656                         $cnt1$$Register, $cnt2$$Register,
12657                         icnt2, $result$$Register,
12658                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12659     }
12660   %}
12661   ins_pipe( pipe_slow );
12662 %}
12663 
12664 // fast search of substring with known size.
12665 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
12666                               rbx_RegI result, legRegD tmp_vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
12667 %{
12668   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12669   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12670   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12671 
12672   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $tmp_vec, $cnt1, $cnt2, $tmp" %}
12673   ins_encode %{
12674     int icnt2 = (int)$int_cnt2$$constant;
12675     if (icnt2 >= 8) {
12676       // IndexOf for constant substrings with size >= 8 elements
12677       // which don't need to be loaded through stack.
12678       __ string_indexofC8($str1$$Register, $str2$$Register,
12679                           $cnt1$$Register, $cnt2$$Register,
12680                           icnt2, $result$$Register,
12681                           $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12682     } else {
12683       // Small strings are loaded through stack if they cross page boundary.
12684       __ string_indexof($str1$$Register, $str2$$Register,
12685                         $cnt1$$Register, $cnt2$$Register,
12686                         icnt2, $result$$Register,
12687                         $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12688     }
12689   %}
12690   ins_pipe( pipe_slow );
12691 %}
12692 
12693 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12694                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12695 %{
12696   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
12697   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12698   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12699 
12700   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12701   ins_encode %{
12702     __ string_indexof($str1$$Register, $str2$$Register,
12703                       $cnt1$$Register, $cnt2$$Register,
12704                       (-1), $result$$Register,
12705                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
12706   %}
12707   ins_pipe( pipe_slow );
12708 %}
12709 
12710 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12711                          rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12712 %{
12713   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
12714   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12715   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12716 
12717   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12718   ins_encode %{
12719     __ string_indexof($str1$$Register, $str2$$Register,
12720                       $cnt1$$Register, $cnt2$$Register,
12721                       (-1), $result$$Register,
12722                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
12723   %}
12724   ins_pipe( pipe_slow );
12725 %}
12726 
12727 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
12728                           rbx_RegI result, legRegD tmp_vec, rcx_RegI tmp, rFlagsReg cr)
12729 %{
12730   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
12731   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12732   effect(TEMP tmp_vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12733 
12734   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12735   ins_encode %{
12736     __ string_indexof($str1$$Register, $str2$$Register,
12737                       $cnt1$$Register, $cnt2$$Register,
12738                       (-1), $result$$Register,
12739                       $tmp_vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
12740   %}
12741   ins_pipe( pipe_slow );
12742 %}
12743 
12744 instruct string_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12745                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12746 %{
12747   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
12748   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12749   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12750   format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12751   ins_encode %{
12752     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12753                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12754   %}
12755   ins_pipe( pipe_slow );
12756 %}
12757 
12758 instruct stringL_indexof_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
12759                               rbx_RegI result, legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, rcx_RegI tmp, rFlagsReg cr)
12760 %{
12761   predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
12762   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
12763   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
12764   format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
12765   ins_encode %{
12766     __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
12767                            $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister, $tmp$$Register);
12768   %}
12769   ins_pipe( pipe_slow );
12770 %}
12771 
12772 // fast string equals
12773 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12774                        legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr)
12775 %{
12776   predicate(!VM_Version::supports_avx512vlbw());
12777   match(Set result (StrEquals (Binary str1 str2) cnt));
12778   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12779 
12780   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12781   ins_encode %{
12782     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12783                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12784                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12785   %}
12786   ins_pipe( pipe_slow );
12787 %}
12788 
12789 instruct string_equals_evex(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
12790                            legRegD tmp1, legRegD tmp2, kReg ktmp, rbx_RegI tmp3, rFlagsReg cr)
12791 %{
12792   predicate(VM_Version::supports_avx512vlbw());
12793   match(Set result (StrEquals (Binary str1 str2) cnt));
12794   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12795 
12796   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12797   ins_encode %{
12798     __ arrays_equals(false, $str1$$Register, $str2$$Register,
12799                      $cnt$$Register, $result$$Register, $tmp3$$Register,
12800                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12801   %}
12802   ins_pipe( pipe_slow );
12803 %}
12804 
12805 // fast array equals
12806 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12807                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12808 %{
12809   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12810   match(Set result (AryEq ary1 ary2));
12811   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12812 
12813   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12814   ins_encode %{
12815     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12816                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12817                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
12818   %}
12819   ins_pipe( pipe_slow );
12820 %}
12821 
12822 instruct array_equalsB_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12823                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12824 %{
12825   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
12826   match(Set result (AryEq ary1 ary2));
12827   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12828 
12829   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12830   ins_encode %{
12831     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12832                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12833                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
12834   %}
12835   ins_pipe( pipe_slow );
12836 %}
12837 
12838 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12839                        legRegD tmp1, legRegD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12840 %{
12841   predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12842   match(Set result (AryEq ary1 ary2));
12843   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12844 
12845   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12846   ins_encode %{
12847     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12848                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12849                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
12850   %}
12851   ins_pipe( pipe_slow );
12852 %}
12853 
12854 instruct array_equalsC_evex(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
12855                             legRegD tmp1, legRegD tmp2, kReg ktmp, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
12856 %{
12857   predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
12858   match(Set result (AryEq ary1 ary2));
12859   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12860 
12861   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12862   ins_encode %{
12863     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
12864                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
12865                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
12866   %}
12867   ins_pipe( pipe_slow );
12868 %}
12869 
12870 instruct arrays_hashcode(rdi_RegP ary1, rdx_RegI cnt1, rbx_RegI result, immU8 basic_type,
12871                          legRegD tmp_vec1, legRegD tmp_vec2, legRegD tmp_vec3, legRegD tmp_vec4,
12872                          legRegD tmp_vec5, legRegD tmp_vec6, legRegD tmp_vec7, legRegD tmp_vec8,
12873                          legRegD tmp_vec9, legRegD tmp_vec10, legRegD tmp_vec11, legRegD tmp_vec12,
12874                          legRegD tmp_vec13, rRegI tmp1, rRegI tmp2, rRegI tmp3, rFlagsReg cr)
12875 %{
12876   predicate(UseAVX >= 2);
12877   match(Set result (VectorizedHashCode (Binary ary1 cnt1) (Binary result basic_type)));
12878   effect(TEMP tmp_vec1, TEMP tmp_vec2, TEMP tmp_vec3, TEMP tmp_vec4, TEMP tmp_vec5, TEMP tmp_vec6,
12879          TEMP tmp_vec7, TEMP tmp_vec8, TEMP tmp_vec9, TEMP tmp_vec10, TEMP tmp_vec11, TEMP tmp_vec12,
12880          TEMP tmp_vec13, TEMP tmp1, TEMP tmp2, TEMP tmp3, USE_KILL ary1, USE_KILL cnt1,
12881          USE basic_type, KILL cr);
12882 
12883   format %{ "Array HashCode array[] $ary1,$cnt1,$result,$basic_type -> $result   // KILL all" %}
12884   ins_encode %{
12885     __ arrays_hashcode($ary1$$Register, $cnt1$$Register, $result$$Register,
12886                        $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
12887                        $tmp_vec1$$XMMRegister, $tmp_vec2$$XMMRegister, $tmp_vec3$$XMMRegister,
12888                        $tmp_vec4$$XMMRegister, $tmp_vec5$$XMMRegister, $tmp_vec6$$XMMRegister,
12889                        $tmp_vec7$$XMMRegister, $tmp_vec8$$XMMRegister, $tmp_vec9$$XMMRegister,
12890                        $tmp_vec10$$XMMRegister, $tmp_vec11$$XMMRegister, $tmp_vec12$$XMMRegister,
12891                        $tmp_vec13$$XMMRegister, (BasicType)$basic_type$$constant);
12892   %}
12893   ins_pipe( pipe_slow );
12894 %}
12895 
12896 instruct count_positives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12897                          legRegD tmp1, legRegD tmp2, rbx_RegI tmp3, rFlagsReg cr,)
12898 %{
12899   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12900   match(Set result (CountPositives ary1 len));
12901   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12902 
12903   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12904   ins_encode %{
12905     __ count_positives($ary1$$Register, $len$$Register,
12906                        $result$$Register, $tmp3$$Register,
12907                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
12908   %}
12909   ins_pipe( pipe_slow );
12910 %}
12911 
12912 instruct count_positives_evex(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
12913                               legRegD tmp1, legRegD tmp2, kReg ktmp1, kReg ktmp2, rbx_RegI tmp3, rFlagsReg cr,)
12914 %{
12915   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12916   match(Set result (CountPositives ary1 len));
12917   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
12918 
12919   format %{ "countPositives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
12920   ins_encode %{
12921     __ count_positives($ary1$$Register, $len$$Register,
12922                        $result$$Register, $tmp3$$Register,
12923                        $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
12924   %}
12925   ins_pipe( pipe_slow );
12926 %}
12927 
12928 // fast char[] to byte[] compression
12929 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12930                          legRegD tmp4, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12931   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12932   match(Set result (StrCompressedCopy src (Binary dst len)));
12933   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst,
12934          USE_KILL len, KILL tmp5, KILL cr);
12935 
12936   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12937   ins_encode %{
12938     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12939                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12940                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12941                            knoreg, knoreg);
12942   %}
12943   ins_pipe( pipe_slow );
12944 %}
12945 
12946 instruct string_compress_evex(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legRegD tmp1, legRegD tmp2, legRegD tmp3,
12947                               legRegD tmp4, kReg ktmp1, kReg ktmp2, rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12948   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12949   match(Set result (StrCompressedCopy src (Binary dst len)));
12950   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst,
12951          USE_KILL len, KILL tmp5, KILL cr);
12952 
12953   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
12954   ins_encode %{
12955     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
12956                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
12957                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
12958                            $ktmp1$$KRegister, $ktmp2$$KRegister);
12959   %}
12960   ins_pipe( pipe_slow );
12961 %}
12962 // fast byte[] to char[] inflation
12963 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12964                         legRegD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
12965   predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
12966   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12967   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12968 
12969   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12970   ins_encode %{
12971     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12972                           $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
12973   %}
12974   ins_pipe( pipe_slow );
12975 %}
12976 
12977 instruct string_inflate_evex(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12978                              legRegD tmp1, kReg ktmp, rcx_RegI tmp2, rFlagsReg cr) %{
12979   predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
12980   match(Set dummy (StrInflatedCopy src (Binary dst len)));
12981   effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
12982 
12983   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
12984   ins_encode %{
12985     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
12986                           $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
12987   %}
12988   ins_pipe( pipe_slow );
12989 %}
12990 
12991 // encode char[] to byte[] in ISO_8859_1
12992 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
12993                           legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
12994                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
12995   predicate(!((EncodeISOArrayNode*)n)->is_ascii());
12996   match(Set result (EncodeISOArray src (Binary dst len)));
12997   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
12998 
12999   format %{ "Encode iso array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
13000   ins_encode %{
13001     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
13002                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13003                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
13004   %}
13005   ins_pipe( pipe_slow );
13006 %}
13007 
13008 // encode char[] to byte[] in ASCII
13009 instruct encode_ascii_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
13010                             legRegD tmp1, legRegD tmp2, legRegD tmp3, legRegD tmp4,
13011                             rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
13012   predicate(((EncodeISOArrayNode*)n)->is_ascii());
13013   match(Set result (EncodeISOArray src (Binary dst len)));
13014   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
13015 
13016   format %{ "Encode ascii array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
13017   ins_encode %{
13018     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
13019                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
13020                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
13021   %}
13022   ins_pipe( pipe_slow );
13023 %}
13024 
13025 //----------Overflow Math Instructions-----------------------------------------
13026 
13027 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13028 %{
13029   match(Set cr (OverflowAddI op1 op2));
13030   effect(DEF cr, USE_KILL op1, USE op2);
13031 
13032   format %{ "addl    $op1, $op2\t# overflow check int" %}
13033 
13034   ins_encode %{
13035     __ addl($op1$$Register, $op2$$Register);
13036   %}
13037   ins_pipe(ialu_reg_reg);
13038 %}
13039 
13040 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
13041 %{
13042   match(Set cr (OverflowAddI op1 op2));
13043   effect(DEF cr, USE_KILL op1, USE op2);
13044 
13045   format %{ "addl    $op1, $op2\t# overflow check int" %}
13046 
13047   ins_encode %{
13048     __ addl($op1$$Register, $op2$$constant);
13049   %}
13050   ins_pipe(ialu_reg_reg);
13051 %}
13052 
13053 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13054 %{
13055   match(Set cr (OverflowAddL op1 op2));
13056   effect(DEF cr, USE_KILL op1, USE op2);
13057 
13058   format %{ "addq    $op1, $op2\t# overflow check long" %}
13059   ins_encode %{
13060     __ addq($op1$$Register, $op2$$Register);
13061   %}
13062   ins_pipe(ialu_reg_reg);
13063 %}
13064 
13065 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
13066 %{
13067   match(Set cr (OverflowAddL op1 op2));
13068   effect(DEF cr, USE_KILL op1, USE op2);
13069 
13070   format %{ "addq    $op1, $op2\t# overflow check long" %}
13071   ins_encode %{
13072     __ addq($op1$$Register, $op2$$constant);
13073   %}
13074   ins_pipe(ialu_reg_reg);
13075 %}
13076 
13077 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13078 %{
13079   match(Set cr (OverflowSubI op1 op2));
13080 
13081   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13082   ins_encode %{
13083     __ cmpl($op1$$Register, $op2$$Register);
13084   %}
13085   ins_pipe(ialu_reg_reg);
13086 %}
13087 
13088 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13089 %{
13090   match(Set cr (OverflowSubI op1 op2));
13091 
13092   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
13093   ins_encode %{
13094     __ cmpl($op1$$Register, $op2$$constant);
13095   %}
13096   ins_pipe(ialu_reg_reg);
13097 %}
13098 
13099 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13100 %{
13101   match(Set cr (OverflowSubL op1 op2));
13102 
13103   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13104   ins_encode %{
13105     __ cmpq($op1$$Register, $op2$$Register);
13106   %}
13107   ins_pipe(ialu_reg_reg);
13108 %}
13109 
13110 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13111 %{
13112   match(Set cr (OverflowSubL op1 op2));
13113 
13114   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
13115   ins_encode %{
13116     __ cmpq($op1$$Register, $op2$$constant);
13117   %}
13118   ins_pipe(ialu_reg_reg);
13119 %}
13120 
13121 instruct overflowNegI_rReg(rFlagsReg cr, immI_0 zero, rax_RegI op2)
13122 %{
13123   match(Set cr (OverflowSubI zero op2));
13124   effect(DEF cr, USE_KILL op2);
13125 
13126   format %{ "negl    $op2\t# overflow check int" %}
13127   ins_encode %{
13128     __ negl($op2$$Register);
13129   %}
13130   ins_pipe(ialu_reg_reg);
13131 %}
13132 
13133 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
13134 %{
13135   match(Set cr (OverflowSubL zero op2));
13136   effect(DEF cr, USE_KILL op2);
13137 
13138   format %{ "negq    $op2\t# overflow check long" %}
13139   ins_encode %{
13140     __ negq($op2$$Register);
13141   %}
13142   ins_pipe(ialu_reg_reg);
13143 %}
13144 
13145 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
13146 %{
13147   match(Set cr (OverflowMulI op1 op2));
13148   effect(DEF cr, USE_KILL op1, USE op2);
13149 
13150   format %{ "imull    $op1, $op2\t# overflow check int" %}
13151   ins_encode %{
13152     __ imull($op1$$Register, $op2$$Register);
13153   %}
13154   ins_pipe(ialu_reg_reg_alu0);
13155 %}
13156 
13157 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
13158 %{
13159   match(Set cr (OverflowMulI op1 op2));
13160   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13161 
13162   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
13163   ins_encode %{
13164     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
13165   %}
13166   ins_pipe(ialu_reg_reg_alu0);
13167 %}
13168 
13169 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
13170 %{
13171   match(Set cr (OverflowMulL op1 op2));
13172   effect(DEF cr, USE_KILL op1, USE op2);
13173 
13174   format %{ "imulq    $op1, $op2\t# overflow check long" %}
13175   ins_encode %{
13176     __ imulq($op1$$Register, $op2$$Register);
13177   %}
13178   ins_pipe(ialu_reg_reg_alu0);
13179 %}
13180 
13181 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
13182 %{
13183   match(Set cr (OverflowMulL op1 op2));
13184   effect(DEF cr, TEMP tmp, USE op1, USE op2);
13185 
13186   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
13187   ins_encode %{
13188     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
13189   %}
13190   ins_pipe(ialu_reg_reg_alu0);
13191 %}
13192 
13193 
13194 //----------Control Flow Instructions------------------------------------------
13195 // Signed compare Instructions
13196 
13197 // XXX more variants!!
13198 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
13199 %{
13200   match(Set cr (CmpI op1 op2));
13201   effect(DEF cr, USE op1, USE op2);
13202 
13203   format %{ "cmpl    $op1, $op2" %}
13204   ins_encode %{
13205     __ cmpl($op1$$Register, $op2$$Register);
13206   %}
13207   ins_pipe(ialu_cr_reg_reg);
13208 %}
13209 
13210 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
13211 %{
13212   match(Set cr (CmpI op1 op2));
13213 
13214   format %{ "cmpl    $op1, $op2" %}
13215   ins_encode %{
13216     __ cmpl($op1$$Register, $op2$$constant);
13217   %}
13218   ins_pipe(ialu_cr_reg_imm);
13219 %}
13220 
13221 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
13222 %{
13223   match(Set cr (CmpI op1 (LoadI op2)));
13224 
13225   ins_cost(500); // XXX
13226   format %{ "cmpl    $op1, $op2" %}
13227   ins_encode %{
13228     __ cmpl($op1$$Register, $op2$$Address);
13229   %}
13230   ins_pipe(ialu_cr_reg_mem);
13231 %}
13232 
13233 instruct testI_reg(rFlagsReg cr, rRegI src, immI_0 zero)
13234 %{
13235   match(Set cr (CmpI src zero));
13236 
13237   format %{ "testl   $src, $src" %}
13238   ins_encode %{
13239     __ testl($src$$Register, $src$$Register);
13240   %}
13241   ins_pipe(ialu_cr_reg_imm);
13242 %}
13243 
13244 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI_0 zero)
13245 %{
13246   match(Set cr (CmpI (AndI src con) zero));
13247 
13248   format %{ "testl   $src, $con" %}
13249   ins_encode %{
13250     __ testl($src$$Register, $con$$constant);
13251   %}
13252   ins_pipe(ialu_cr_reg_imm);
13253 %}
13254 
13255 instruct testI_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2, immI_0 zero)
13256 %{
13257   match(Set cr (CmpI (AndI src1 src2) zero));
13258 
13259   format %{ "testl   $src1, $src2" %}
13260   ins_encode %{
13261     __ testl($src1$$Register, $src2$$Register);
13262   %}
13263   ins_pipe(ialu_cr_reg_imm);
13264 %}
13265 
13266 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI_0 zero)
13267 %{
13268   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
13269 
13270   format %{ "testl   $src, $mem" %}
13271   ins_encode %{
13272     __ testl($src$$Register, $mem$$Address);
13273   %}
13274   ins_pipe(ialu_cr_reg_mem);
13275 %}
13276 
13277 // Unsigned compare Instructions; really, same as signed except they
13278 // produce an rFlagsRegU instead of rFlagsReg.
13279 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
13280 %{
13281   match(Set cr (CmpU op1 op2));
13282 
13283   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13284   ins_encode %{
13285     __ cmpl($op1$$Register, $op2$$Register);
13286   %}
13287   ins_pipe(ialu_cr_reg_reg);
13288 %}
13289 
13290 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
13291 %{
13292   match(Set cr (CmpU op1 op2));
13293 
13294   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13295   ins_encode %{
13296     __ cmpl($op1$$Register, $op2$$constant);
13297   %}
13298   ins_pipe(ialu_cr_reg_imm);
13299 %}
13300 
13301 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
13302 %{
13303   match(Set cr (CmpU op1 (LoadI op2)));
13304 
13305   ins_cost(500); // XXX
13306   format %{ "cmpl    $op1, $op2\t# unsigned" %}
13307   ins_encode %{
13308     __ cmpl($op1$$Register, $op2$$Address);
13309   %}
13310   ins_pipe(ialu_cr_reg_mem);
13311 %}
13312 
13313 instruct testU_reg(rFlagsRegU cr, rRegI src, immI_0 zero)
13314 %{
13315   match(Set cr (CmpU src zero));
13316 
13317   format %{ "testl   $src, $src\t# unsigned" %}
13318   ins_encode %{
13319     __ testl($src$$Register, $src$$Register);
13320   %}
13321   ins_pipe(ialu_cr_reg_imm);
13322 %}
13323 
13324 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
13325 %{
13326   match(Set cr (CmpP op1 op2));
13327 
13328   format %{ "cmpq    $op1, $op2\t# ptr" %}
13329   ins_encode %{
13330     __ cmpq($op1$$Register, $op2$$Register);
13331   %}
13332   ins_pipe(ialu_cr_reg_reg);
13333 %}
13334 
13335 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
13336 %{
13337   match(Set cr (CmpP op1 (LoadP op2)));
13338   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13339 
13340   ins_cost(500); // XXX
13341   format %{ "cmpq    $op1, $op2\t# ptr" %}
13342   ins_encode %{
13343     __ cmpq($op1$$Register, $op2$$Address);
13344   %}
13345   ins_pipe(ialu_cr_reg_mem);
13346 %}
13347 
13348 // XXX this is generalized by compP_rReg_mem???
13349 // Compare raw pointer (used in out-of-heap check).
13350 // Only works because non-oop pointers must be raw pointers
13351 // and raw pointers have no anti-dependencies.
13352 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
13353 %{
13354   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none &&
13355             n->in(2)->as_Load()->barrier_data() == 0);
13356   match(Set cr (CmpP op1 (LoadP op2)));
13357 
13358   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
13359   ins_encode %{
13360     __ cmpq($op1$$Register, $op2$$Address);
13361   %}
13362   ins_pipe(ialu_cr_reg_mem);
13363 %}
13364 
13365 // This will generate a signed flags result. This should be OK since
13366 // any compare to a zero should be eq/neq.
13367 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
13368 %{
13369   match(Set cr (CmpP src zero));
13370 
13371   format %{ "testq   $src, $src\t# ptr" %}
13372   ins_encode %{
13373     __ testq($src$$Register, $src$$Register);
13374   %}
13375   ins_pipe(ialu_cr_reg_imm);
13376 %}
13377 
13378 // This will generate a signed flags result. This should be OK since
13379 // any compare to a zero should be eq/neq.
13380 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
13381 %{
13382   predicate((!UseCompressedOops || (CompressedOops::base() != nullptr)) &&
13383             n->in(1)->as_Load()->barrier_data() == 0);
13384   match(Set cr (CmpP (LoadP op) zero));
13385 
13386   ins_cost(500); // XXX
13387   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
13388   ins_encode %{
13389     __ testq($op$$Address, 0xFFFFFFFF);
13390   %}
13391   ins_pipe(ialu_cr_reg_imm);
13392 %}
13393 
13394 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
13395 %{
13396   predicate(UseCompressedOops && (CompressedOops::base() == nullptr) &&
13397             n->in(1)->as_Load()->barrier_data() == 0);
13398   match(Set cr (CmpP (LoadP mem) zero));
13399 
13400   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
13401   ins_encode %{
13402     __ cmpq(r12, $mem$$Address);
13403   %}
13404   ins_pipe(ialu_cr_reg_mem);
13405 %}
13406 
13407 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
13408 %{
13409   match(Set cr (CmpN op1 op2));
13410 
13411   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13412   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
13413   ins_pipe(ialu_cr_reg_reg);
13414 %}
13415 
13416 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
13417 %{
13418   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13419   match(Set cr (CmpN src (LoadN mem)));
13420 
13421   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
13422   ins_encode %{
13423     __ cmpl($src$$Register, $mem$$Address);
13424   %}
13425   ins_pipe(ialu_cr_reg_mem);
13426 %}
13427 
13428 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
13429   match(Set cr (CmpN op1 op2));
13430 
13431   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
13432   ins_encode %{
13433     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
13434   %}
13435   ins_pipe(ialu_cr_reg_imm);
13436 %}
13437 
13438 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
13439 %{
13440   predicate(n->in(2)->as_Load()->barrier_data() == 0);
13441   match(Set cr (CmpN src (LoadN mem)));
13442 
13443   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
13444   ins_encode %{
13445     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
13446   %}
13447   ins_pipe(ialu_cr_reg_mem);
13448 %}
13449 
13450 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
13451   match(Set cr (CmpN op1 op2));
13452 
13453   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
13454   ins_encode %{
13455     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
13456   %}
13457   ins_pipe(ialu_cr_reg_imm);
13458 %}
13459 
13460 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
13461 %{
13462   predicate(!UseCompactObjectHeaders);
13463   match(Set cr (CmpN src (LoadNKlass mem)));
13464 
13465   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
13466   ins_encode %{
13467     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
13468   %}
13469   ins_pipe(ialu_cr_reg_mem);
13470 %}
13471 
13472 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
13473   match(Set cr (CmpN src zero));
13474 
13475   format %{ "testl   $src, $src\t# compressed ptr" %}
13476   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
13477   ins_pipe(ialu_cr_reg_imm);
13478 %}
13479 
13480 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
13481 %{
13482   predicate(CompressedOops::base() != nullptr &&
13483             n->in(1)->as_Load()->barrier_data() == 0);
13484   match(Set cr (CmpN (LoadN mem) zero));
13485 
13486   ins_cost(500); // XXX
13487   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
13488   ins_encode %{
13489     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
13490   %}
13491   ins_pipe(ialu_cr_reg_mem);
13492 %}
13493 
13494 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
13495 %{
13496   predicate(CompressedOops::base() == nullptr &&
13497             n->in(1)->as_Load()->barrier_data() == 0);
13498   match(Set cr (CmpN (LoadN mem) zero));
13499 
13500   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
13501   ins_encode %{
13502     __ cmpl(r12, $mem$$Address);
13503   %}
13504   ins_pipe(ialu_cr_reg_mem);
13505 %}
13506 
13507 // Yanked all unsigned pointer compare operations.
13508 // Pointer compares are done with CmpP which is already unsigned.
13509 
13510 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
13511 %{
13512   match(Set cr (CmpL op1 op2));
13513 
13514   format %{ "cmpq    $op1, $op2" %}
13515   ins_encode %{
13516     __ cmpq($op1$$Register, $op2$$Register);
13517   %}
13518   ins_pipe(ialu_cr_reg_reg);
13519 %}
13520 
13521 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
13522 %{
13523   match(Set cr (CmpL op1 op2));
13524 
13525   format %{ "cmpq    $op1, $op2" %}
13526   ins_encode %{
13527     __ cmpq($op1$$Register, $op2$$constant);
13528   %}
13529   ins_pipe(ialu_cr_reg_imm);
13530 %}
13531 
13532 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
13533 %{
13534   match(Set cr (CmpL op1 (LoadL op2)));
13535 
13536   format %{ "cmpq    $op1, $op2" %}
13537   ins_encode %{
13538     __ cmpq($op1$$Register, $op2$$Address);
13539   %}
13540   ins_pipe(ialu_cr_reg_mem);
13541 %}
13542 
13543 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
13544 %{
13545   match(Set cr (CmpL src zero));
13546 
13547   format %{ "testq   $src, $src" %}
13548   ins_encode %{
13549     __ testq($src$$Register, $src$$Register);
13550   %}
13551   ins_pipe(ialu_cr_reg_imm);
13552 %}
13553 
13554 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
13555 %{
13556   match(Set cr (CmpL (AndL src con) zero));
13557 
13558   format %{ "testq   $src, $con\t# long" %}
13559   ins_encode %{
13560     __ testq($src$$Register, $con$$constant);
13561   %}
13562   ins_pipe(ialu_cr_reg_imm);
13563 %}
13564 
13565 instruct testL_reg_reg(rFlagsReg cr, rRegL src1, rRegL src2, immL0 zero)
13566 %{
13567   match(Set cr (CmpL (AndL src1 src2) zero));
13568 
13569   format %{ "testq   $src1, $src2\t# long" %}
13570   ins_encode %{
13571     __ testq($src1$$Register, $src2$$Register);
13572   %}
13573   ins_pipe(ialu_cr_reg_imm);
13574 %}
13575 
13576 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
13577 %{
13578   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
13579 
13580   format %{ "testq   $src, $mem" %}
13581   ins_encode %{
13582     __ testq($src$$Register, $mem$$Address);
13583   %}
13584   ins_pipe(ialu_cr_reg_mem);
13585 %}
13586 
13587 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
13588 %{
13589   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
13590 
13591   format %{ "testq   $src, $mem" %}
13592   ins_encode %{
13593     __ testq($src$$Register, $mem$$Address);
13594   %}
13595   ins_pipe(ialu_cr_reg_mem);
13596 %}
13597 
13598 // Manifest a CmpU result in an integer register.  Very painful.
13599 // This is the test to avoid.
13600 instruct cmpU3_reg_reg(rRegI dst, rRegI src1, rRegI src2, rFlagsReg flags)
13601 %{
13602   match(Set dst (CmpU3 src1 src2));
13603   effect(KILL flags);
13604 
13605   ins_cost(275); // XXX
13606   format %{ "cmpl    $src1, $src2\t# CmpL3\n\t"
13607             "movl    $dst, -1\n\t"
13608             "jb,u    done\n\t"
13609             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13610     "done:" %}
13611   ins_encode %{
13612     Label done;
13613     __ cmpl($src1$$Register, $src2$$Register);
13614     __ movl($dst$$Register, -1);
13615     __ jccb(Assembler::below, done);
13616     __ setcc(Assembler::notZero, $dst$$Register);
13617     __ bind(done);
13618   %}
13619   ins_pipe(pipe_slow);
13620 %}
13621 
13622 // Manifest a CmpL result in an integer register.  Very painful.
13623 // This is the test to avoid.
13624 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13625 %{
13626   match(Set dst (CmpL3 src1 src2));
13627   effect(KILL flags);
13628 
13629   ins_cost(275); // XXX
13630   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13631             "movl    $dst, -1\n\t"
13632             "jl,s    done\n\t"
13633             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13634     "done:" %}
13635   ins_encode %{
13636     Label done;
13637     __ cmpq($src1$$Register, $src2$$Register);
13638     __ movl($dst$$Register, -1);
13639     __ jccb(Assembler::less, done);
13640     __ setcc(Assembler::notZero, $dst$$Register);
13641     __ bind(done);
13642   %}
13643   ins_pipe(pipe_slow);
13644 %}
13645 
13646 // Manifest a CmpUL result in an integer register.  Very painful.
13647 // This is the test to avoid.
13648 instruct cmpUL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
13649 %{
13650   match(Set dst (CmpUL3 src1 src2));
13651   effect(KILL flags);
13652 
13653   ins_cost(275); // XXX
13654   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
13655             "movl    $dst, -1\n\t"
13656             "jb,u    done\n\t"
13657             "setcc   $dst \t# emits setne + movzbl or setzune for APX"
13658     "done:" %}
13659   ins_encode %{
13660     Label done;
13661     __ cmpq($src1$$Register, $src2$$Register);
13662     __ movl($dst$$Register, -1);
13663     __ jccb(Assembler::below, done);
13664     __ setcc(Assembler::notZero, $dst$$Register);
13665     __ bind(done);
13666   %}
13667   ins_pipe(pipe_slow);
13668 %}
13669 
13670 // Unsigned long compare Instructions; really, same as signed long except they
13671 // produce an rFlagsRegU instead of rFlagsReg.
13672 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
13673 %{
13674   match(Set cr (CmpUL op1 op2));
13675 
13676   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13677   ins_encode %{
13678     __ cmpq($op1$$Register, $op2$$Register);
13679   %}
13680   ins_pipe(ialu_cr_reg_reg);
13681 %}
13682 
13683 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
13684 %{
13685   match(Set cr (CmpUL op1 op2));
13686 
13687   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13688   ins_encode %{
13689     __ cmpq($op1$$Register, $op2$$constant);
13690   %}
13691   ins_pipe(ialu_cr_reg_imm);
13692 %}
13693 
13694 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
13695 %{
13696   match(Set cr (CmpUL op1 (LoadL op2)));
13697 
13698   format %{ "cmpq    $op1, $op2\t# unsigned" %}
13699   ins_encode %{
13700     __ cmpq($op1$$Register, $op2$$Address);
13701   %}
13702   ins_pipe(ialu_cr_reg_mem);
13703 %}
13704 
13705 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
13706 %{
13707   match(Set cr (CmpUL src zero));
13708 
13709   format %{ "testq   $src, $src\t# unsigned" %}
13710   ins_encode %{
13711     __ testq($src$$Register, $src$$Register);
13712   %}
13713   ins_pipe(ialu_cr_reg_imm);
13714 %}
13715 
13716 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
13717 %{
13718   match(Set cr (CmpI (LoadB mem) imm));
13719 
13720   ins_cost(125);
13721   format %{ "cmpb    $mem, $imm" %}
13722   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
13723   ins_pipe(ialu_cr_reg_mem);
13724 %}
13725 
13726 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU7 imm, immI_0 zero)
13727 %{
13728   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
13729 
13730   ins_cost(125);
13731   format %{ "testb   $mem, $imm\t# ubyte" %}
13732   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13733   ins_pipe(ialu_cr_reg_mem);
13734 %}
13735 
13736 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
13737 %{
13738   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
13739 
13740   ins_cost(125);
13741   format %{ "testb   $mem, $imm\t# byte" %}
13742   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
13743   ins_pipe(ialu_cr_reg_mem);
13744 %}
13745 
13746 //----------Max and Min--------------------------------------------------------
13747 // Min Instructions
13748 
13749 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
13750 %{
13751   predicate(!UseAPX);
13752   effect(USE_DEF dst, USE src, USE cr);
13753 
13754   format %{ "cmovlgt $dst, $src\t# min" %}
13755   ins_encode %{
13756     __ cmovl(Assembler::greater, $dst$$Register, $src$$Register);
13757   %}
13758   ins_pipe(pipe_cmov_reg);
13759 %}
13760 
13761 instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13762 %{
13763   predicate(UseAPX);
13764   effect(DEF dst, USE src1, USE src2, USE cr);
13765 
13766   format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
13767   ins_encode %{
13768     __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
13769   %}
13770   ins_pipe(pipe_cmov_reg);
13771 %}
13772 
13773 instruct minI_rReg(rRegI dst, rRegI src)
13774 %{
13775   predicate(!UseAPX);
13776   match(Set dst (MinI dst src));
13777 
13778   ins_cost(200);
13779   expand %{
13780     rFlagsReg cr;
13781     compI_rReg(cr, dst, src);
13782     cmovI_reg_g(dst, src, cr);
13783   %}
13784 %}
13785 
13786 instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13787 %{
13788   predicate(UseAPX);
13789   match(Set dst (MinI src1 src2));
13790   effect(DEF dst, USE src1, USE src2);
13791 
13792   ins_cost(200);
13793   expand %{
13794     rFlagsReg cr;
13795     compI_rReg(cr, src1, src2);
13796     cmovI_reg_g_ndd(dst, src1, src2, cr);
13797   %}
13798 %}
13799 
13800 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
13801 %{
13802   predicate(!UseAPX);
13803   effect(USE_DEF dst, USE src, USE cr);
13804 
13805   format %{ "cmovllt $dst, $src\t# max" %}
13806   ins_encode %{
13807     __ cmovl(Assembler::less, $dst$$Register, $src$$Register);
13808   %}
13809   ins_pipe(pipe_cmov_reg);
13810 %}
13811 
13812 instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
13813 %{
13814   predicate(UseAPX);
13815   effect(DEF dst, USE src1, USE src2, USE cr);
13816 
13817   format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
13818   ins_encode %{
13819     __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
13820   %}
13821   ins_pipe(pipe_cmov_reg);
13822 %}
13823 
13824 instruct maxI_rReg(rRegI dst, rRegI src)
13825 %{
13826   predicate(!UseAPX);
13827   match(Set dst (MaxI dst src));
13828 
13829   ins_cost(200);
13830   expand %{
13831     rFlagsReg cr;
13832     compI_rReg(cr, dst, src);
13833     cmovI_reg_l(dst, src, cr);
13834   %}
13835 %}
13836 
13837 instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
13838 %{
13839   predicate(UseAPX);
13840   match(Set dst (MaxI src1 src2));
13841   effect(DEF dst, USE src1, USE src2);
13842 
13843   ins_cost(200);
13844   expand %{
13845     rFlagsReg cr;
13846     compI_rReg(cr, src1, src2);
13847     cmovI_reg_l_ndd(dst, src1, src2, cr);
13848   %}
13849 %}
13850 
13851 // ============================================================================
13852 // Branch Instructions
13853 
13854 // Jump Direct - Label defines a relative address from JMP+1
13855 instruct jmpDir(label labl)
13856 %{
13857   match(Goto);
13858   effect(USE labl);
13859 
13860   ins_cost(300);
13861   format %{ "jmp     $labl" %}
13862   size(5);
13863   ins_encode %{
13864     Label* L = $labl$$label;
13865     __ jmp(*L, false); // Always long jump
13866   %}
13867   ins_pipe(pipe_jmp);
13868 %}
13869 
13870 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13871 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
13872 %{
13873   match(If cop cr);
13874   effect(USE labl);
13875 
13876   ins_cost(300);
13877   format %{ "j$cop     $labl" %}
13878   size(6);
13879   ins_encode %{
13880     Label* L = $labl$$label;
13881     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13882   %}
13883   ins_pipe(pipe_jcc);
13884 %}
13885 
13886 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13887 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
13888 %{
13889   match(CountedLoopEnd cop cr);
13890   effect(USE labl);
13891 
13892   ins_cost(300);
13893   format %{ "j$cop     $labl\t# loop end" %}
13894   size(6);
13895   ins_encode %{
13896     Label* L = $labl$$label;
13897     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13898   %}
13899   ins_pipe(pipe_jcc);
13900 %}
13901 
13902 // Jump Direct Conditional - using unsigned comparison
13903 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
13904   match(If cop cmp);
13905   effect(USE labl);
13906 
13907   ins_cost(300);
13908   format %{ "j$cop,u   $labl" %}
13909   size(6);
13910   ins_encode %{
13911     Label* L = $labl$$label;
13912     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13913   %}
13914   ins_pipe(pipe_jcc);
13915 %}
13916 
13917 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
13918   match(If cop cmp);
13919   effect(USE labl);
13920 
13921   ins_cost(200);
13922   format %{ "j$cop,u   $labl" %}
13923   size(6);
13924   ins_encode %{
13925     Label* L = $labl$$label;
13926     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13927   %}
13928   ins_pipe(pipe_jcc);
13929 %}
13930 
13931 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
13932   match(If cop cmp);
13933   effect(USE labl);
13934 
13935   ins_cost(200);
13936   format %{ $$template
13937     if ($cop$$cmpcode == Assembler::notEqual) {
13938       $$emit$$"jp,u    $labl\n\t"
13939       $$emit$$"j$cop,u   $labl"
13940     } else {
13941       $$emit$$"jp,u    done\n\t"
13942       $$emit$$"j$cop,u   $labl\n\t"
13943       $$emit$$"done:"
13944     }
13945   %}
13946   ins_encode %{
13947     Label* l = $labl$$label;
13948     if ($cop$$cmpcode == Assembler::notEqual) {
13949       __ jcc(Assembler::parity, *l, false);
13950       __ jcc(Assembler::notEqual, *l, false);
13951     } else if ($cop$$cmpcode == Assembler::equal) {
13952       Label done;
13953       __ jccb(Assembler::parity, done);
13954       __ jcc(Assembler::equal, *l, false);
13955       __ bind(done);
13956     } else {
13957        ShouldNotReachHere();
13958     }
13959   %}
13960   ins_pipe(pipe_jcc);
13961 %}
13962 
13963 // ============================================================================
13964 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
13965 // superklass array for an instance of the superklass.  Set a hidden
13966 // internal cache on a hit (cache is checked with exposed code in
13967 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
13968 // encoding ALSO sets flags.
13969 
13970 instruct partialSubtypeCheck(rdi_RegP result,
13971                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
13972                              rFlagsReg cr)
13973 %{
13974   match(Set result (PartialSubtypeCheck sub super));
13975   predicate(!UseSecondarySupersTable);
13976   effect(KILL rcx, KILL cr);
13977 
13978   ins_cost(1100);  // slightly larger than the next version
13979   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
13980             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
13981             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
13982             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
13983             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
13984             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
13985             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
13986     "miss:\t" %}
13987 
13988   ins_encode %{
13989     Label miss;
13990     // NB: Callers may assume that, when $result is a valid register,
13991     // check_klass_subtype_slow_path_linear sets it to a nonzero
13992     // value.
13993     __ check_klass_subtype_slow_path_linear($sub$$Register, $super$$Register,
13994                                             $rcx$$Register, $result$$Register,
13995                                             nullptr, &miss,
13996                                             /*set_cond_codes:*/ true);
13997     __ xorptr($result$$Register, $result$$Register);
13998     __ bind(miss);
13999   %}
14000 
14001   ins_pipe(pipe_slow);
14002 %}
14003 
14004 // ============================================================================
14005 // Two versions of hashtable-based partialSubtypeCheck, both used when
14006 // we need to search for a super class in the secondary supers array.
14007 // The first is used when we don't know _a priori_ the class being
14008 // searched for. The second, far more common, is used when we do know:
14009 // this is used for instanceof, checkcast, and any case where C2 can
14010 // determine it by constant propagation.
14011 
14012 instruct partialSubtypeCheckVarSuper(rsi_RegP sub, rax_RegP super, rdi_RegP result,
14013                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14014                                        rFlagsReg cr)
14015 %{
14016   match(Set result (PartialSubtypeCheck sub super));
14017   predicate(UseSecondarySupersTable);
14018   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14019 
14020   ins_cost(1000);
14021   format %{ "partialSubtypeCheck $result, $sub, $super" %}
14022 
14023   ins_encode %{
14024     __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register, $temp1$$Register, $temp2$$Register,
14025 					 $temp3$$Register, $temp4$$Register, $result$$Register);
14026   %}
14027 
14028   ins_pipe(pipe_slow);
14029 %}
14030 
14031 instruct partialSubtypeCheckConstSuper(rsi_RegP sub, rax_RegP super_reg, immP super_con, rdi_RegP result,
14032                                        rdx_RegL temp1, rcx_RegL temp2, rbx_RegP temp3, r11_RegL temp4,
14033                                        rFlagsReg cr)
14034 %{
14035   match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
14036   predicate(UseSecondarySupersTable);
14037   effect(KILL cr, TEMP temp1, TEMP temp2, TEMP temp3, TEMP temp4);
14038 
14039   ins_cost(700);  // smaller than the next version
14040   format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
14041 
14042   ins_encode %{
14043     u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
14044     if (InlineSecondarySupersTest) {
14045       __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register, $temp1$$Register, $temp2$$Register,
14046                                        $temp3$$Register, $temp4$$Register, $result$$Register,
14047                                        super_klass_slot);
14048     } else {
14049       __ call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
14050     }
14051   %}
14052 
14053   ins_pipe(pipe_slow);
14054 %}
14055 
14056 // ============================================================================
14057 // Branch Instructions -- short offset versions
14058 //
14059 // These instructions are used to replace jumps of a long offset (the default
14060 // match) with jumps of a shorter offset.  These instructions are all tagged
14061 // with the ins_short_branch attribute, which causes the ADLC to suppress the
14062 // match rules in general matching.  Instead, the ADLC generates a conversion
14063 // method in the MachNode which can be used to do in-place replacement of the
14064 // long variant with the shorter variant.  The compiler will determine if a
14065 // branch can be taken by the is_short_branch_offset() predicate in the machine
14066 // specific code section of the file.
14067 
14068 // Jump Direct - Label defines a relative address from JMP+1
14069 instruct jmpDir_short(label labl) %{
14070   match(Goto);
14071   effect(USE labl);
14072 
14073   ins_cost(300);
14074   format %{ "jmp,s   $labl" %}
14075   size(2);
14076   ins_encode %{
14077     Label* L = $labl$$label;
14078     __ jmpb(*L);
14079   %}
14080   ins_pipe(pipe_jmp);
14081   ins_short_branch(1);
14082 %}
14083 
14084 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14085 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
14086   match(If cop cr);
14087   effect(USE labl);
14088 
14089   ins_cost(300);
14090   format %{ "j$cop,s   $labl" %}
14091   size(2);
14092   ins_encode %{
14093     Label* L = $labl$$label;
14094     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14095   %}
14096   ins_pipe(pipe_jcc);
14097   ins_short_branch(1);
14098 %}
14099 
14100 // Jump Direct Conditional - Label defines a relative address from Jcc+1
14101 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
14102   match(CountedLoopEnd cop cr);
14103   effect(USE labl);
14104 
14105   ins_cost(300);
14106   format %{ "j$cop,s   $labl\t# loop end" %}
14107   size(2);
14108   ins_encode %{
14109     Label* L = $labl$$label;
14110     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14111   %}
14112   ins_pipe(pipe_jcc);
14113   ins_short_branch(1);
14114 %}
14115 
14116 // Jump Direct Conditional - using unsigned comparison
14117 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
14118   match(If cop cmp);
14119   effect(USE labl);
14120 
14121   ins_cost(300);
14122   format %{ "j$cop,us  $labl" %}
14123   size(2);
14124   ins_encode %{
14125     Label* L = $labl$$label;
14126     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14127   %}
14128   ins_pipe(pipe_jcc);
14129   ins_short_branch(1);
14130 %}
14131 
14132 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
14133   match(If cop cmp);
14134   effect(USE labl);
14135 
14136   ins_cost(300);
14137   format %{ "j$cop,us  $labl" %}
14138   size(2);
14139   ins_encode %{
14140     Label* L = $labl$$label;
14141     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
14142   %}
14143   ins_pipe(pipe_jcc);
14144   ins_short_branch(1);
14145 %}
14146 
14147 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
14148   match(If cop cmp);
14149   effect(USE labl);
14150 
14151   ins_cost(300);
14152   format %{ $$template
14153     if ($cop$$cmpcode == Assembler::notEqual) {
14154       $$emit$$"jp,u,s  $labl\n\t"
14155       $$emit$$"j$cop,u,s  $labl"
14156     } else {
14157       $$emit$$"jp,u,s  done\n\t"
14158       $$emit$$"j$cop,u,s  $labl\n\t"
14159       $$emit$$"done:"
14160     }
14161   %}
14162   size(4);
14163   ins_encode %{
14164     Label* l = $labl$$label;
14165     if ($cop$$cmpcode == Assembler::notEqual) {
14166       __ jccb(Assembler::parity, *l);
14167       __ jccb(Assembler::notEqual, *l);
14168     } else if ($cop$$cmpcode == Assembler::equal) {
14169       Label done;
14170       __ jccb(Assembler::parity, done);
14171       __ jccb(Assembler::equal, *l);
14172       __ bind(done);
14173     } else {
14174        ShouldNotReachHere();
14175     }
14176   %}
14177   ins_pipe(pipe_jcc);
14178   ins_short_branch(1);
14179 %}
14180 
14181 // ============================================================================
14182 // inlined locking and unlocking
14183 
14184 instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
14185   match(Set cr (FastLock object box));
14186   effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
14187   ins_cost(300);
14188   format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
14189   ins_encode %{
14190     __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14191   %}
14192   ins_pipe(pipe_slow);
14193 %}
14194 
14195 instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
14196   match(Set cr (FastUnlock object rax_reg));
14197   effect(TEMP tmp, USE_KILL rax_reg);
14198   ins_cost(300);
14199   format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
14200   ins_encode %{
14201     __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
14202   %}
14203   ins_pipe(pipe_slow);
14204 %}
14205 
14206 
14207 // ============================================================================
14208 // Safepoint Instructions
14209 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
14210 %{
14211   match(SafePoint poll);
14212   effect(KILL cr, USE poll);
14213 
14214   format %{ "testl   rax, [$poll]\t"
14215             "# Safepoint: poll for GC" %}
14216   ins_cost(125);
14217   ins_encode %{
14218     __ relocate(relocInfo::poll_type);
14219     address pre_pc = __ pc();
14220     __ testl(rax, Address($poll$$Register, 0));
14221     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
14222   %}
14223   ins_pipe(ialu_reg_mem);
14224 %}
14225 
14226 instruct mask_all_evexL(kReg dst, rRegL src) %{
14227   match(Set dst (MaskAll src));
14228   format %{ "mask_all_evexL $dst, $src \t! mask all operation" %}
14229   ins_encode %{
14230     int mask_len = Matcher::vector_length(this);
14231     __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
14232   %}
14233   ins_pipe( pipe_slow );
14234 %}
14235 
14236 instruct mask_all_evexI_GT32(kReg dst, rRegI src, rRegL tmp) %{
14237   predicate(Matcher::vector_length(n) > 32);
14238   match(Set dst (MaskAll src));
14239   effect(TEMP tmp);
14240   format %{ "mask_all_evexI_GT32 $dst, $src \t! using $tmp as TEMP" %}
14241   ins_encode %{
14242     int mask_len = Matcher::vector_length(this);
14243     __ movslq($tmp$$Register, $src$$Register);
14244     __ vector_maskall_operation($dst$$KRegister, $tmp$$Register, mask_len);
14245   %}
14246   ins_pipe( pipe_slow );
14247 %}
14248 
14249 // ============================================================================
14250 // Procedure Call/Return Instructions
14251 // Call Java Static Instruction
14252 // Note: If this code changes, the corresponding ret_addr_offset() and
14253 //       compute_padding() functions will have to be adjusted.
14254 instruct CallStaticJavaDirect(method meth) %{
14255   match(CallStaticJava);
14256   effect(USE meth);
14257 
14258   ins_cost(300);
14259   format %{ "call,static " %}
14260   opcode(0xE8); /* E8 cd */
14261   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
14262   ins_pipe(pipe_slow);
14263   ins_alignment(4);
14264 %}
14265 
14266 // Call Java Dynamic Instruction
14267 // Note: If this code changes, the corresponding ret_addr_offset() and
14268 //       compute_padding() functions will have to be adjusted.
14269 instruct CallDynamicJavaDirect(method meth)
14270 %{
14271   match(CallDynamicJava);
14272   effect(USE meth);
14273 
14274   ins_cost(300);
14275   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
14276             "call,dynamic " %}
14277   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
14278   ins_pipe(pipe_slow);
14279   ins_alignment(4);
14280 %}
14281 
14282 // Call Runtime Instruction
14283 instruct CallRuntimeDirect(method meth)
14284 %{
14285   match(CallRuntime);
14286   effect(USE meth);
14287 
14288   ins_cost(300);
14289   format %{ "call,runtime " %}
14290   ins_encode(clear_avx, Java_To_Runtime(meth));
14291   ins_pipe(pipe_slow);
14292 %}
14293 
14294 // Call runtime without safepoint
14295 instruct CallLeafDirect(method meth)
14296 %{
14297   match(CallLeaf);
14298   effect(USE meth);
14299 
14300   ins_cost(300);
14301   format %{ "call_leaf,runtime " %}
14302   ins_encode(clear_avx, Java_To_Runtime(meth));
14303   ins_pipe(pipe_slow);
14304 %}
14305 
14306 // Call runtime without safepoint and with vector arguments
14307 instruct CallLeafDirectVector(method meth)
14308 %{
14309   match(CallLeafVector);
14310   effect(USE meth);
14311 
14312   ins_cost(300);
14313   format %{ "call_leaf,vector " %}
14314   ins_encode(Java_To_Runtime(meth));
14315   ins_pipe(pipe_slow);
14316 %}
14317 
14318 // Call runtime without safepoint
14319 instruct CallLeafNoFPDirect(method meth)
14320 %{
14321   match(CallLeafNoFP);
14322   effect(USE meth);
14323 
14324   ins_cost(300);
14325   format %{ "call_leaf_nofp,runtime " %}
14326   ins_encode(clear_avx, Java_To_Runtime(meth));
14327   ins_pipe(pipe_slow);
14328 %}
14329 
14330 // Return Instruction
14331 // Remove the return address & jump to it.
14332 // Notice: We always emit a nop after a ret to make sure there is room
14333 // for safepoint patching
14334 instruct Ret()
14335 %{
14336   match(Return);
14337 
14338   format %{ "ret" %}
14339   ins_encode %{
14340     __ ret(0);
14341   %}
14342   ins_pipe(pipe_jmp);
14343 %}
14344 
14345 // Tail Call; Jump from runtime stub to Java code.
14346 // Also known as an 'interprocedural jump'.
14347 // Target of jump will eventually return to caller.
14348 // TailJump below removes the return address.
14349 // Don't use rbp for 'jump_target' because a MachEpilogNode has already been
14350 // emitted just above the TailCall which has reset rbp to the caller state.
14351 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_ptr)
14352 %{
14353   match(TailCall jump_target method_ptr);
14354 
14355   ins_cost(300);
14356   format %{ "jmp     $jump_target\t# rbx holds method" %}
14357   ins_encode %{
14358     __ jmp($jump_target$$Register);
14359   %}
14360   ins_pipe(pipe_jmp);
14361 %}
14362 
14363 // Tail Jump; remove the return address; jump to target.
14364 // TailCall above leaves the return address around.
14365 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
14366 %{
14367   match(TailJump jump_target ex_oop);
14368 
14369   ins_cost(300);
14370   format %{ "popq    rdx\t# pop return address\n\t"
14371             "jmp     $jump_target" %}
14372   ins_encode %{
14373     __ popq(as_Register(RDX_enc));
14374     __ jmp($jump_target$$Register);
14375   %}
14376   ins_pipe(pipe_jmp);
14377 %}
14378 
14379 // Forward exception.
14380 instruct ForwardExceptionjmp()
14381 %{
14382   match(ForwardException);
14383 
14384   format %{ "jmp     forward_exception_stub" %}
14385   ins_encode %{
14386     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
14387   %}
14388   ins_pipe(pipe_jmp);
14389 %}
14390 
14391 // Create exception oop: created by stack-crawling runtime code.
14392 // Created exception is now available to this handler, and is setup
14393 // just prior to jumping to this handler.  No code emitted.
14394 instruct CreateException(rax_RegP ex_oop)
14395 %{
14396   match(Set ex_oop (CreateEx));
14397 
14398   size(0);
14399   // use the following format syntax
14400   format %{ "# exception oop is in rax; no code emitted" %}
14401   ins_encode();
14402   ins_pipe(empty);
14403 %}
14404 
14405 // Rethrow exception:
14406 // The exception oop will come in the first argument position.
14407 // Then JUMP (not call) to the rethrow stub code.
14408 instruct RethrowException()
14409 %{
14410   match(Rethrow);
14411 
14412   // use the following format syntax
14413   format %{ "jmp     rethrow_stub" %}
14414   ins_encode %{
14415     __ jump(RuntimeAddress(OptoRuntime::rethrow_stub()), noreg);
14416   %}
14417   ins_pipe(pipe_jmp);
14418 %}
14419 
14420 // ============================================================================
14421 // This name is KNOWN by the ADLC and cannot be changed.
14422 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14423 // for this guy.
14424 instruct tlsLoadP(r15_RegP dst) %{
14425   match(Set dst (ThreadLocal));
14426   effect(DEF dst);
14427 
14428   size(0);
14429   format %{ "# TLS is in R15" %}
14430   ins_encode( /*empty encoding*/ );
14431   ins_pipe(ialu_reg_reg);
14432 %}
14433 
14434 
14435 //----------PEEPHOLE RULES-----------------------------------------------------
14436 // These must follow all instruction definitions as they use the names
14437 // defined in the instructions definitions.
14438 //
14439 // peeppredicate ( rule_predicate );
14440 // // the predicate unless which the peephole rule will be ignored
14441 //
14442 // peepmatch ( root_instr_name [preceding_instruction]* );
14443 //
14444 // peepprocedure ( procedure_name );
14445 // // provide a procedure name to perform the optimization, the procedure should
14446 // // reside in the architecture dependent peephole file, the method has the
14447 // // signature of MachNode* (Block*, int, PhaseRegAlloc*, (MachNode*)(*)(), int...)
14448 // // with the arguments being the basic block, the current node index inside the
14449 // // block, the register allocator, the functions upon invoked return a new node
14450 // // defined in peepreplace, and the rules of the nodes appearing in the
14451 // // corresponding peepmatch, the function return true if successful, else
14452 // // return false
14453 //
14454 // peepconstraint %{
14455 // (instruction_number.operand_name relational_op instruction_number.operand_name
14456 //  [, ...] );
14457 // // instruction numbers are zero-based using left to right order in peepmatch
14458 //
14459 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14460 // // provide an instruction_number.operand_name for each operand that appears
14461 // // in the replacement instruction's match rule
14462 //
14463 // ---------VM FLAGS---------------------------------------------------------
14464 //
14465 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14466 //
14467 // Each peephole rule is given an identifying number starting with zero and
14468 // increasing by one in the order seen by the parser.  An individual peephole
14469 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14470 // on the command-line.
14471 //
14472 // ---------CURRENT LIMITATIONS----------------------------------------------
14473 //
14474 // Only transformations inside a basic block (do we need more for peephole)
14475 //
14476 // ---------EXAMPLE----------------------------------------------------------
14477 //
14478 // // pertinent parts of existing instructions in architecture description
14479 // instruct movI(rRegI dst, rRegI src)
14480 // %{
14481 //   match(Set dst (CopyI src));
14482 // %}
14483 //
14484 // instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
14485 // %{
14486 //   match(Set dst (AddI dst src));
14487 //   effect(KILL cr);
14488 // %}
14489 //
14490 // instruct leaI_rReg_immI(rRegI dst, immI_1 src)
14491 // %{
14492 //   match(Set dst (AddI dst src));
14493 // %}
14494 //
14495 // 1. Simple replacement
14496 // - Only match adjacent instructions in same basic block
14497 // - Only equality constraints
14498 // - Only constraints between operands, not (0.dest_reg == RAX_enc)
14499 // - Only one replacement instruction
14500 //
14501 // // Change (inc mov) to lea
14502 // peephole %{
14503 //   // lea should only be emitted when beneficial
14504 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14505 //   // increment preceded by register-register move
14506 //   peepmatch ( incI_rReg movI );
14507 //   // require that the destination register of the increment
14508 //   // match the destination register of the move
14509 //   peepconstraint ( 0.dst == 1.dst );
14510 //   // construct a replacement instruction that sets
14511 //   // the destination to ( move's source register + one )
14512 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
14513 // %}
14514 //
14515 // 2. Procedural replacement
14516 // - More flexible finding relevent nodes
14517 // - More flexible constraints
14518 // - More flexible transformations
14519 // - May utilise architecture-dependent API more effectively
14520 // - Currently only one replacement instruction due to adlc parsing capabilities
14521 //
14522 // // Change (inc mov) to lea
14523 // peephole %{
14524 //   // lea should only be emitted when beneficial
14525 //   peeppredicate( VM_Version::supports_fast_2op_lea() );
14526 //   // the rule numbers of these nodes inside are passed into the function below
14527 //   peepmatch ( incI_rReg movI );
14528 //   // the method that takes the responsibility of transformation
14529 //   peepprocedure ( inc_mov_to_lea );
14530 //   // the replacement is a leaI_rReg_immI, a lambda upon invoked creating this
14531 //   // node is passed into the function above
14532 //   peepreplace ( leaI_rReg_immI() );
14533 // %}
14534 
14535 // These instructions is not matched by the matcher but used by the peephole
14536 instruct leaI_rReg_rReg_peep(rRegI dst, rRegI src1, rRegI src2)
14537 %{
14538   predicate(false);
14539   match(Set dst (AddI src1 src2));
14540   format %{ "leal    $dst, [$src1 + $src2]" %}
14541   ins_encode %{
14542     Register dst = $dst$$Register;
14543     Register src1 = $src1$$Register;
14544     Register src2 = $src2$$Register;
14545     if (src1 != rbp && src1 != r13) {
14546       __ leal(dst, Address(src1, src2, Address::times_1));
14547     } else {
14548       assert(src2 != rbp && src2 != r13, "");
14549       __ leal(dst, Address(src2, src1, Address::times_1));
14550     }
14551   %}
14552   ins_pipe(ialu_reg_reg);
14553 %}
14554 
14555 instruct leaI_rReg_immI_peep(rRegI dst, rRegI src1, immI src2)
14556 %{
14557   predicate(false);
14558   match(Set dst (AddI src1 src2));
14559   format %{ "leal    $dst, [$src1 + $src2]" %}
14560   ins_encode %{
14561     __ leal($dst$$Register, Address($src1$$Register, $src2$$constant));
14562   %}
14563   ins_pipe(ialu_reg_reg);
14564 %}
14565 
14566 instruct leaI_rReg_immI2_peep(rRegI dst, rRegI src, immI2 shift)
14567 %{
14568   predicate(false);
14569   match(Set dst (LShiftI src shift));
14570   format %{ "leal    $dst, [$src << $shift]" %}
14571   ins_encode %{
14572     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14573     Register src = $src$$Register;
14574     if (scale == Address::times_2 && src != rbp && src != r13) {
14575       __ leal($dst$$Register, Address(src, src, Address::times_1));
14576     } else {
14577       __ leal($dst$$Register, Address(noreg, src, scale));
14578     }
14579   %}
14580   ins_pipe(ialu_reg_reg);
14581 %}
14582 
14583 instruct leaL_rReg_rReg_peep(rRegL dst, rRegL src1, rRegL src2)
14584 %{
14585   predicate(false);
14586   match(Set dst (AddL src1 src2));
14587   format %{ "leaq    $dst, [$src1 + $src2]" %}
14588   ins_encode %{
14589     Register dst = $dst$$Register;
14590     Register src1 = $src1$$Register;
14591     Register src2 = $src2$$Register;
14592     if (src1 != rbp && src1 != r13) {
14593       __ leaq(dst, Address(src1, src2, Address::times_1));
14594     } else {
14595       assert(src2 != rbp && src2 != r13, "");
14596       __ leaq(dst, Address(src2, src1, Address::times_1));
14597     }
14598   %}
14599   ins_pipe(ialu_reg_reg);
14600 %}
14601 
14602 instruct leaL_rReg_immL32_peep(rRegL dst, rRegL src1, immL32 src2)
14603 %{
14604   predicate(false);
14605   match(Set dst (AddL src1 src2));
14606   format %{ "leaq    $dst, [$src1 + $src2]" %}
14607   ins_encode %{
14608     __ leaq($dst$$Register, Address($src1$$Register, $src2$$constant));
14609   %}
14610   ins_pipe(ialu_reg_reg);
14611 %}
14612 
14613 instruct leaL_rReg_immI2_peep(rRegL dst, rRegL src, immI2 shift)
14614 %{
14615   predicate(false);
14616   match(Set dst (LShiftL src shift));
14617   format %{ "leaq    $dst, [$src << $shift]" %}
14618   ins_encode %{
14619     Address::ScaleFactor scale = static_cast<Address::ScaleFactor>($shift$$constant);
14620     Register src = $src$$Register;
14621     if (scale == Address::times_2 && src != rbp && src != r13) {
14622       __ leaq($dst$$Register, Address(src, src, Address::times_1));
14623     } else {
14624       __ leaq($dst$$Register, Address(noreg, src, scale));
14625     }
14626   %}
14627   ins_pipe(ialu_reg_reg);
14628 %}
14629 
14630 // These peephole rules replace mov + I pairs (where I is one of {add, inc, dec,
14631 // sal}) with lea instructions. The {add, sal} rules are beneficial in
14632 // processors with at least partial ALU support for lea
14633 // (supports_fast_2op_lea()), whereas the {inc, dec} rules are only generally
14634 // beneficial for processors with full ALU support
14635 // (VM_Version::supports_fast_3op_lea()) and Intel Cascade Lake.
14636 
14637 peephole
14638 %{
14639   peeppredicate(VM_Version::supports_fast_2op_lea());
14640   peepmatch (addI_rReg);
14641   peepprocedure (lea_coalesce_reg);
14642   peepreplace (leaI_rReg_rReg_peep());
14643 %}
14644 
14645 peephole
14646 %{
14647   peeppredicate(VM_Version::supports_fast_2op_lea());
14648   peepmatch (addI_rReg_imm);
14649   peepprocedure (lea_coalesce_imm);
14650   peepreplace (leaI_rReg_immI_peep());
14651 %}
14652 
14653 peephole
14654 %{
14655   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14656                 VM_Version::is_intel_cascade_lake());
14657   peepmatch (incI_rReg);
14658   peepprocedure (lea_coalesce_imm);
14659   peepreplace (leaI_rReg_immI_peep());
14660 %}
14661 
14662 peephole
14663 %{
14664   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14665                 VM_Version::is_intel_cascade_lake());
14666   peepmatch (decI_rReg);
14667   peepprocedure (lea_coalesce_imm);
14668   peepreplace (leaI_rReg_immI_peep());
14669 %}
14670 
14671 peephole
14672 %{
14673   peeppredicate(VM_Version::supports_fast_2op_lea());
14674   peepmatch (salI_rReg_immI2);
14675   peepprocedure (lea_coalesce_imm);
14676   peepreplace (leaI_rReg_immI2_peep());
14677 %}
14678 
14679 peephole
14680 %{
14681   peeppredicate(VM_Version::supports_fast_2op_lea());
14682   peepmatch (addL_rReg);
14683   peepprocedure (lea_coalesce_reg);
14684   peepreplace (leaL_rReg_rReg_peep());
14685 %}
14686 
14687 peephole
14688 %{
14689   peeppredicate(VM_Version::supports_fast_2op_lea());
14690   peepmatch (addL_rReg_imm);
14691   peepprocedure (lea_coalesce_imm);
14692   peepreplace (leaL_rReg_immL32_peep());
14693 %}
14694 
14695 peephole
14696 %{
14697   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14698                 VM_Version::is_intel_cascade_lake());
14699   peepmatch (incL_rReg);
14700   peepprocedure (lea_coalesce_imm);
14701   peepreplace (leaL_rReg_immL32_peep());
14702 %}
14703 
14704 peephole
14705 %{
14706   peeppredicate(VM_Version::supports_fast_3op_lea() ||
14707                 VM_Version::is_intel_cascade_lake());
14708   peepmatch (decL_rReg);
14709   peepprocedure (lea_coalesce_imm);
14710   peepreplace (leaL_rReg_immL32_peep());
14711 %}
14712 
14713 peephole
14714 %{
14715   peeppredicate(VM_Version::supports_fast_2op_lea());
14716   peepmatch (salL_rReg_immI2);
14717   peepprocedure (lea_coalesce_imm);
14718   peepreplace (leaL_rReg_immI2_peep());
14719 %}
14720 
14721 peephole
14722 %{
14723   peepmatch (leaPCompressedOopOffset);
14724   peepprocedure (lea_remove_redundant);
14725 %}
14726 
14727 peephole
14728 %{
14729   peepmatch (leaP8Narrow);
14730   peepprocedure (lea_remove_redundant);
14731 %}
14732 
14733 peephole
14734 %{
14735   peepmatch (leaP32Narrow);
14736   peepprocedure (lea_remove_redundant);
14737 %}
14738 
14739 // These peephole rules matches instructions which set flags and are followed by a testI/L_reg
14740 // The test instruction is redudanent in case the downstream instuctions (like JCC or CMOV) only use flags that are already set by the previous instruction
14741 
14742 //int variant
14743 peephole
14744 %{
14745   peepmatch (testI_reg);
14746   peepprocedure (test_may_remove);
14747 %}
14748 
14749 //long variant
14750 peephole
14751 %{
14752   peepmatch (testL_reg);
14753   peepprocedure (test_may_remove);
14754 %}
14755 
14756 
14757 //----------SMARTSPILL RULES---------------------------------------------------
14758 // These must follow all instruction definitions as they use the names
14759 // defined in the instructions definitions.